1 /* Execute compiled code */
2
3 /* XXX TO DO:
4 XXX speed up searching for keywords by using a dictionary
5 XXX document it!
6 */
7
8 /* enable more aggressive intra-module optimizations, where available */
9 /* affects both release and debug builds - see bpo-43271 */
10 #define PY_LOCAL_AGGRESSIVE
11
12 #include "Python.h"
13 #include "pycore_abstract.h" // _PyIndex_Check()
14 #include "pycore_call.h" // _PyObject_FastCallDictTstate()
15 #include "pycore_ceval.h" // _PyEval_SignalAsyncExc()
16 #include "pycore_code.h" // _PyCode_InitOpcache()
17 #include "pycore_initconfig.h" // _PyStatus_OK()
18 #include "pycore_object.h" // _PyObject_GC_TRACK()
19 #include "pycore_pyerrors.h" // _PyErr_Fetch()
20 #include "pycore_pylifecycle.h" // _PyErr_Print()
21 #include "pycore_pymem.h" // _PyMem_IsPtrFreed()
22 #include "pycore_pystate.h" // _PyInterpreterState_GET()
23 #include "pycore_sysmodule.h" // _PySys_Audit()
24 #include "pycore_tuple.h" // _PyTuple_ITEMS()
25
26 #include "code.h"
27 #include "dictobject.h"
28 #include "frameobject.h"
29 #include "opcode.h"
30 #include "pydtrace.h"
31 #include "setobject.h"
32 #include "structmember.h" // struct PyMemberDef, T_OFFSET_EX
33
34 #include <ctype.h>
35
36 typedef struct {
37 PyCodeObject *code; // The code object for the bounds. May be NULL.
38 PyCodeAddressRange bounds; // Only valid if code != NULL.
39 CFrame cframe;
40 } PyTraceInfo;
41
42
43 #ifdef Py_DEBUG
44 /* For debugging the interpreter: */
45 #define LLTRACE 1 /* Low-level trace feature */
46 #define CHECKEXC 1 /* Double-check exception checking */
47 #endif
48
49 #if !defined(Py_BUILD_CORE)
50 # error "ceval.c must be build with Py_BUILD_CORE define for best performance"
51 #endif
52
53 _Py_IDENTIFIER(__name__);
54
55 /* Forward declarations */
56 Py_LOCAL_INLINE(PyObject *) call_function(
57 PyThreadState *tstate, PyTraceInfo *, PyObject ***pp_stack,
58 Py_ssize_t oparg, PyObject *kwnames);
59 static PyObject * do_call_core(
60 PyThreadState *tstate, PyTraceInfo *, PyObject *func,
61 PyObject *callargs, PyObject *kwdict);
62
63 #ifdef LLTRACE
64 static int lltrace;
65 static int prtrace(PyThreadState *, PyObject *, const char *);
66 #endif
67 static int call_trace(Py_tracefunc, PyObject *,
68 PyThreadState *, PyFrameObject *,
69 PyTraceInfo *,
70 int, PyObject *);
71 static int call_trace_protected(Py_tracefunc, PyObject *,
72 PyThreadState *, PyFrameObject *,
73 PyTraceInfo *,
74 int, PyObject *);
75 static void call_exc_trace(Py_tracefunc, PyObject *,
76 PyThreadState *, PyFrameObject *,
77 PyTraceInfo *trace_info);
78 static int maybe_call_line_trace(Py_tracefunc, PyObject *,
79 PyThreadState *, PyFrameObject *,
80 PyTraceInfo *, int);
81 static void maybe_dtrace_line(PyFrameObject *, PyTraceInfo *, int);
82 static void dtrace_function_entry(PyFrameObject *);
83 static void dtrace_function_return(PyFrameObject *);
84
85 static PyObject * import_name(PyThreadState *, PyFrameObject *,
86 PyObject *, PyObject *, PyObject *);
87 static PyObject * import_from(PyThreadState *, PyObject *, PyObject *);
88 static int import_all_from(PyThreadState *, PyObject *, PyObject *);
89 static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *);
90 static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
91 static PyObject * unicode_concatenate(PyThreadState *, PyObject *, PyObject *,
92 PyFrameObject *, const _Py_CODEUNIT *);
93 static PyObject * special_lookup(PyThreadState *, PyObject *, _Py_Identifier *);
94 static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg);
95 static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs);
96 static void format_awaitable_error(PyThreadState *, PyTypeObject *, int, int);
97
98 #define NAME_ERROR_MSG \
99 "name '%.200s' is not defined"
100 #define UNBOUNDLOCAL_ERROR_MSG \
101 "local variable '%.200s' referenced before assignment"
102 #define UNBOUNDFREE_ERROR_MSG \
103 "free variable '%.200s' referenced before assignment" \
104 " in enclosing scope"
105
106 /* Dynamic execution profile */
107 #ifdef DYNAMIC_EXECUTION_PROFILE
108 #ifdef DXPAIRS
109 static long dxpairs[257][256];
110 #define dxp dxpairs[256]
111 #else
112 static long dxp[256];
113 #endif
114 #endif
115
116 /* per opcode cache */
117 static int opcache_min_runs = 1024; /* create opcache when code executed this many times */
118 #define OPCODE_CACHE_MAX_TRIES 20
119 #define OPCACHE_STATS 0 /* Enable stats */
120
121 // This function allows to deactivate the opcode cache. As different cache mechanisms may hold
122 // references, this can mess with the reference leak detector functionality so the cache needs
123 // to be deactivated in such scenarios to avoid false positives. See bpo-3714 for more information.
124 void
_PyEval_DeactivateOpCache(void)125 _PyEval_DeactivateOpCache(void)
126 {
127 opcache_min_runs = 0;
128 }
129
130 #if OPCACHE_STATS
131 static size_t opcache_code_objects = 0;
132 static size_t opcache_code_objects_extra_mem = 0;
133
134 static size_t opcache_global_opts = 0;
135 static size_t opcache_global_hits = 0;
136 static size_t opcache_global_misses = 0;
137
138 static size_t opcache_attr_opts = 0;
139 static size_t opcache_attr_hits = 0;
140 static size_t opcache_attr_misses = 0;
141 static size_t opcache_attr_deopts = 0;
142 static size_t opcache_attr_total = 0;
143 #endif
144
145
146 #ifndef NDEBUG
147 /* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and
148 PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen
149 when a thread continues to run after Python finalization, especially
150 daemon threads. */
151 static int
is_tstate_valid(PyThreadState * tstate)152 is_tstate_valid(PyThreadState *tstate)
153 {
154 assert(!_PyMem_IsPtrFreed(tstate));
155 assert(!_PyMem_IsPtrFreed(tstate->interp));
156 return 1;
157 }
158 #endif
159
160
161 /* This can set eval_breaker to 0 even though gil_drop_request became
162 1. We believe this is all right because the eval loop will release
163 the GIL eventually anyway. */
164 static inline void
COMPUTE_EVAL_BREAKER(PyInterpreterState * interp,struct _ceval_runtime_state * ceval,struct _ceval_state * ceval2)165 COMPUTE_EVAL_BREAKER(PyInterpreterState *interp,
166 struct _ceval_runtime_state *ceval,
167 struct _ceval_state *ceval2)
168 {
169 _Py_atomic_store_relaxed(&ceval2->eval_breaker,
170 _Py_atomic_load_relaxed(&ceval2->gil_drop_request)
171 | (_Py_atomic_load_relaxed(&ceval->signals_pending)
172 && _Py_ThreadCanHandleSignals(interp))
173 | (_Py_atomic_load_relaxed(&ceval2->pending.calls_to_do)
174 && _Py_ThreadCanHandlePendingCalls())
175 | ceval2->pending.async_exc);
176 }
177
178
179 static inline void
SET_GIL_DROP_REQUEST(PyInterpreterState * interp)180 SET_GIL_DROP_REQUEST(PyInterpreterState *interp)
181 {
182 struct _ceval_state *ceval2 = &interp->ceval;
183 _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 1);
184 _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
185 }
186
187
188 static inline void
RESET_GIL_DROP_REQUEST(PyInterpreterState * interp)189 RESET_GIL_DROP_REQUEST(PyInterpreterState *interp)
190 {
191 struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
192 struct _ceval_state *ceval2 = &interp->ceval;
193 _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 0);
194 COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
195 }
196
197
198 static inline void
SIGNAL_PENDING_CALLS(PyInterpreterState * interp)199 SIGNAL_PENDING_CALLS(PyInterpreterState *interp)
200 {
201 struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
202 struct _ceval_state *ceval2 = &interp->ceval;
203 _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 1);
204 COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
205 }
206
207
208 static inline void
UNSIGNAL_PENDING_CALLS(PyInterpreterState * interp)209 UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp)
210 {
211 struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
212 struct _ceval_state *ceval2 = &interp->ceval;
213 _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 0);
214 COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
215 }
216
217
218 static inline void
SIGNAL_PENDING_SIGNALS(PyInterpreterState * interp,int force)219 SIGNAL_PENDING_SIGNALS(PyInterpreterState *interp, int force)
220 {
221 struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
222 struct _ceval_state *ceval2 = &interp->ceval;
223 _Py_atomic_store_relaxed(&ceval->signals_pending, 1);
224 if (force) {
225 _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
226 }
227 else {
228 /* eval_breaker is not set to 1 if thread_can_handle_signals() is false */
229 COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
230 }
231 }
232
233
234 static inline void
UNSIGNAL_PENDING_SIGNALS(PyInterpreterState * interp)235 UNSIGNAL_PENDING_SIGNALS(PyInterpreterState *interp)
236 {
237 struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
238 struct _ceval_state *ceval2 = &interp->ceval;
239 _Py_atomic_store_relaxed(&ceval->signals_pending, 0);
240 COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
241 }
242
243
244 static inline void
SIGNAL_ASYNC_EXC(PyInterpreterState * interp)245 SIGNAL_ASYNC_EXC(PyInterpreterState *interp)
246 {
247 struct _ceval_state *ceval2 = &interp->ceval;
248 ceval2->pending.async_exc = 1;
249 _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
250 }
251
252
253 static inline void
UNSIGNAL_ASYNC_EXC(PyInterpreterState * interp)254 UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp)
255 {
256 struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
257 struct _ceval_state *ceval2 = &interp->ceval;
258 ceval2->pending.async_exc = 0;
259 COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
260 }
261
262
263 #ifdef HAVE_ERRNO_H
264 #include <errno.h>
265 #endif
266 #include "ceval_gil.h"
267
268 void _Py_NO_RETURN
_Py_FatalError_TstateNULL(const char * func)269 _Py_FatalError_TstateNULL(const char *func)
270 {
271 _Py_FatalErrorFunc(func,
272 "the function must be called with the GIL held, "
273 "but the GIL is released "
274 "(the current Python thread state is NULL)");
275 }
276
277 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
278 int
_PyEval_ThreadsInitialized(PyInterpreterState * interp)279 _PyEval_ThreadsInitialized(PyInterpreterState *interp)
280 {
281 return gil_created(&interp->ceval.gil);
282 }
283
284 int
PyEval_ThreadsInitialized(void)285 PyEval_ThreadsInitialized(void)
286 {
287 // Fatal error if there is no current interpreter
288 PyInterpreterState *interp = PyInterpreterState_Get();
289 return _PyEval_ThreadsInitialized(interp);
290 }
291 #else
292 int
_PyEval_ThreadsInitialized(_PyRuntimeState * runtime)293 _PyEval_ThreadsInitialized(_PyRuntimeState *runtime)
294 {
295 return gil_created(&runtime->ceval.gil);
296 }
297
298 int
PyEval_ThreadsInitialized(void)299 PyEval_ThreadsInitialized(void)
300 {
301 _PyRuntimeState *runtime = &_PyRuntime;
302 return _PyEval_ThreadsInitialized(runtime);
303 }
304 #endif
305
306 PyStatus
_PyEval_InitGIL(PyThreadState * tstate)307 _PyEval_InitGIL(PyThreadState *tstate)
308 {
309 #ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
310 if (!_Py_IsMainInterpreter(tstate->interp)) {
311 /* Currently, the GIL is shared by all interpreters,
312 and only the main interpreter is responsible to create
313 and destroy it. */
314 return _PyStatus_OK();
315 }
316 #endif
317
318 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
319 struct _gil_runtime_state *gil = &tstate->interp->ceval.gil;
320 #else
321 struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;
322 #endif
323 assert(!gil_created(gil));
324
325 PyThread_init_thread();
326 create_gil(gil);
327
328 take_gil(tstate);
329
330 assert(gil_created(gil));
331 return _PyStatus_OK();
332 }
333
334 void
_PyEval_FiniGIL(PyInterpreterState * interp)335 _PyEval_FiniGIL(PyInterpreterState *interp)
336 {
337 #ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
338 if (!_Py_IsMainInterpreter(interp)) {
339 /* Currently, the GIL is shared by all interpreters,
340 and only the main interpreter is responsible to create
341 and destroy it. */
342 return;
343 }
344 #endif
345
346 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
347 struct _gil_runtime_state *gil = &interp->ceval.gil;
348 #else
349 struct _gil_runtime_state *gil = &interp->runtime->ceval.gil;
350 #endif
351 if (!gil_created(gil)) {
352 /* First Py_InitializeFromConfig() call: the GIL doesn't exist
353 yet: do nothing. */
354 return;
355 }
356
357 destroy_gil(gil);
358 assert(!gil_created(gil));
359 }
360
361 void
PyEval_InitThreads(void)362 PyEval_InitThreads(void)
363 {
364 /* Do nothing: kept for backward compatibility */
365 }
366
367 void
_PyEval_Fini(void)368 _PyEval_Fini(void)
369 {
370 #if OPCACHE_STATS
371 fprintf(stderr, "-- Opcode cache number of objects = %zd\n",
372 opcache_code_objects);
373
374 fprintf(stderr, "-- Opcode cache total extra mem = %zd\n",
375 opcache_code_objects_extra_mem);
376
377 fprintf(stderr, "\n");
378
379 fprintf(stderr, "-- Opcode cache LOAD_GLOBAL hits = %zd (%d%%)\n",
380 opcache_global_hits,
381 (int) (100.0 * opcache_global_hits /
382 (opcache_global_hits + opcache_global_misses)));
383
384 fprintf(stderr, "-- Opcode cache LOAD_GLOBAL misses = %zd (%d%%)\n",
385 opcache_global_misses,
386 (int) (100.0 * opcache_global_misses /
387 (opcache_global_hits + opcache_global_misses)));
388
389 fprintf(stderr, "-- Opcode cache LOAD_GLOBAL opts = %zd\n",
390 opcache_global_opts);
391
392 fprintf(stderr, "\n");
393
394 fprintf(stderr, "-- Opcode cache LOAD_ATTR hits = %zd (%d%%)\n",
395 opcache_attr_hits,
396 (int) (100.0 * opcache_attr_hits /
397 opcache_attr_total));
398
399 fprintf(stderr, "-- Opcode cache LOAD_ATTR misses = %zd (%d%%)\n",
400 opcache_attr_misses,
401 (int) (100.0 * opcache_attr_misses /
402 opcache_attr_total));
403
404 fprintf(stderr, "-- Opcode cache LOAD_ATTR opts = %zd\n",
405 opcache_attr_opts);
406
407 fprintf(stderr, "-- Opcode cache LOAD_ATTR deopts = %zd\n",
408 opcache_attr_deopts);
409
410 fprintf(stderr, "-- Opcode cache LOAD_ATTR total = %zd\n",
411 opcache_attr_total);
412 #endif
413 }
414
415 void
PyEval_AcquireLock(void)416 PyEval_AcquireLock(void)
417 {
418 _PyRuntimeState *runtime = &_PyRuntime;
419 PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
420 _Py_EnsureTstateNotNULL(tstate);
421
422 take_gil(tstate);
423 }
424
425 void
PyEval_ReleaseLock(void)426 PyEval_ReleaseLock(void)
427 {
428 _PyRuntimeState *runtime = &_PyRuntime;
429 PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
430 /* This function must succeed when the current thread state is NULL.
431 We therefore avoid PyThreadState_Get() which dumps a fatal error
432 in debug mode. */
433 struct _ceval_runtime_state *ceval = &runtime->ceval;
434 struct _ceval_state *ceval2 = &tstate->interp->ceval;
435 drop_gil(ceval, ceval2, tstate);
436 }
437
438 void
_PyEval_ReleaseLock(PyThreadState * tstate)439 _PyEval_ReleaseLock(PyThreadState *tstate)
440 {
441 struct _ceval_runtime_state *ceval = &tstate->interp->runtime->ceval;
442 struct _ceval_state *ceval2 = &tstate->interp->ceval;
443 drop_gil(ceval, ceval2, tstate);
444 }
445
446 void
PyEval_AcquireThread(PyThreadState * tstate)447 PyEval_AcquireThread(PyThreadState *tstate)
448 {
449 _Py_EnsureTstateNotNULL(tstate);
450
451 take_gil(tstate);
452
453 struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;
454 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
455 (void)_PyThreadState_Swap(gilstate, tstate);
456 #else
457 if (_PyThreadState_Swap(gilstate, tstate) != NULL) {
458 Py_FatalError("non-NULL old thread state");
459 }
460 #endif
461 }
462
463 void
PyEval_ReleaseThread(PyThreadState * tstate)464 PyEval_ReleaseThread(PyThreadState *tstate)
465 {
466 assert(is_tstate_valid(tstate));
467
468 _PyRuntimeState *runtime = tstate->interp->runtime;
469 PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);
470 if (new_tstate != tstate) {
471 Py_FatalError("wrong thread state");
472 }
473 struct _ceval_runtime_state *ceval = &runtime->ceval;
474 struct _ceval_state *ceval2 = &tstate->interp->ceval;
475 drop_gil(ceval, ceval2, tstate);
476 }
477
478 #ifdef HAVE_FORK
479 /* This function is called from PyOS_AfterFork_Child to destroy all threads
480 which are not running in the child process, and clear internal locks
481 which might be held by those threads. */
482 PyStatus
_PyEval_ReInitThreads(PyThreadState * tstate)483 _PyEval_ReInitThreads(PyThreadState *tstate)
484 {
485 _PyRuntimeState *runtime = tstate->interp->runtime;
486
487 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
488 struct _gil_runtime_state *gil = &tstate->interp->ceval.gil;
489 #else
490 struct _gil_runtime_state *gil = &runtime->ceval.gil;
491 #endif
492 if (!gil_created(gil)) {
493 return _PyStatus_OK();
494 }
495 recreate_gil(gil);
496
497 take_gil(tstate);
498
499 struct _pending_calls *pending = &tstate->interp->ceval.pending;
500 if (_PyThread_at_fork_reinit(&pending->lock) < 0) {
501 return _PyStatus_ERR("Can't reinitialize pending calls lock");
502 }
503
504 /* Destroy all threads except the current one */
505 _PyThreadState_DeleteExcept(runtime, tstate);
506 return _PyStatus_OK();
507 }
508 #endif
509
510 /* This function is used to signal that async exceptions are waiting to be
511 raised. */
512
513 void
_PyEval_SignalAsyncExc(PyInterpreterState * interp)514 _PyEval_SignalAsyncExc(PyInterpreterState *interp)
515 {
516 SIGNAL_ASYNC_EXC(interp);
517 }
518
519 PyThreadState *
PyEval_SaveThread(void)520 PyEval_SaveThread(void)
521 {
522 _PyRuntimeState *runtime = &_PyRuntime;
523 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
524 PyThreadState *old_tstate = _PyThreadState_GET();
525 PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, old_tstate);
526 #else
527 PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);
528 #endif
529 _Py_EnsureTstateNotNULL(tstate);
530
531 struct _ceval_runtime_state *ceval = &runtime->ceval;
532 struct _ceval_state *ceval2 = &tstate->interp->ceval;
533 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
534 assert(gil_created(&ceval2->gil));
535 #else
536 assert(gil_created(&ceval->gil));
537 #endif
538 drop_gil(ceval, ceval2, tstate);
539 return tstate;
540 }
541
542 void
PyEval_RestoreThread(PyThreadState * tstate)543 PyEval_RestoreThread(PyThreadState *tstate)
544 {
545 _Py_EnsureTstateNotNULL(tstate);
546
547 take_gil(tstate);
548
549 struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;
550 _PyThreadState_Swap(gilstate, tstate);
551 }
552
553
554 /* Mechanism whereby asynchronously executing callbacks (e.g. UNIX
555 signal handlers or Mac I/O completion routines) can schedule calls
556 to a function to be called synchronously.
557 The synchronous function is called with one void* argument.
558 It should return 0 for success or -1 for failure -- failure should
559 be accompanied by an exception.
560
561 If registry succeeds, the registry function returns 0; if it fails
562 (e.g. due to too many pending calls) it returns -1 (without setting
563 an exception condition).
564
565 Note that because registry may occur from within signal handlers,
566 or other asynchronous events, calling malloc() is unsafe!
567
568 Any thread can schedule pending calls, but only the main thread
569 will execute them.
570 There is no facility to schedule calls to a particular thread, but
571 that should be easy to change, should that ever be required. In
572 that case, the static variables here should go into the python
573 threadstate.
574 */
575
576 void
_PyEval_SignalReceived(PyInterpreterState * interp)577 _PyEval_SignalReceived(PyInterpreterState *interp)
578 {
579 #ifdef MS_WINDOWS
580 // bpo-42296: On Windows, _PyEval_SignalReceived() is called from a signal
581 // handler which can run in a thread different than the Python thread, in
582 // which case _Py_ThreadCanHandleSignals() is wrong. Ignore
583 // _Py_ThreadCanHandleSignals() and always set eval_breaker to 1.
584 //
585 // The next eval_frame_handle_pending() call will call
586 // _Py_ThreadCanHandleSignals() to recompute eval_breaker.
587 int force = 1;
588 #else
589 int force = 0;
590 #endif
591 /* bpo-30703: Function called when the C signal handler of Python gets a
592 signal. We cannot queue a callback using _PyEval_AddPendingCall() since
593 that function is not async-signal-safe. */
594 SIGNAL_PENDING_SIGNALS(interp, force);
595 }
596
597 /* Push one item onto the queue while holding the lock. */
598 static int
_push_pending_call(struct _pending_calls * pending,int (* func)(void *),void * arg)599 _push_pending_call(struct _pending_calls *pending,
600 int (*func)(void *), void *arg)
601 {
602 int i = pending->last;
603 int j = (i + 1) % NPENDINGCALLS;
604 if (j == pending->first) {
605 return -1; /* Queue full */
606 }
607 pending->calls[i].func = func;
608 pending->calls[i].arg = arg;
609 pending->last = j;
610 return 0;
611 }
612
613 /* Pop one item off the queue while holding the lock. */
614 static void
_pop_pending_call(struct _pending_calls * pending,int (** func)(void *),void ** arg)615 _pop_pending_call(struct _pending_calls *pending,
616 int (**func)(void *), void **arg)
617 {
618 int i = pending->first;
619 if (i == pending->last) {
620 return; /* Queue empty */
621 }
622
623 *func = pending->calls[i].func;
624 *arg = pending->calls[i].arg;
625 pending->first = (i + 1) % NPENDINGCALLS;
626 }
627
628 /* This implementation is thread-safe. It allows
629 scheduling to be made from any thread, and even from an executing
630 callback.
631 */
632
633 int
_PyEval_AddPendingCall(PyInterpreterState * interp,int (* func)(void *),void * arg)634 _PyEval_AddPendingCall(PyInterpreterState *interp,
635 int (*func)(void *), void *arg)
636 {
637 struct _pending_calls *pending = &interp->ceval.pending;
638
639 /* Ensure that _PyEval_InitPendingCalls() was called
640 and that _PyEval_FiniPendingCalls() is not called yet. */
641 assert(pending->lock != NULL);
642
643 PyThread_acquire_lock(pending->lock, WAIT_LOCK);
644 int result = _push_pending_call(pending, func, arg);
645 PyThread_release_lock(pending->lock);
646
647 /* signal main loop */
648 SIGNAL_PENDING_CALLS(interp);
649 return result;
650 }
651
652 int
Py_AddPendingCall(int (* func)(void *),void * arg)653 Py_AddPendingCall(int (*func)(void *), void *arg)
654 {
655 /* Best-effort to support subinterpreters and calls with the GIL released.
656
657 First attempt _PyThreadState_GET() since it supports subinterpreters.
658
659 If the GIL is released, _PyThreadState_GET() returns NULL . In this
660 case, use PyGILState_GetThisThreadState() which works even if the GIL
661 is released.
662
663 Sadly, PyGILState_GetThisThreadState() doesn't support subinterpreters:
664 see bpo-10915 and bpo-15751.
665
666 Py_AddPendingCall() doesn't require the caller to hold the GIL. */
667 PyThreadState *tstate = _PyThreadState_GET();
668 if (tstate == NULL) {
669 tstate = PyGILState_GetThisThreadState();
670 }
671
672 PyInterpreterState *interp;
673 if (tstate != NULL) {
674 interp = tstate->interp;
675 }
676 else {
677 /* Last resort: use the main interpreter */
678 interp = _PyRuntime.interpreters.main;
679 }
680 return _PyEval_AddPendingCall(interp, func, arg);
681 }
682
683 static int
handle_signals(PyThreadState * tstate)684 handle_signals(PyThreadState *tstate)
685 {
686 assert(is_tstate_valid(tstate));
687 if (!_Py_ThreadCanHandleSignals(tstate->interp)) {
688 return 0;
689 }
690
691 UNSIGNAL_PENDING_SIGNALS(tstate->interp);
692 if (_PyErr_CheckSignalsTstate(tstate) < 0) {
693 /* On failure, re-schedule a call to handle_signals(). */
694 SIGNAL_PENDING_SIGNALS(tstate->interp, 0);
695 return -1;
696 }
697 return 0;
698 }
699
700 static int
make_pending_calls(PyInterpreterState * interp)701 make_pending_calls(PyInterpreterState *interp)
702 {
703 /* only execute pending calls on main thread */
704 if (!_Py_ThreadCanHandlePendingCalls()) {
705 return 0;
706 }
707
708 /* don't perform recursive pending calls */
709 static int busy = 0;
710 if (busy) {
711 return 0;
712 }
713 busy = 1;
714
715 /* unsignal before starting to call callbacks, so that any callback
716 added in-between re-signals */
717 UNSIGNAL_PENDING_CALLS(interp);
718 int res = 0;
719
720 /* perform a bounded number of calls, in case of recursion */
721 struct _pending_calls *pending = &interp->ceval.pending;
722 for (int i=0; i<NPENDINGCALLS; i++) {
723 int (*func)(void *) = NULL;
724 void *arg = NULL;
725
726 /* pop one item off the queue while holding the lock */
727 PyThread_acquire_lock(pending->lock, WAIT_LOCK);
728 _pop_pending_call(pending, &func, &arg);
729 PyThread_release_lock(pending->lock);
730
731 /* having released the lock, perform the callback */
732 if (func == NULL) {
733 break;
734 }
735 res = func(arg);
736 if (res) {
737 goto error;
738 }
739 }
740
741 busy = 0;
742 return res;
743
744 error:
745 busy = 0;
746 SIGNAL_PENDING_CALLS(interp);
747 return res;
748 }
749
750 void
_Py_FinishPendingCalls(PyThreadState * tstate)751 _Py_FinishPendingCalls(PyThreadState *tstate)
752 {
753 assert(PyGILState_Check());
754 assert(is_tstate_valid(tstate));
755
756 struct _pending_calls *pending = &tstate->interp->ceval.pending;
757
758 if (!_Py_atomic_load_relaxed(&(pending->calls_to_do))) {
759 return;
760 }
761
762 if (make_pending_calls(tstate->interp) < 0) {
763 PyObject *exc, *val, *tb;
764 _PyErr_Fetch(tstate, &exc, &val, &tb);
765 PyErr_BadInternalCall();
766 _PyErr_ChainExceptions(exc, val, tb);
767 _PyErr_Print(tstate);
768 }
769 }
770
771 /* Py_MakePendingCalls() is a simple wrapper for the sake
772 of backward-compatibility. */
773 int
Py_MakePendingCalls(void)774 Py_MakePendingCalls(void)
775 {
776 assert(PyGILState_Check());
777
778 PyThreadState *tstate = _PyThreadState_GET();
779 assert(is_tstate_valid(tstate));
780
781 /* Python signal handler doesn't really queue a callback: it only signals
782 that a signal was received, see _PyEval_SignalReceived(). */
783 int res = handle_signals(tstate);
784 if (res != 0) {
785 return res;
786 }
787
788 res = make_pending_calls(tstate->interp);
789 if (res != 0) {
790 return res;
791 }
792
793 return 0;
794 }
795
796 /* The interpreter's recursion limit */
797
798 #ifndef Py_DEFAULT_RECURSION_LIMIT
799 # define Py_DEFAULT_RECURSION_LIMIT 1000
800 #endif
801
802 void
_PyEval_InitRuntimeState(struct _ceval_runtime_state * ceval)803 _PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval)
804 {
805 #ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
806 _gil_initialize(&ceval->gil);
807 #endif
808 }
809
810 int
_PyEval_InitState(struct _ceval_state * ceval)811 _PyEval_InitState(struct _ceval_state *ceval)
812 {
813 ceval->recursion_limit = Py_DEFAULT_RECURSION_LIMIT;
814
815 struct _pending_calls *pending = &ceval->pending;
816 assert(pending->lock == NULL);
817
818 pending->lock = PyThread_allocate_lock();
819 if (pending->lock == NULL) {
820 return -1;
821 }
822
823 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
824 _gil_initialize(&ceval->gil);
825 #endif
826
827 return 0;
828 }
829
830 void
_PyEval_FiniState(struct _ceval_state * ceval)831 _PyEval_FiniState(struct _ceval_state *ceval)
832 {
833 struct _pending_calls *pending = &ceval->pending;
834 if (pending->lock != NULL) {
835 PyThread_free_lock(pending->lock);
836 pending->lock = NULL;
837 }
838 }
839
840 int
Py_GetRecursionLimit(void)841 Py_GetRecursionLimit(void)
842 {
843 PyInterpreterState *interp = _PyInterpreterState_GET();
844 return interp->ceval.recursion_limit;
845 }
846
847 void
Py_SetRecursionLimit(int new_limit)848 Py_SetRecursionLimit(int new_limit)
849 {
850 PyThreadState *tstate = _PyThreadState_GET();
851 tstate->interp->ceval.recursion_limit = new_limit;
852 }
853
854 /* The function _Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall()
855 if the recursion_depth reaches recursion_limit.
856 If USE_STACKCHECK, the macro decrements recursion_limit
857 to guarantee that _Py_CheckRecursiveCall() is regularly called.
858 Without USE_STACKCHECK, there is no need for this. */
859 int
_Py_CheckRecursiveCall(PyThreadState * tstate,const char * where)860 _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
861 {
862 int recursion_limit = tstate->interp->ceval.recursion_limit;
863
864 #ifdef USE_STACKCHECK
865 tstate->stackcheck_counter = 0;
866 if (PyOS_CheckStack()) {
867 --tstate->recursion_depth;
868 _PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow");
869 return -1;
870 }
871 #endif
872 if (tstate->recursion_headroom) {
873 if (tstate->recursion_depth > recursion_limit + 50) {
874 /* Overflowing while handling an overflow. Give up. */
875 Py_FatalError("Cannot recover from stack overflow.");
876 }
877 }
878 else {
879 if (tstate->recursion_depth > recursion_limit) {
880 tstate->recursion_headroom++;
881 _PyErr_Format(tstate, PyExc_RecursionError,
882 "maximum recursion depth exceeded%s",
883 where);
884 tstate->recursion_headroom--;
885 --tstate->recursion_depth;
886 return -1;
887 }
888 }
889 return 0;
890 }
891
892
893 // PEP 634: Structural Pattern Matching
894
895
896 // Return a tuple of values corresponding to keys, with error checks for
897 // duplicate/missing keys.
898 static PyObject*
match_keys(PyThreadState * tstate,PyObject * map,PyObject * keys)899 match_keys(PyThreadState *tstate, PyObject *map, PyObject *keys)
900 {
901 assert(PyTuple_CheckExact(keys));
902 Py_ssize_t nkeys = PyTuple_GET_SIZE(keys);
903 if (!nkeys) {
904 // No keys means no items.
905 return PyTuple_New(0);
906 }
907 PyObject *seen = NULL;
908 PyObject *dummy = NULL;
909 PyObject *values = NULL;
910 // We use the two argument form of map.get(key, default) for two reasons:
911 // - Atomically check for a key and get its value without error handling.
912 // - Don't cause key creation or resizing in dict subclasses like
913 // collections.defaultdict that define __missing__ (or similar).
914 _Py_IDENTIFIER(get);
915 PyObject *get = _PyObject_GetAttrId(map, &PyId_get);
916 if (get == NULL) {
917 goto fail;
918 }
919 seen = PySet_New(NULL);
920 if (seen == NULL) {
921 goto fail;
922 }
923 // dummy = object()
924 dummy = _PyObject_CallNoArg((PyObject *)&PyBaseObject_Type);
925 if (dummy == NULL) {
926 goto fail;
927 }
928 values = PyList_New(0);
929 if (values == NULL) {
930 goto fail;
931 }
932 for (Py_ssize_t i = 0; i < nkeys; i++) {
933 PyObject *key = PyTuple_GET_ITEM(keys, i);
934 if (PySet_Contains(seen, key) || PySet_Add(seen, key)) {
935 if (!_PyErr_Occurred(tstate)) {
936 // Seen it before!
937 _PyErr_Format(tstate, PyExc_ValueError,
938 "mapping pattern checks duplicate key (%R)", key);
939 }
940 goto fail;
941 }
942 PyObject *value = PyObject_CallFunctionObjArgs(get, key, dummy, NULL);
943 if (value == NULL) {
944 goto fail;
945 }
946 if (value == dummy) {
947 // key not in map!
948 Py_DECREF(value);
949 Py_DECREF(values);
950 // Return None:
951 Py_INCREF(Py_None);
952 values = Py_None;
953 goto done;
954 }
955 PyList_Append(values, value);
956 Py_DECREF(value);
957 }
958 Py_SETREF(values, PyList_AsTuple(values));
959 // Success:
960 done:
961 Py_DECREF(get);
962 Py_DECREF(seen);
963 Py_DECREF(dummy);
964 return values;
965 fail:
966 Py_XDECREF(get);
967 Py_XDECREF(seen);
968 Py_XDECREF(dummy);
969 Py_XDECREF(values);
970 return NULL;
971 }
972
973 // Extract a named attribute from the subject, with additional bookkeeping to
974 // raise TypeErrors for repeated lookups. On failure, return NULL (with no
975 // error set). Use _PyErr_Occurred(tstate) to disambiguate.
976 static PyObject*
match_class_attr(PyThreadState * tstate,PyObject * subject,PyObject * type,PyObject * name,PyObject * seen)977 match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type,
978 PyObject *name, PyObject *seen)
979 {
980 assert(PyUnicode_CheckExact(name));
981 assert(PySet_CheckExact(seen));
982 if (PySet_Contains(seen, name) || PySet_Add(seen, name)) {
983 if (!_PyErr_Occurred(tstate)) {
984 // Seen it before!
985 _PyErr_Format(tstate, PyExc_TypeError,
986 "%s() got multiple sub-patterns for attribute %R",
987 ((PyTypeObject*)type)->tp_name, name);
988 }
989 return NULL;
990 }
991 PyObject *attr = PyObject_GetAttr(subject, name);
992 if (attr == NULL && _PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
993 _PyErr_Clear(tstate);
994 }
995 return attr;
996 }
997
998 // On success (match), return a tuple of extracted attributes. On failure (no
999 // match), return NULL. Use _PyErr_Occurred(tstate) to disambiguate.
1000 static PyObject*
match_class(PyThreadState * tstate,PyObject * subject,PyObject * type,Py_ssize_t nargs,PyObject * kwargs)1001 match_class(PyThreadState *tstate, PyObject *subject, PyObject *type,
1002 Py_ssize_t nargs, PyObject *kwargs)
1003 {
1004 if (!PyType_Check(type)) {
1005 const char *e = "called match pattern must be a type";
1006 _PyErr_Format(tstate, PyExc_TypeError, e);
1007 return NULL;
1008 }
1009 assert(PyTuple_CheckExact(kwargs));
1010 // First, an isinstance check:
1011 if (PyObject_IsInstance(subject, type) <= 0) {
1012 return NULL;
1013 }
1014 // So far so good:
1015 PyObject *seen = PySet_New(NULL);
1016 if (seen == NULL) {
1017 return NULL;
1018 }
1019 PyObject *attrs = PyList_New(0);
1020 if (attrs == NULL) {
1021 Py_DECREF(seen);
1022 return NULL;
1023 }
1024 // NOTE: From this point on, goto fail on failure:
1025 PyObject *match_args = NULL;
1026 // First, the positional subpatterns:
1027 if (nargs) {
1028 int match_self = 0;
1029 match_args = PyObject_GetAttrString(type, "__match_args__");
1030 if (match_args) {
1031 if (!PyTuple_CheckExact(match_args)) {
1032 const char *e = "%s.__match_args__ must be a tuple (got %s)";
1033 _PyErr_Format(tstate, PyExc_TypeError, e,
1034 ((PyTypeObject *)type)->tp_name,
1035 Py_TYPE(match_args)->tp_name);
1036 goto fail;
1037 }
1038 }
1039 else if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
1040 _PyErr_Clear(tstate);
1041 // _Py_TPFLAGS_MATCH_SELF is only acknowledged if the type does not
1042 // define __match_args__. This is natural behavior for subclasses:
1043 // it's as if __match_args__ is some "magic" value that is lost as
1044 // soon as they redefine it.
1045 match_args = PyTuple_New(0);
1046 match_self = PyType_HasFeature((PyTypeObject*)type,
1047 _Py_TPFLAGS_MATCH_SELF);
1048 }
1049 else {
1050 goto fail;
1051 }
1052 assert(PyTuple_CheckExact(match_args));
1053 Py_ssize_t allowed = match_self ? 1 : PyTuple_GET_SIZE(match_args);
1054 if (allowed < nargs) {
1055 const char *plural = (allowed == 1) ? "" : "s";
1056 _PyErr_Format(tstate, PyExc_TypeError,
1057 "%s() accepts %d positional sub-pattern%s (%d given)",
1058 ((PyTypeObject*)type)->tp_name,
1059 allowed, plural, nargs);
1060 goto fail;
1061 }
1062 if (match_self) {
1063 // Easy. Copy the subject itself, and move on to kwargs.
1064 PyList_Append(attrs, subject);
1065 }
1066 else {
1067 for (Py_ssize_t i = 0; i < nargs; i++) {
1068 PyObject *name = PyTuple_GET_ITEM(match_args, i);
1069 if (!PyUnicode_CheckExact(name)) {
1070 _PyErr_Format(tstate, PyExc_TypeError,
1071 "__match_args__ elements must be strings "
1072 "(got %s)", Py_TYPE(name)->tp_name);
1073 goto fail;
1074 }
1075 PyObject *attr = match_class_attr(tstate, subject, type, name,
1076 seen);
1077 if (attr == NULL) {
1078 goto fail;
1079 }
1080 PyList_Append(attrs, attr);
1081 Py_DECREF(attr);
1082 }
1083 }
1084 Py_CLEAR(match_args);
1085 }
1086 // Finally, the keyword subpatterns:
1087 for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwargs); i++) {
1088 PyObject *name = PyTuple_GET_ITEM(kwargs, i);
1089 PyObject *attr = match_class_attr(tstate, subject, type, name, seen);
1090 if (attr == NULL) {
1091 goto fail;
1092 }
1093 PyList_Append(attrs, attr);
1094 Py_DECREF(attr);
1095 }
1096 Py_SETREF(attrs, PyList_AsTuple(attrs));
1097 Py_DECREF(seen);
1098 return attrs;
1099 fail:
1100 // We really don't care whether an error was raised or not... that's our
1101 // caller's problem. All we know is that the match failed.
1102 Py_XDECREF(match_args);
1103 Py_DECREF(seen);
1104 Py_DECREF(attrs);
1105 return NULL;
1106 }
1107
1108
1109 static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause);
1110 static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **);
1111
1112
1113 PyObject *
PyEval_EvalCode(PyObject * co,PyObject * globals,PyObject * locals)1114 PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)
1115 {
1116 PyThreadState *tstate = PyThreadState_GET();
1117 if (locals == NULL) {
1118 locals = globals;
1119 }
1120 PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref
1121 if (builtins == NULL) {
1122 return NULL;
1123 }
1124 PyFrameConstructor desc = {
1125 .fc_globals = globals,
1126 .fc_builtins = builtins,
1127 .fc_name = ((PyCodeObject *)co)->co_name,
1128 .fc_qualname = ((PyCodeObject *)co)->co_name,
1129 .fc_code = co,
1130 .fc_defaults = NULL,
1131 .fc_kwdefaults = NULL,
1132 .fc_closure = NULL
1133 };
1134 return _PyEval_Vector(tstate, &desc, locals, NULL, 0, NULL);
1135 }
1136
1137
1138 /* Interpreter main loop */
1139
1140 PyObject *
PyEval_EvalFrame(PyFrameObject * f)1141 PyEval_EvalFrame(PyFrameObject *f)
1142 {
1143 /* Function kept for backward compatibility */
1144 PyThreadState *tstate = _PyThreadState_GET();
1145 return _PyEval_EvalFrame(tstate, f, 0);
1146 }
1147
1148 PyObject *
PyEval_EvalFrameEx(PyFrameObject * f,int throwflag)1149 PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
1150 {
1151 PyThreadState *tstate = _PyThreadState_GET();
1152 return _PyEval_EvalFrame(tstate, f, throwflag);
1153 }
1154
1155
1156 /* Handle signals, pending calls, GIL drop request
1157 and asynchronous exception */
1158 static int
eval_frame_handle_pending(PyThreadState * tstate)1159 eval_frame_handle_pending(PyThreadState *tstate)
1160 {
1161 _PyRuntimeState * const runtime = &_PyRuntime;
1162 struct _ceval_runtime_state *ceval = &runtime->ceval;
1163
1164 /* Pending signals */
1165 if (_Py_atomic_load_relaxed(&ceval->signals_pending)) {
1166 if (handle_signals(tstate) != 0) {
1167 return -1;
1168 }
1169 }
1170
1171 /* Pending calls */
1172 struct _ceval_state *ceval2 = &tstate->interp->ceval;
1173 if (_Py_atomic_load_relaxed(&ceval2->pending.calls_to_do)) {
1174 if (make_pending_calls(tstate->interp) != 0) {
1175 return -1;
1176 }
1177 }
1178
1179 /* GIL drop request */
1180 if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) {
1181 /* Give another thread a chance */
1182 if (_PyThreadState_Swap(&runtime->gilstate, NULL) != tstate) {
1183 Py_FatalError("tstate mix-up");
1184 }
1185 drop_gil(ceval, ceval2, tstate);
1186
1187 /* Other threads may run now */
1188
1189 take_gil(tstate);
1190
1191 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
1192 (void)_PyThreadState_Swap(&runtime->gilstate, tstate);
1193 #else
1194 if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) {
1195 Py_FatalError("orphan tstate");
1196 }
1197 #endif
1198 }
1199
1200 /* Check for asynchronous exception. */
1201 if (tstate->async_exc != NULL) {
1202 PyObject *exc = tstate->async_exc;
1203 tstate->async_exc = NULL;
1204 UNSIGNAL_ASYNC_EXC(tstate->interp);
1205 _PyErr_SetNone(tstate, exc);
1206 Py_DECREF(exc);
1207 return -1;
1208 }
1209
1210 #ifdef MS_WINDOWS
1211 // bpo-42296: On Windows, _PyEval_SignalReceived() can be called in a
1212 // different thread than the Python thread, in which case
1213 // _Py_ThreadCanHandleSignals() is wrong. Recompute eval_breaker in the
1214 // current Python thread with the correct _Py_ThreadCanHandleSignals()
1215 // value. It prevents to interrupt the eval loop at every instruction if
1216 // the current Python thread cannot handle signals (if
1217 // _Py_ThreadCanHandleSignals() is false).
1218 COMPUTE_EVAL_BREAKER(tstate->interp, ceval, ceval2);
1219 #endif
1220
1221 return 0;
1222 }
1223
1224
1225 /* Computed GOTOs, or
1226 the-optimization-commonly-but-improperly-known-as-"threaded code"
1227 using gcc's labels-as-values extension
1228 (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).
1229
1230 The traditional bytecode evaluation loop uses a "switch" statement, which
1231 decent compilers will optimize as a single indirect branch instruction
1232 combined with a lookup table of jump addresses. However, since the
1233 indirect jump instruction is shared by all opcodes, the CPU will have a
1234 hard time making the right prediction for where to jump next (actually,
1235 it will be always wrong except in the uncommon case of a sequence of
1236 several identical opcodes).
1237
1238 "Threaded code" in contrast, uses an explicit jump table and an explicit
1239 indirect jump instruction at the end of each opcode. Since the jump
1240 instruction is at a different address for each opcode, the CPU will make a
1241 separate prediction for each of these instructions, which is equivalent to
1242 predicting the second opcode of each opcode pair. These predictions have
1243 a much better chance to turn out valid, especially in small bytecode loops.
1244
1245 A mispredicted branch on a modern CPU flushes the whole pipeline and
1246 can cost several CPU cycles (depending on the pipeline depth),
1247 and potentially many more instructions (depending on the pipeline width).
1248 A correctly predicted branch, however, is nearly free.
1249
1250 At the time of this writing, the "threaded code" version is up to 15-20%
1251 faster than the normal "switch" version, depending on the compiler and the
1252 CPU architecture.
1253
1254 We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined,
1255 because it would render the measurements invalid.
1256
1257
1258 NOTE: care must be taken that the compiler doesn't try to "optimize" the
1259 indirect jumps by sharing them between all opcodes. Such optimizations
1260 can be disabled on gcc by using the -fno-gcse flag (or possibly
1261 -fno-crossjumping).
1262 */
1263
1264 /* Use macros rather than inline functions, to make it as clear as possible
1265 * to the C compiler that the tracing check is a simple test then branch.
1266 * We want to be sure that the compiler knows this before it generates
1267 * the CFG.
1268 */
1269 #ifdef LLTRACE
1270 #define OR_LLTRACE || lltrace
1271 #else
1272 #define OR_LLTRACE
1273 #endif
1274
1275 #ifdef WITH_DTRACE
1276 #define OR_DTRACE_LINE || PyDTrace_LINE_ENABLED()
1277 #else
1278 #define OR_DTRACE_LINE
1279 #endif
1280
1281 #ifdef DYNAMIC_EXECUTION_PROFILE
1282 #undef USE_COMPUTED_GOTOS
1283 #define USE_COMPUTED_GOTOS 0
1284 #endif
1285
1286 #ifdef HAVE_COMPUTED_GOTOS
1287 #ifndef USE_COMPUTED_GOTOS
1288 #define USE_COMPUTED_GOTOS 1
1289 #endif
1290 #else
1291 #if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS
1292 #error "Computed gotos are not supported on this compiler."
1293 #endif
1294 #undef USE_COMPUTED_GOTOS
1295 #define USE_COMPUTED_GOTOS 0
1296 #endif
1297
1298 #if USE_COMPUTED_GOTOS
1299 #define TARGET(op) op: TARGET_##op
1300 #define DISPATCH() \
1301 { \
1302 if (trace_info.cframe.use_tracing OR_DTRACE_LINE OR_LLTRACE) { \
1303 goto tracing_dispatch; \
1304 } \
1305 f->f_lasti = INSTR_OFFSET(); \
1306 NEXTOPARG(); \
1307 goto *opcode_targets[opcode]; \
1308 }
1309 #else
1310 #define TARGET(op) op
1311 #define DISPATCH() goto predispatch;
1312 #endif
1313
1314
1315 #define CHECK_EVAL_BREAKER() \
1316 if (_Py_atomic_load_relaxed(eval_breaker)) { \
1317 continue; \
1318 }
1319
1320
1321 /* Tuple access macros */
1322
1323 #ifndef Py_DEBUG
1324 #define GETITEM(v, i) PyTuple_GET_ITEM((PyTupleObject *)(v), (i))
1325 #else
1326 #define GETITEM(v, i) PyTuple_GetItem((v), (i))
1327 #endif
1328
1329 /* Code access macros */
1330
1331 /* The integer overflow is checked by an assertion below. */
1332 #define INSTR_OFFSET() ((int)(next_instr - first_instr))
1333 #define NEXTOPARG() do { \
1334 _Py_CODEUNIT word = *next_instr; \
1335 opcode = _Py_OPCODE(word); \
1336 oparg = _Py_OPARG(word); \
1337 next_instr++; \
1338 } while (0)
1339 #define JUMPTO(x) (next_instr = first_instr + (x))
1340 #define JUMPBY(x) (next_instr += (x))
1341
1342 /* OpCode prediction macros
1343 Some opcodes tend to come in pairs thus making it possible to
1344 predict the second code when the first is run. For example,
1345 COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE.
1346
1347 Verifying the prediction costs a single high-speed test of a register
1348 variable against a constant. If the pairing was good, then the
1349 processor's own internal branch predication has a high likelihood of
1350 success, resulting in a nearly zero-overhead transition to the
1351 next opcode. A successful prediction saves a trip through the eval-loop
1352 including its unpredictable switch-case branch. Combined with the
1353 processor's internal branch prediction, a successful PREDICT has the
1354 effect of making the two opcodes run as if they were a single new opcode
1355 with the bodies combined.
1356
1357 If collecting opcode statistics, your choices are to either keep the
1358 predictions turned-on and interpret the results as if some opcodes
1359 had been combined or turn-off predictions so that the opcode frequency
1360 counter updates for both opcodes.
1361
1362 Opcode prediction is disabled with threaded code, since the latter allows
1363 the CPU to record separate branch prediction information for each
1364 opcode.
1365
1366 */
1367
1368 #define PREDICT_ID(op) PRED_##op
1369
1370 #if defined(DYNAMIC_EXECUTION_PROFILE) || USE_COMPUTED_GOTOS
1371 #define PREDICT(op) if (0) goto PREDICT_ID(op)
1372 #else
1373 #define PREDICT(op) \
1374 do { \
1375 _Py_CODEUNIT word = *next_instr; \
1376 opcode = _Py_OPCODE(word); \
1377 if (opcode == op) { \
1378 oparg = _Py_OPARG(word); \
1379 next_instr++; \
1380 goto PREDICT_ID(op); \
1381 } \
1382 } while(0)
1383 #endif
1384 #define PREDICTED(op) PREDICT_ID(op):
1385
1386
1387 /* Stack manipulation macros */
1388
1389 /* The stack can grow at most MAXINT deep, as co_nlocals and
1390 co_stacksize are ints. */
1391 #define STACK_LEVEL() ((int)(stack_pointer - f->f_valuestack))
1392 #define EMPTY() (STACK_LEVEL() == 0)
1393 #define TOP() (stack_pointer[-1])
1394 #define SECOND() (stack_pointer[-2])
1395 #define THIRD() (stack_pointer[-3])
1396 #define FOURTH() (stack_pointer[-4])
1397 #define PEEK(n) (stack_pointer[-(n)])
1398 #define SET_TOP(v) (stack_pointer[-1] = (v))
1399 #define SET_SECOND(v) (stack_pointer[-2] = (v))
1400 #define SET_THIRD(v) (stack_pointer[-3] = (v))
1401 #define SET_FOURTH(v) (stack_pointer[-4] = (v))
1402 #define BASIC_STACKADJ(n) (stack_pointer += n)
1403 #define BASIC_PUSH(v) (*stack_pointer++ = (v))
1404 #define BASIC_POP() (*--stack_pointer)
1405
1406 #ifdef LLTRACE
1407 #define PUSH(v) { (void)(BASIC_PUSH(v), \
1408 lltrace && prtrace(tstate, TOP(), "push")); \
1409 assert(STACK_LEVEL() <= co->co_stacksize); }
1410 #define POP() ((void)(lltrace && prtrace(tstate, TOP(), "pop")), \
1411 BASIC_POP())
1412 #define STACK_GROW(n) do { \
1413 assert(n >= 0); \
1414 (void)(BASIC_STACKADJ(n), \
1415 lltrace && prtrace(tstate, TOP(), "stackadj")); \
1416 assert(STACK_LEVEL() <= co->co_stacksize); \
1417 } while (0)
1418 #define STACK_SHRINK(n) do { \
1419 assert(n >= 0); \
1420 (void)(lltrace && prtrace(tstate, TOP(), "stackadj")); \
1421 (void)(BASIC_STACKADJ(-n)); \
1422 assert(STACK_LEVEL() <= co->co_stacksize); \
1423 } while (0)
1424 #define EXT_POP(STACK_POINTER) ((void)(lltrace && \
1425 prtrace(tstate, (STACK_POINTER)[-1], "ext_pop")), \
1426 *--(STACK_POINTER))
1427 #else
1428 #define PUSH(v) BASIC_PUSH(v)
1429 #define POP() BASIC_POP()
1430 #define STACK_GROW(n) BASIC_STACKADJ(n)
1431 #define STACK_SHRINK(n) BASIC_STACKADJ(-n)
1432 #define EXT_POP(STACK_POINTER) (*--(STACK_POINTER))
1433 #endif
1434
1435 /* Local variable macros */
1436
1437 #define GETLOCAL(i) (fastlocals[i])
1438
1439 /* The SETLOCAL() macro must not DECREF the local variable in-place and
1440 then store the new value; it must copy the old value to a temporary
1441 value, then store the new value, and then DECREF the temporary value.
1442 This is because it is possible that during the DECREF the frame is
1443 accessed by other code (e.g. a __del__ method or gc.collect()) and the
1444 variable would be pointing to already-freed memory. */
1445 #define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \
1446 GETLOCAL(i) = value; \
1447 Py_XDECREF(tmp); } while (0)
1448
1449
1450 #define UNWIND_BLOCK(b) \
1451 while (STACK_LEVEL() > (b)->b_level) { \
1452 PyObject *v = POP(); \
1453 Py_XDECREF(v); \
1454 }
1455
1456 #define UNWIND_EXCEPT_HANDLER(b) \
1457 do { \
1458 PyObject *type, *value, *traceback; \
1459 _PyErr_StackItem *exc_info; \
1460 assert(STACK_LEVEL() >= (b)->b_level + 3); \
1461 while (STACK_LEVEL() > (b)->b_level + 3) { \
1462 value = POP(); \
1463 Py_XDECREF(value); \
1464 } \
1465 exc_info = tstate->exc_info; \
1466 type = exc_info->exc_type; \
1467 value = exc_info->exc_value; \
1468 traceback = exc_info->exc_traceback; \
1469 exc_info->exc_type = POP(); \
1470 exc_info->exc_value = POP(); \
1471 exc_info->exc_traceback = POP(); \
1472 Py_XDECREF(type); \
1473 Py_XDECREF(value); \
1474 Py_XDECREF(traceback); \
1475 } while(0)
1476
1477 /* macros for opcode cache */
1478 #define OPCACHE_CHECK() \
1479 do { \
1480 co_opcache = NULL; \
1481 if (co->co_opcache != NULL) { \
1482 unsigned char co_opcache_offset = \
1483 co->co_opcache_map[next_instr - first_instr]; \
1484 if (co_opcache_offset > 0) { \
1485 assert(co_opcache_offset <= co->co_opcache_size); \
1486 co_opcache = &co->co_opcache[co_opcache_offset - 1]; \
1487 assert(co_opcache != NULL); \
1488 } \
1489 } \
1490 } while (0)
1491
1492 #define OPCACHE_DEOPT() \
1493 do { \
1494 if (co_opcache != NULL) { \
1495 co_opcache->optimized = -1; \
1496 unsigned char co_opcache_offset = \
1497 co->co_opcache_map[next_instr - first_instr]; \
1498 assert(co_opcache_offset <= co->co_opcache_size); \
1499 co->co_opcache_map[co_opcache_offset] = 0; \
1500 co_opcache = NULL; \
1501 } \
1502 } while (0)
1503
1504 #define OPCACHE_DEOPT_LOAD_ATTR() \
1505 do { \
1506 if (co_opcache != NULL) { \
1507 OPCACHE_STAT_ATTR_DEOPT(); \
1508 OPCACHE_DEOPT(); \
1509 } \
1510 } while (0)
1511
1512 #define OPCACHE_MAYBE_DEOPT_LOAD_ATTR() \
1513 do { \
1514 if (co_opcache != NULL && --co_opcache->optimized <= 0) { \
1515 OPCACHE_DEOPT_LOAD_ATTR(); \
1516 } \
1517 } while (0)
1518
1519 #if OPCACHE_STATS
1520
1521 #define OPCACHE_STAT_GLOBAL_HIT() \
1522 do { \
1523 if (co->co_opcache != NULL) opcache_global_hits++; \
1524 } while (0)
1525
1526 #define OPCACHE_STAT_GLOBAL_MISS() \
1527 do { \
1528 if (co->co_opcache != NULL) opcache_global_misses++; \
1529 } while (0)
1530
1531 #define OPCACHE_STAT_GLOBAL_OPT() \
1532 do { \
1533 if (co->co_opcache != NULL) opcache_global_opts++; \
1534 } while (0)
1535
1536 #define OPCACHE_STAT_ATTR_HIT() \
1537 do { \
1538 if (co->co_opcache != NULL) opcache_attr_hits++; \
1539 } while (0)
1540
1541 #define OPCACHE_STAT_ATTR_MISS() \
1542 do { \
1543 if (co->co_opcache != NULL) opcache_attr_misses++; \
1544 } while (0)
1545
1546 #define OPCACHE_STAT_ATTR_OPT() \
1547 do { \
1548 if (co->co_opcache!= NULL) opcache_attr_opts++; \
1549 } while (0)
1550
1551 #define OPCACHE_STAT_ATTR_DEOPT() \
1552 do { \
1553 if (co->co_opcache != NULL) opcache_attr_deopts++; \
1554 } while (0)
1555
1556 #define OPCACHE_STAT_ATTR_TOTAL() \
1557 do { \
1558 if (co->co_opcache != NULL) opcache_attr_total++; \
1559 } while (0)
1560
1561 #else /* OPCACHE_STATS */
1562
1563 #define OPCACHE_STAT_GLOBAL_HIT()
1564 #define OPCACHE_STAT_GLOBAL_MISS()
1565 #define OPCACHE_STAT_GLOBAL_OPT()
1566
1567 #define OPCACHE_STAT_ATTR_HIT()
1568 #define OPCACHE_STAT_ATTR_MISS()
1569 #define OPCACHE_STAT_ATTR_OPT()
1570 #define OPCACHE_STAT_ATTR_DEOPT()
1571 #define OPCACHE_STAT_ATTR_TOTAL()
1572
1573 #endif
1574
1575
1576 PyObject* _Py_HOT_FUNCTION
_PyEval_EvalFrameDefault(PyThreadState * tstate,PyFrameObject * f,int throwflag)1577 _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
1578 {
1579 _Py_EnsureTstateNotNULL(tstate);
1580
1581 #if USE_COMPUTED_GOTOS
1582 /* Import the static jump table */
1583 #include "opcode_targets.h"
1584 #endif
1585
1586 #ifdef DXPAIRS
1587 int lastopcode = 0;
1588 #endif
1589 PyObject **stack_pointer; /* Next free slot in value stack */
1590 const _Py_CODEUNIT *next_instr;
1591 int opcode; /* Current opcode */
1592 int oparg; /* Current opcode argument, if any */
1593 PyObject **fastlocals, **freevars;
1594 PyObject *retval = NULL; /* Return value */
1595 _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
1596 PyCodeObject *co;
1597
1598 const _Py_CODEUNIT *first_instr;
1599 PyObject *names;
1600 PyObject *consts;
1601 _PyOpcache *co_opcache;
1602
1603 #ifdef LLTRACE
1604 _Py_IDENTIFIER(__ltrace__);
1605 #endif
1606
1607 if (_Py_EnterRecursiveCall(tstate, "")) {
1608 return NULL;
1609 }
1610
1611 PyTraceInfo trace_info;
1612 /* Mark trace_info as uninitialized */
1613 trace_info.code = NULL;
1614
1615 /* WARNING: Because the CFrame lives on the C stack,
1616 * but can be accessed from a heap allocated object (tstate)
1617 * strict stack discipline must be maintained.
1618 */
1619 CFrame *prev_cframe = tstate->cframe;
1620 trace_info.cframe.use_tracing = prev_cframe->use_tracing;
1621 trace_info.cframe.previous = prev_cframe;
1622 tstate->cframe = &trace_info.cframe;
1623
1624 /* push frame */
1625 tstate->frame = f;
1626 co = f->f_code;
1627
1628 if (trace_info.cframe.use_tracing) {
1629 if (tstate->c_tracefunc != NULL) {
1630 /* tstate->c_tracefunc, if defined, is a
1631 function that will be called on *every* entry
1632 to a code block. Its return value, if not
1633 None, is a function that will be called at
1634 the start of each executed line of code.
1635 (Actually, the function must return itself
1636 in order to continue tracing.) The trace
1637 functions are called with three arguments:
1638 a pointer to the current frame, a string
1639 indicating why the function is called, and
1640 an argument which depends on the situation.
1641 The global trace function is also called
1642 whenever an exception is detected. */
1643 if (call_trace_protected(tstate->c_tracefunc,
1644 tstate->c_traceobj,
1645 tstate, f, &trace_info,
1646 PyTrace_CALL, Py_None)) {
1647 /* Trace function raised an error */
1648 goto exit_eval_frame;
1649 }
1650 }
1651 if (tstate->c_profilefunc != NULL) {
1652 /* Similar for c_profilefunc, except it needn't
1653 return itself and isn't called for "line" events */
1654 if (call_trace_protected(tstate->c_profilefunc,
1655 tstate->c_profileobj,
1656 tstate, f, &trace_info,
1657 PyTrace_CALL, Py_None)) {
1658 /* Profile function raised an error */
1659 goto exit_eval_frame;
1660 }
1661 }
1662 }
1663
1664 if (PyDTrace_FUNCTION_ENTRY_ENABLED())
1665 dtrace_function_entry(f);
1666
1667 names = co->co_names;
1668 consts = co->co_consts;
1669 fastlocals = f->f_localsplus;
1670 freevars = f->f_localsplus + co->co_nlocals;
1671 assert(PyBytes_Check(co->co_code));
1672 assert(PyBytes_GET_SIZE(co->co_code) <= INT_MAX);
1673 assert(PyBytes_GET_SIZE(co->co_code) % sizeof(_Py_CODEUNIT) == 0);
1674 assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(co->co_code), sizeof(_Py_CODEUNIT)));
1675 first_instr = (_Py_CODEUNIT *) PyBytes_AS_STRING(co->co_code);
1676 /*
1677 f->f_lasti refers to the index of the last instruction,
1678 unless it's -1 in which case next_instr should be first_instr.
1679
1680 YIELD_FROM sets f_lasti to itself, in order to repeatedly yield
1681 multiple values.
1682
1683 When the PREDICT() macros are enabled, some opcode pairs follow in
1684 direct succession without updating f->f_lasti. A successful
1685 prediction effectively links the two codes together as if they
1686 were a single new opcode; accordingly,f->f_lasti will point to
1687 the first code in the pair (for instance, GET_ITER followed by
1688 FOR_ITER is effectively a single opcode and f->f_lasti will point
1689 to the beginning of the combined pair.)
1690 */
1691 assert(f->f_lasti >= -1);
1692 next_instr = first_instr + f->f_lasti + 1;
1693 stack_pointer = f->f_valuestack + f->f_stackdepth;
1694 /* Set f->f_stackdepth to -1.
1695 * Update when returning or calling trace function.
1696 Having f_stackdepth <= 0 ensures that invalid
1697 values are not visible to the cycle GC.
1698 We choose -1 rather than 0 to assist debugging.
1699 */
1700 f->f_stackdepth = -1;
1701 f->f_state = FRAME_EXECUTING;
1702
1703 if (co->co_opcache_flag < opcache_min_runs) {
1704 co->co_opcache_flag++;
1705 if (co->co_opcache_flag == opcache_min_runs) {
1706 if (_PyCode_InitOpcache(co) < 0) {
1707 goto exit_eval_frame;
1708 }
1709 #if OPCACHE_STATS
1710 opcache_code_objects_extra_mem +=
1711 PyBytes_Size(co->co_code) / sizeof(_Py_CODEUNIT) +
1712 sizeof(_PyOpcache) * co->co_opcache_size;
1713 opcache_code_objects++;
1714 #endif
1715 }
1716 }
1717
1718 #ifdef LLTRACE
1719 {
1720 int r = _PyDict_ContainsId(f->f_globals, &PyId___ltrace__);
1721 if (r < 0) {
1722 goto exit_eval_frame;
1723 }
1724 lltrace = r;
1725 }
1726 #endif
1727
1728 if (throwflag) { /* support for generator.throw() */
1729 goto error;
1730 }
1731
1732 #ifdef Py_DEBUG
1733 /* _PyEval_EvalFrameDefault() must not be called with an exception set,
1734 because it can clear it (directly or indirectly) and so the
1735 caller loses its exception */
1736 assert(!_PyErr_Occurred(tstate));
1737 #endif
1738
1739 main_loop:
1740 for (;;) {
1741 assert(stack_pointer >= f->f_valuestack); /* else underflow */
1742 assert(STACK_LEVEL() <= co->co_stacksize); /* else overflow */
1743 assert(!_PyErr_Occurred(tstate));
1744
1745 /* Do periodic things. Doing this every time through
1746 the loop would add too much overhead, so we do it
1747 only every Nth instruction. We also do it if
1748 ``pending.calls_to_do'' is set, i.e. when an asynchronous
1749 event needs attention (e.g. a signal handler or
1750 async I/O handler); see Py_AddPendingCall() and
1751 Py_MakePendingCalls() above. */
1752
1753 if (_Py_atomic_load_relaxed(eval_breaker)) {
1754 opcode = _Py_OPCODE(*next_instr);
1755 if (opcode != SETUP_FINALLY &&
1756 opcode != SETUP_WITH &&
1757 opcode != BEFORE_ASYNC_WITH &&
1758 opcode != YIELD_FROM) {
1759 /* Few cases where we skip running signal handlers and other
1760 pending calls:
1761 - If we're about to enter the 'with:'. It will prevent
1762 emitting a resource warning in the common idiom
1763 'with open(path) as file:'.
1764 - If we're about to enter the 'async with:'.
1765 - If we're about to enter the 'try:' of a try/finally (not
1766 *very* useful, but might help in some cases and it's
1767 traditional)
1768 - If we're resuming a chain of nested 'yield from' or
1769 'await' calls, then each frame is parked with YIELD_FROM
1770 as its next opcode. If the user hit control-C we want to
1771 wait until we've reached the innermost frame before
1772 running the signal handler and raising KeyboardInterrupt
1773 (see bpo-30039).
1774 */
1775 if (eval_frame_handle_pending(tstate) != 0) {
1776 goto error;
1777 }
1778 }
1779 }
1780
1781 tracing_dispatch:
1782 {
1783 int instr_prev = f->f_lasti;
1784 f->f_lasti = INSTR_OFFSET();
1785 NEXTOPARG();
1786
1787 if (PyDTrace_LINE_ENABLED())
1788 maybe_dtrace_line(f, &trace_info, instr_prev);
1789
1790 /* line-by-line tracing support */
1791
1792 if (trace_info.cframe.use_tracing &&
1793 tstate->c_tracefunc != NULL && !tstate->tracing) {
1794 int err;
1795 /* see maybe_call_line_trace()
1796 for expository comments */
1797 f->f_stackdepth = (int)(stack_pointer - f->f_valuestack);
1798
1799 err = maybe_call_line_trace(tstate->c_tracefunc,
1800 tstate->c_traceobj,
1801 tstate, f,
1802 &trace_info, instr_prev);
1803 /* Reload possibly changed frame fields */
1804 JUMPTO(f->f_lasti);
1805 stack_pointer = f->f_valuestack+f->f_stackdepth;
1806 f->f_stackdepth = -1;
1807 if (err) {
1808 /* trace function raised an exception */
1809 goto error;
1810 }
1811 NEXTOPARG();
1812 }
1813 }
1814
1815 #ifdef LLTRACE
1816 /* Instruction tracing */
1817
1818 if (lltrace) {
1819 if (HAS_ARG(opcode)) {
1820 printf("%d: %d, %d\n",
1821 f->f_lasti, opcode, oparg);
1822 }
1823 else {
1824 printf("%d: %d\n",
1825 f->f_lasti, opcode);
1826 }
1827 }
1828 #endif
1829 #if USE_COMPUTED_GOTOS == 0
1830 goto dispatch_opcode;
1831
1832 predispatch:
1833 if (trace_info.cframe.use_tracing OR_DTRACE_LINE OR_LLTRACE) {
1834 goto tracing_dispatch;
1835 }
1836 f->f_lasti = INSTR_OFFSET();
1837 NEXTOPARG();
1838 #endif
1839 dispatch_opcode:
1840 #ifdef DYNAMIC_EXECUTION_PROFILE
1841 #ifdef DXPAIRS
1842 dxpairs[lastopcode][opcode]++;
1843 lastopcode = opcode;
1844 #endif
1845 dxp[opcode]++;
1846 #endif
1847
1848 switch (opcode) {
1849
1850 /* BEWARE!
1851 It is essential that any operation that fails must goto error
1852 and that all operation that succeed call DISPATCH() ! */
1853
1854 case TARGET(NOP): {
1855 DISPATCH();
1856 }
1857
1858 case TARGET(LOAD_FAST): {
1859 PyObject *value = GETLOCAL(oparg);
1860 if (value == NULL) {
1861 format_exc_check_arg(tstate, PyExc_UnboundLocalError,
1862 UNBOUNDLOCAL_ERROR_MSG,
1863 PyTuple_GetItem(co->co_varnames, oparg));
1864 goto error;
1865 }
1866 Py_INCREF(value);
1867 PUSH(value);
1868 DISPATCH();
1869 }
1870
1871 case TARGET(LOAD_CONST): {
1872 PREDICTED(LOAD_CONST);
1873 PyObject *value = GETITEM(consts, oparg);
1874 Py_INCREF(value);
1875 PUSH(value);
1876 DISPATCH();
1877 }
1878
1879 case TARGET(STORE_FAST): {
1880 PREDICTED(STORE_FAST);
1881 PyObject *value = POP();
1882 SETLOCAL(oparg, value);
1883 DISPATCH();
1884 }
1885
1886 case TARGET(POP_TOP): {
1887 PyObject *value = POP();
1888 Py_DECREF(value);
1889 DISPATCH();
1890 }
1891
1892 case TARGET(ROT_TWO): {
1893 PyObject *top = TOP();
1894 PyObject *second = SECOND();
1895 SET_TOP(second);
1896 SET_SECOND(top);
1897 DISPATCH();
1898 }
1899
1900 case TARGET(ROT_THREE): {
1901 PyObject *top = TOP();
1902 PyObject *second = SECOND();
1903 PyObject *third = THIRD();
1904 SET_TOP(second);
1905 SET_SECOND(third);
1906 SET_THIRD(top);
1907 DISPATCH();
1908 }
1909
1910 case TARGET(ROT_FOUR): {
1911 PyObject *top = TOP();
1912 PyObject *second = SECOND();
1913 PyObject *third = THIRD();
1914 PyObject *fourth = FOURTH();
1915 SET_TOP(second);
1916 SET_SECOND(third);
1917 SET_THIRD(fourth);
1918 SET_FOURTH(top);
1919 DISPATCH();
1920 }
1921
1922 case TARGET(DUP_TOP): {
1923 PyObject *top = TOP();
1924 Py_INCREF(top);
1925 PUSH(top);
1926 DISPATCH();
1927 }
1928
1929 case TARGET(DUP_TOP_TWO): {
1930 PyObject *top = TOP();
1931 PyObject *second = SECOND();
1932 Py_INCREF(top);
1933 Py_INCREF(second);
1934 STACK_GROW(2);
1935 SET_TOP(top);
1936 SET_SECOND(second);
1937 DISPATCH();
1938 }
1939
1940 case TARGET(UNARY_POSITIVE): {
1941 PyObject *value = TOP();
1942 PyObject *res = PyNumber_Positive(value);
1943 Py_DECREF(value);
1944 SET_TOP(res);
1945 if (res == NULL)
1946 goto error;
1947 DISPATCH();
1948 }
1949
1950 case TARGET(UNARY_NEGATIVE): {
1951 PyObject *value = TOP();
1952 PyObject *res = PyNumber_Negative(value);
1953 Py_DECREF(value);
1954 SET_TOP(res);
1955 if (res == NULL)
1956 goto error;
1957 DISPATCH();
1958 }
1959
1960 case TARGET(UNARY_NOT): {
1961 PyObject *value = TOP();
1962 int err = PyObject_IsTrue(value);
1963 Py_DECREF(value);
1964 if (err == 0) {
1965 Py_INCREF(Py_True);
1966 SET_TOP(Py_True);
1967 DISPATCH();
1968 }
1969 else if (err > 0) {
1970 Py_INCREF(Py_False);
1971 SET_TOP(Py_False);
1972 DISPATCH();
1973 }
1974 STACK_SHRINK(1);
1975 goto error;
1976 }
1977
1978 case TARGET(UNARY_INVERT): {
1979 PyObject *value = TOP();
1980 PyObject *res = PyNumber_Invert(value);
1981 Py_DECREF(value);
1982 SET_TOP(res);
1983 if (res == NULL)
1984 goto error;
1985 DISPATCH();
1986 }
1987
1988 case TARGET(BINARY_POWER): {
1989 PyObject *exp = POP();
1990 PyObject *base = TOP();
1991 PyObject *res = PyNumber_Power(base, exp, Py_None);
1992 Py_DECREF(base);
1993 Py_DECREF(exp);
1994 SET_TOP(res);
1995 if (res == NULL)
1996 goto error;
1997 DISPATCH();
1998 }
1999
2000 case TARGET(BINARY_MULTIPLY): {
2001 PyObject *right = POP();
2002 PyObject *left = TOP();
2003 PyObject *res = PyNumber_Multiply(left, right);
2004 Py_DECREF(left);
2005 Py_DECREF(right);
2006 SET_TOP(res);
2007 if (res == NULL)
2008 goto error;
2009 DISPATCH();
2010 }
2011
2012 case TARGET(BINARY_MATRIX_MULTIPLY): {
2013 PyObject *right = POP();
2014 PyObject *left = TOP();
2015 PyObject *res = PyNumber_MatrixMultiply(left, right);
2016 Py_DECREF(left);
2017 Py_DECREF(right);
2018 SET_TOP(res);
2019 if (res == NULL)
2020 goto error;
2021 DISPATCH();
2022 }
2023
2024 case TARGET(BINARY_TRUE_DIVIDE): {
2025 PyObject *divisor = POP();
2026 PyObject *dividend = TOP();
2027 PyObject *quotient = PyNumber_TrueDivide(dividend, divisor);
2028 Py_DECREF(dividend);
2029 Py_DECREF(divisor);
2030 SET_TOP(quotient);
2031 if (quotient == NULL)
2032 goto error;
2033 DISPATCH();
2034 }
2035
2036 case TARGET(BINARY_FLOOR_DIVIDE): {
2037 PyObject *divisor = POP();
2038 PyObject *dividend = TOP();
2039 PyObject *quotient = PyNumber_FloorDivide(dividend, divisor);
2040 Py_DECREF(dividend);
2041 Py_DECREF(divisor);
2042 SET_TOP(quotient);
2043 if (quotient == NULL)
2044 goto error;
2045 DISPATCH();
2046 }
2047
2048 case TARGET(BINARY_MODULO): {
2049 PyObject *divisor = POP();
2050 PyObject *dividend = TOP();
2051 PyObject *res;
2052 if (PyUnicode_CheckExact(dividend) && (
2053 !PyUnicode_Check(divisor) || PyUnicode_CheckExact(divisor))) {
2054 // fast path; string formatting, but not if the RHS is a str subclass
2055 // (see issue28598)
2056 res = PyUnicode_Format(dividend, divisor);
2057 } else {
2058 res = PyNumber_Remainder(dividend, divisor);
2059 }
2060 Py_DECREF(divisor);
2061 Py_DECREF(dividend);
2062 SET_TOP(res);
2063 if (res == NULL)
2064 goto error;
2065 DISPATCH();
2066 }
2067
2068 case TARGET(BINARY_ADD): {
2069 PyObject *right = POP();
2070 PyObject *left = TOP();
2071 PyObject *sum;
2072 /* NOTE(vstinner): Please don't try to micro-optimize int+int on
2073 CPython using bytecode, it is simply worthless.
2074 See http://bugs.python.org/issue21955 and
2075 http://bugs.python.org/issue10044 for the discussion. In short,
2076 no patch shown any impact on a realistic benchmark, only a minor
2077 speedup on microbenchmarks. */
2078 if (PyUnicode_CheckExact(left) &&
2079 PyUnicode_CheckExact(right)) {
2080 sum = unicode_concatenate(tstate, left, right, f, next_instr);
2081 /* unicode_concatenate consumed the ref to left */
2082 }
2083 else {
2084 sum = PyNumber_Add(left, right);
2085 Py_DECREF(left);
2086 }
2087 Py_DECREF(right);
2088 SET_TOP(sum);
2089 if (sum == NULL)
2090 goto error;
2091 DISPATCH();
2092 }
2093
2094 case TARGET(BINARY_SUBTRACT): {
2095 PyObject *right = POP();
2096 PyObject *left = TOP();
2097 PyObject *diff = PyNumber_Subtract(left, right);
2098 Py_DECREF(right);
2099 Py_DECREF(left);
2100 SET_TOP(diff);
2101 if (diff == NULL)
2102 goto error;
2103 DISPATCH();
2104 }
2105
2106 case TARGET(BINARY_SUBSCR): {
2107 PyObject *sub = POP();
2108 PyObject *container = TOP();
2109 PyObject *res = PyObject_GetItem(container, sub);
2110 Py_DECREF(container);
2111 Py_DECREF(sub);
2112 SET_TOP(res);
2113 if (res == NULL)
2114 goto error;
2115 DISPATCH();
2116 }
2117
2118 case TARGET(BINARY_LSHIFT): {
2119 PyObject *right = POP();
2120 PyObject *left = TOP();
2121 PyObject *res = PyNumber_Lshift(left, right);
2122 Py_DECREF(left);
2123 Py_DECREF(right);
2124 SET_TOP(res);
2125 if (res == NULL)
2126 goto error;
2127 DISPATCH();
2128 }
2129
2130 case TARGET(BINARY_RSHIFT): {
2131 PyObject *right = POP();
2132 PyObject *left = TOP();
2133 PyObject *res = PyNumber_Rshift(left, right);
2134 Py_DECREF(left);
2135 Py_DECREF(right);
2136 SET_TOP(res);
2137 if (res == NULL)
2138 goto error;
2139 DISPATCH();
2140 }
2141
2142 case TARGET(BINARY_AND): {
2143 PyObject *right = POP();
2144 PyObject *left = TOP();
2145 PyObject *res = PyNumber_And(left, right);
2146 Py_DECREF(left);
2147 Py_DECREF(right);
2148 SET_TOP(res);
2149 if (res == NULL)
2150 goto error;
2151 DISPATCH();
2152 }
2153
2154 case TARGET(BINARY_XOR): {
2155 PyObject *right = POP();
2156 PyObject *left = TOP();
2157 PyObject *res = PyNumber_Xor(left, right);
2158 Py_DECREF(left);
2159 Py_DECREF(right);
2160 SET_TOP(res);
2161 if (res == NULL)
2162 goto error;
2163 DISPATCH();
2164 }
2165
2166 case TARGET(BINARY_OR): {
2167 PyObject *right = POP();
2168 PyObject *left = TOP();
2169 PyObject *res = PyNumber_Or(left, right);
2170 Py_DECREF(left);
2171 Py_DECREF(right);
2172 SET_TOP(res);
2173 if (res == NULL)
2174 goto error;
2175 DISPATCH();
2176 }
2177
2178 case TARGET(LIST_APPEND): {
2179 PyObject *v = POP();
2180 PyObject *list = PEEK(oparg);
2181 int err;
2182 err = PyList_Append(list, v);
2183 Py_DECREF(v);
2184 if (err != 0)
2185 goto error;
2186 PREDICT(JUMP_ABSOLUTE);
2187 DISPATCH();
2188 }
2189
2190 case TARGET(SET_ADD): {
2191 PyObject *v = POP();
2192 PyObject *set = PEEK(oparg);
2193 int err;
2194 err = PySet_Add(set, v);
2195 Py_DECREF(v);
2196 if (err != 0)
2197 goto error;
2198 PREDICT(JUMP_ABSOLUTE);
2199 DISPATCH();
2200 }
2201
2202 case TARGET(INPLACE_POWER): {
2203 PyObject *exp = POP();
2204 PyObject *base = TOP();
2205 PyObject *res = PyNumber_InPlacePower(base, exp, Py_None);
2206 Py_DECREF(base);
2207 Py_DECREF(exp);
2208 SET_TOP(res);
2209 if (res == NULL)
2210 goto error;
2211 DISPATCH();
2212 }
2213
2214 case TARGET(INPLACE_MULTIPLY): {
2215 PyObject *right = POP();
2216 PyObject *left = TOP();
2217 PyObject *res = PyNumber_InPlaceMultiply(left, right);
2218 Py_DECREF(left);
2219 Py_DECREF(right);
2220 SET_TOP(res);
2221 if (res == NULL)
2222 goto error;
2223 DISPATCH();
2224 }
2225
2226 case TARGET(INPLACE_MATRIX_MULTIPLY): {
2227 PyObject *right = POP();
2228 PyObject *left = TOP();
2229 PyObject *res = PyNumber_InPlaceMatrixMultiply(left, right);
2230 Py_DECREF(left);
2231 Py_DECREF(right);
2232 SET_TOP(res);
2233 if (res == NULL)
2234 goto error;
2235 DISPATCH();
2236 }
2237
2238 case TARGET(INPLACE_TRUE_DIVIDE): {
2239 PyObject *divisor = POP();
2240 PyObject *dividend = TOP();
2241 PyObject *quotient = PyNumber_InPlaceTrueDivide(dividend, divisor);
2242 Py_DECREF(dividend);
2243 Py_DECREF(divisor);
2244 SET_TOP(quotient);
2245 if (quotient == NULL)
2246 goto error;
2247 DISPATCH();
2248 }
2249
2250 case TARGET(INPLACE_FLOOR_DIVIDE): {
2251 PyObject *divisor = POP();
2252 PyObject *dividend = TOP();
2253 PyObject *quotient = PyNumber_InPlaceFloorDivide(dividend, divisor);
2254 Py_DECREF(dividend);
2255 Py_DECREF(divisor);
2256 SET_TOP(quotient);
2257 if (quotient == NULL)
2258 goto error;
2259 DISPATCH();
2260 }
2261
2262 case TARGET(INPLACE_MODULO): {
2263 PyObject *right = POP();
2264 PyObject *left = TOP();
2265 PyObject *mod = PyNumber_InPlaceRemainder(left, right);
2266 Py_DECREF(left);
2267 Py_DECREF(right);
2268 SET_TOP(mod);
2269 if (mod == NULL)
2270 goto error;
2271 DISPATCH();
2272 }
2273
2274 case TARGET(INPLACE_ADD): {
2275 PyObject *right = POP();
2276 PyObject *left = TOP();
2277 PyObject *sum;
2278 if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) {
2279 sum = unicode_concatenate(tstate, left, right, f, next_instr);
2280 /* unicode_concatenate consumed the ref to left */
2281 }
2282 else {
2283 sum = PyNumber_InPlaceAdd(left, right);
2284 Py_DECREF(left);
2285 }
2286 Py_DECREF(right);
2287 SET_TOP(sum);
2288 if (sum == NULL)
2289 goto error;
2290 DISPATCH();
2291 }
2292
2293 case TARGET(INPLACE_SUBTRACT): {
2294 PyObject *right = POP();
2295 PyObject *left = TOP();
2296 PyObject *diff = PyNumber_InPlaceSubtract(left, right);
2297 Py_DECREF(left);
2298 Py_DECREF(right);
2299 SET_TOP(diff);
2300 if (diff == NULL)
2301 goto error;
2302 DISPATCH();
2303 }
2304
2305 case TARGET(INPLACE_LSHIFT): {
2306 PyObject *right = POP();
2307 PyObject *left = TOP();
2308 PyObject *res = PyNumber_InPlaceLshift(left, right);
2309 Py_DECREF(left);
2310 Py_DECREF(right);
2311 SET_TOP(res);
2312 if (res == NULL)
2313 goto error;
2314 DISPATCH();
2315 }
2316
2317 case TARGET(INPLACE_RSHIFT): {
2318 PyObject *right = POP();
2319 PyObject *left = TOP();
2320 PyObject *res = PyNumber_InPlaceRshift(left, right);
2321 Py_DECREF(left);
2322 Py_DECREF(right);
2323 SET_TOP(res);
2324 if (res == NULL)
2325 goto error;
2326 DISPATCH();
2327 }
2328
2329 case TARGET(INPLACE_AND): {
2330 PyObject *right = POP();
2331 PyObject *left = TOP();
2332 PyObject *res = PyNumber_InPlaceAnd(left, right);
2333 Py_DECREF(left);
2334 Py_DECREF(right);
2335 SET_TOP(res);
2336 if (res == NULL)
2337 goto error;
2338 DISPATCH();
2339 }
2340
2341 case TARGET(INPLACE_XOR): {
2342 PyObject *right = POP();
2343 PyObject *left = TOP();
2344 PyObject *res = PyNumber_InPlaceXor(left, right);
2345 Py_DECREF(left);
2346 Py_DECREF(right);
2347 SET_TOP(res);
2348 if (res == NULL)
2349 goto error;
2350 DISPATCH();
2351 }
2352
2353 case TARGET(INPLACE_OR): {
2354 PyObject *right = POP();
2355 PyObject *left = TOP();
2356 PyObject *res = PyNumber_InPlaceOr(left, right);
2357 Py_DECREF(left);
2358 Py_DECREF(right);
2359 SET_TOP(res);
2360 if (res == NULL)
2361 goto error;
2362 DISPATCH();
2363 }
2364
2365 case TARGET(STORE_SUBSCR): {
2366 PyObject *sub = TOP();
2367 PyObject *container = SECOND();
2368 PyObject *v = THIRD();
2369 int err;
2370 STACK_SHRINK(3);
2371 /* container[sub] = v */
2372 err = PyObject_SetItem(container, sub, v);
2373 Py_DECREF(v);
2374 Py_DECREF(container);
2375 Py_DECREF(sub);
2376 if (err != 0)
2377 goto error;
2378 DISPATCH();
2379 }
2380
2381 case TARGET(DELETE_SUBSCR): {
2382 PyObject *sub = TOP();
2383 PyObject *container = SECOND();
2384 int err;
2385 STACK_SHRINK(2);
2386 /* del container[sub] */
2387 err = PyObject_DelItem(container, sub);
2388 Py_DECREF(container);
2389 Py_DECREF(sub);
2390 if (err != 0)
2391 goto error;
2392 DISPATCH();
2393 }
2394
2395 case TARGET(PRINT_EXPR): {
2396 _Py_IDENTIFIER(displayhook);
2397 PyObject *value = POP();
2398 PyObject *hook = _PySys_GetObjectId(&PyId_displayhook);
2399 PyObject *res;
2400 if (hook == NULL) {
2401 _PyErr_SetString(tstate, PyExc_RuntimeError,
2402 "lost sys.displayhook");
2403 Py_DECREF(value);
2404 goto error;
2405 }
2406 res = PyObject_CallOneArg(hook, value);
2407 Py_DECREF(value);
2408 if (res == NULL)
2409 goto error;
2410 Py_DECREF(res);
2411 DISPATCH();
2412 }
2413
2414 case TARGET(RAISE_VARARGS): {
2415 PyObject *cause = NULL, *exc = NULL;
2416 switch (oparg) {
2417 case 2:
2418 cause = POP(); /* cause */
2419 /* fall through */
2420 case 1:
2421 exc = POP(); /* exc */
2422 /* fall through */
2423 case 0:
2424 if (do_raise(tstate, exc, cause)) {
2425 goto exception_unwind;
2426 }
2427 break;
2428 default:
2429 _PyErr_SetString(tstate, PyExc_SystemError,
2430 "bad RAISE_VARARGS oparg");
2431 break;
2432 }
2433 goto error;
2434 }
2435
2436 case TARGET(RETURN_VALUE): {
2437 retval = POP();
2438 assert(f->f_iblock == 0);
2439 assert(EMPTY());
2440 f->f_state = FRAME_RETURNED;
2441 f->f_stackdepth = 0;
2442 goto exiting;
2443 }
2444
2445 case TARGET(GET_AITER): {
2446 unaryfunc getter = NULL;
2447 PyObject *iter = NULL;
2448 PyObject *obj = TOP();
2449 PyTypeObject *type = Py_TYPE(obj);
2450
2451 if (type->tp_as_async != NULL) {
2452 getter = type->tp_as_async->am_aiter;
2453 }
2454
2455 if (getter != NULL) {
2456 iter = (*getter)(obj);
2457 Py_DECREF(obj);
2458 if (iter == NULL) {
2459 SET_TOP(NULL);
2460 goto error;
2461 }
2462 }
2463 else {
2464 SET_TOP(NULL);
2465 _PyErr_Format(tstate, PyExc_TypeError,
2466 "'async for' requires an object with "
2467 "__aiter__ method, got %.100s",
2468 type->tp_name);
2469 Py_DECREF(obj);
2470 goto error;
2471 }
2472
2473 if (Py_TYPE(iter)->tp_as_async == NULL ||
2474 Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
2475
2476 SET_TOP(NULL);
2477 _PyErr_Format(tstate, PyExc_TypeError,
2478 "'async for' received an object from __aiter__ "
2479 "that does not implement __anext__: %.100s",
2480 Py_TYPE(iter)->tp_name);
2481 Py_DECREF(iter);
2482 goto error;
2483 }
2484
2485 SET_TOP(iter);
2486 DISPATCH();
2487 }
2488
2489 case TARGET(GET_ANEXT): {
2490 unaryfunc getter = NULL;
2491 PyObject *next_iter = NULL;
2492 PyObject *awaitable = NULL;
2493 PyObject *aiter = TOP();
2494 PyTypeObject *type = Py_TYPE(aiter);
2495
2496 if (PyAsyncGen_CheckExact(aiter)) {
2497 awaitable = type->tp_as_async->am_anext(aiter);
2498 if (awaitable == NULL) {
2499 goto error;
2500 }
2501 } else {
2502 if (type->tp_as_async != NULL){
2503 getter = type->tp_as_async->am_anext;
2504 }
2505
2506 if (getter != NULL) {
2507 next_iter = (*getter)(aiter);
2508 if (next_iter == NULL) {
2509 goto error;
2510 }
2511 }
2512 else {
2513 _PyErr_Format(tstate, PyExc_TypeError,
2514 "'async for' requires an iterator with "
2515 "__anext__ method, got %.100s",
2516 type->tp_name);
2517 goto error;
2518 }
2519
2520 awaitable = _PyCoro_GetAwaitableIter(next_iter);
2521 if (awaitable == NULL) {
2522 _PyErr_FormatFromCause(
2523 PyExc_TypeError,
2524 "'async for' received an invalid object "
2525 "from __anext__: %.100s",
2526 Py_TYPE(next_iter)->tp_name);
2527
2528 Py_DECREF(next_iter);
2529 goto error;
2530 } else {
2531 Py_DECREF(next_iter);
2532 }
2533 }
2534
2535 PUSH(awaitable);
2536 PREDICT(LOAD_CONST);
2537 DISPATCH();
2538 }
2539
2540 case TARGET(GET_AWAITABLE): {
2541 PREDICTED(GET_AWAITABLE);
2542 PyObject *iterable = TOP();
2543 PyObject *iter = _PyCoro_GetAwaitableIter(iterable);
2544
2545 if (iter == NULL) {
2546 int opcode_at_minus_3 = 0;
2547 if ((next_instr - first_instr) > 2) {
2548 opcode_at_minus_3 = _Py_OPCODE(next_instr[-3]);
2549 }
2550 format_awaitable_error(tstate, Py_TYPE(iterable),
2551 opcode_at_minus_3,
2552 _Py_OPCODE(next_instr[-2]));
2553 }
2554
2555 Py_DECREF(iterable);
2556
2557 if (iter != NULL && PyCoro_CheckExact(iter)) {
2558 PyObject *yf = _PyGen_yf((PyGenObject*)iter);
2559 if (yf != NULL) {
2560 /* `iter` is a coroutine object that is being
2561 awaited, `yf` is a pointer to the current awaitable
2562 being awaited on. */
2563 Py_DECREF(yf);
2564 Py_CLEAR(iter);
2565 _PyErr_SetString(tstate, PyExc_RuntimeError,
2566 "coroutine is being awaited already");
2567 /* The code below jumps to `error` if `iter` is NULL. */
2568 }
2569 }
2570
2571 SET_TOP(iter); /* Even if it's NULL */
2572
2573 if (iter == NULL) {
2574 goto error;
2575 }
2576
2577 PREDICT(LOAD_CONST);
2578 DISPATCH();
2579 }
2580
2581 case TARGET(YIELD_FROM): {
2582 PyObject *v = POP();
2583 PyObject *receiver = TOP();
2584 PySendResult gen_status;
2585 if (tstate->c_tracefunc == NULL) {
2586 gen_status = PyIter_Send(receiver, v, &retval);
2587 } else {
2588 _Py_IDENTIFIER(send);
2589 if (Py_IsNone(v) && PyIter_Check(receiver)) {
2590 retval = Py_TYPE(receiver)->tp_iternext(receiver);
2591 }
2592 else {
2593 retval = _PyObject_CallMethodIdOneArg(receiver, &PyId_send, v);
2594 }
2595 if (retval == NULL) {
2596 if (tstate->c_tracefunc != NULL
2597 && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration))
2598 call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f, &trace_info);
2599 if (_PyGen_FetchStopIterationValue(&retval) == 0) {
2600 gen_status = PYGEN_RETURN;
2601 }
2602 else {
2603 gen_status = PYGEN_ERROR;
2604 }
2605 }
2606 else {
2607 gen_status = PYGEN_NEXT;
2608 }
2609 }
2610 Py_DECREF(v);
2611 if (gen_status == PYGEN_ERROR) {
2612 assert (retval == NULL);
2613 goto error;
2614 }
2615 if (gen_status == PYGEN_RETURN) {
2616 assert (retval != NULL);
2617
2618 Py_DECREF(receiver);
2619 SET_TOP(retval);
2620 retval = NULL;
2621 DISPATCH();
2622 }
2623 assert (gen_status == PYGEN_NEXT);
2624 /* receiver remains on stack, retval is value to be yielded */
2625 /* and repeat... */
2626 assert(f->f_lasti > 0);
2627 f->f_lasti -= 1;
2628 f->f_state = FRAME_SUSPENDED;
2629 f->f_stackdepth = (int)(stack_pointer - f->f_valuestack);
2630 goto exiting;
2631 }
2632
2633 case TARGET(YIELD_VALUE): {
2634 retval = POP();
2635
2636 if (co->co_flags & CO_ASYNC_GENERATOR) {
2637 PyObject *w = _PyAsyncGenValueWrapperNew(retval);
2638 Py_DECREF(retval);
2639 if (w == NULL) {
2640 retval = NULL;
2641 goto error;
2642 }
2643 retval = w;
2644 }
2645 f->f_state = FRAME_SUSPENDED;
2646 f->f_stackdepth = (int)(stack_pointer - f->f_valuestack);
2647 goto exiting;
2648 }
2649
2650 case TARGET(GEN_START): {
2651 PyObject *none = POP();
2652 assert(none == Py_None);
2653 assert(oparg < 3);
2654 Py_DECREF(none);
2655 DISPATCH();
2656 }
2657
2658 case TARGET(POP_EXCEPT): {
2659 PyObject *type, *value, *traceback;
2660 _PyErr_StackItem *exc_info;
2661 PyTryBlock *b = PyFrame_BlockPop(f);
2662 if (b->b_type != EXCEPT_HANDLER) {
2663 _PyErr_SetString(tstate, PyExc_SystemError,
2664 "popped block is not an except handler");
2665 goto error;
2666 }
2667 assert(STACK_LEVEL() >= (b)->b_level + 3 &&
2668 STACK_LEVEL() <= (b)->b_level + 4);
2669 exc_info = tstate->exc_info;
2670 type = exc_info->exc_type;
2671 value = exc_info->exc_value;
2672 traceback = exc_info->exc_traceback;
2673 exc_info->exc_type = POP();
2674 exc_info->exc_value = POP();
2675 exc_info->exc_traceback = POP();
2676 Py_XDECREF(type);
2677 Py_XDECREF(value);
2678 Py_XDECREF(traceback);
2679 DISPATCH();
2680 }
2681
2682 case TARGET(POP_BLOCK): {
2683 PyFrame_BlockPop(f);
2684 DISPATCH();
2685 }
2686
2687 case TARGET(RERAISE): {
2688 assert(f->f_iblock > 0);
2689 if (oparg) {
2690 f->f_lasti = f->f_blockstack[f->f_iblock-1].b_handler;
2691 }
2692 PyObject *exc = POP();
2693 PyObject *val = POP();
2694 PyObject *tb = POP();
2695 assert(PyExceptionClass_Check(exc));
2696 _PyErr_Restore(tstate, exc, val, tb);
2697 goto exception_unwind;
2698 }
2699
2700 case TARGET(END_ASYNC_FOR): {
2701 PyObject *exc = POP();
2702 assert(PyExceptionClass_Check(exc));
2703 if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) {
2704 PyTryBlock *b = PyFrame_BlockPop(f);
2705 assert(b->b_type == EXCEPT_HANDLER);
2706 Py_DECREF(exc);
2707 UNWIND_EXCEPT_HANDLER(b);
2708 Py_DECREF(POP());
2709 JUMPBY(oparg);
2710 DISPATCH();
2711 }
2712 else {
2713 PyObject *val = POP();
2714 PyObject *tb = POP();
2715 _PyErr_Restore(tstate, exc, val, tb);
2716 goto exception_unwind;
2717 }
2718 }
2719
2720 case TARGET(LOAD_ASSERTION_ERROR): {
2721 PyObject *value = PyExc_AssertionError;
2722 Py_INCREF(value);
2723 PUSH(value);
2724 DISPATCH();
2725 }
2726
2727 case TARGET(LOAD_BUILD_CLASS): {
2728 _Py_IDENTIFIER(__build_class__);
2729
2730 PyObject *bc;
2731 if (PyDict_CheckExact(f->f_builtins)) {
2732 bc = _PyDict_GetItemIdWithError(f->f_builtins, &PyId___build_class__);
2733 if (bc == NULL) {
2734 if (!_PyErr_Occurred(tstate)) {
2735 _PyErr_SetString(tstate, PyExc_NameError,
2736 "__build_class__ not found");
2737 }
2738 goto error;
2739 }
2740 Py_INCREF(bc);
2741 }
2742 else {
2743 PyObject *build_class_str = _PyUnicode_FromId(&PyId___build_class__);
2744 if (build_class_str == NULL)
2745 goto error;
2746 bc = PyObject_GetItem(f->f_builtins, build_class_str);
2747 if (bc == NULL) {
2748 if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
2749 _PyErr_SetString(tstate, PyExc_NameError,
2750 "__build_class__ not found");
2751 goto error;
2752 }
2753 }
2754 PUSH(bc);
2755 DISPATCH();
2756 }
2757
2758 case TARGET(STORE_NAME): {
2759 PyObject *name = GETITEM(names, oparg);
2760 PyObject *v = POP();
2761 PyObject *ns = f->f_locals;
2762 int err;
2763 if (ns == NULL) {
2764 _PyErr_Format(tstate, PyExc_SystemError,
2765 "no locals found when storing %R", name);
2766 Py_DECREF(v);
2767 goto error;
2768 }
2769 if (PyDict_CheckExact(ns))
2770 err = PyDict_SetItem(ns, name, v);
2771 else
2772 err = PyObject_SetItem(ns, name, v);
2773 Py_DECREF(v);
2774 if (err != 0)
2775 goto error;
2776 DISPATCH();
2777 }
2778
2779 case TARGET(DELETE_NAME): {
2780 PyObject *name = GETITEM(names, oparg);
2781 PyObject *ns = f->f_locals;
2782 int err;
2783 if (ns == NULL) {
2784 _PyErr_Format(tstate, PyExc_SystemError,
2785 "no locals when deleting %R", name);
2786 goto error;
2787 }
2788 err = PyObject_DelItem(ns, name);
2789 if (err != 0) {
2790 format_exc_check_arg(tstate, PyExc_NameError,
2791 NAME_ERROR_MSG,
2792 name);
2793 goto error;
2794 }
2795 DISPATCH();
2796 }
2797
2798 case TARGET(UNPACK_SEQUENCE): {
2799 PREDICTED(UNPACK_SEQUENCE);
2800 PyObject *seq = POP(), *item, **items;
2801 if (PyTuple_CheckExact(seq) &&
2802 PyTuple_GET_SIZE(seq) == oparg) {
2803 items = ((PyTupleObject *)seq)->ob_item;
2804 while (oparg--) {
2805 item = items[oparg];
2806 Py_INCREF(item);
2807 PUSH(item);
2808 }
2809 } else if (PyList_CheckExact(seq) &&
2810 PyList_GET_SIZE(seq) == oparg) {
2811 items = ((PyListObject *)seq)->ob_item;
2812 while (oparg--) {
2813 item = items[oparg];
2814 Py_INCREF(item);
2815 PUSH(item);
2816 }
2817 } else if (unpack_iterable(tstate, seq, oparg, -1,
2818 stack_pointer + oparg)) {
2819 STACK_GROW(oparg);
2820 } else {
2821 /* unpack_iterable() raised an exception */
2822 Py_DECREF(seq);
2823 goto error;
2824 }
2825 Py_DECREF(seq);
2826 DISPATCH();
2827 }
2828
2829 case TARGET(UNPACK_EX): {
2830 int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
2831 PyObject *seq = POP();
2832
2833 if (unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8,
2834 stack_pointer + totalargs)) {
2835 stack_pointer += totalargs;
2836 } else {
2837 Py_DECREF(seq);
2838 goto error;
2839 }
2840 Py_DECREF(seq);
2841 DISPATCH();
2842 }
2843
2844 case TARGET(STORE_ATTR): {
2845 PyObject *name = GETITEM(names, oparg);
2846 PyObject *owner = TOP();
2847 PyObject *v = SECOND();
2848 int err;
2849 STACK_SHRINK(2);
2850 err = PyObject_SetAttr(owner, name, v);
2851 Py_DECREF(v);
2852 Py_DECREF(owner);
2853 if (err != 0)
2854 goto error;
2855 DISPATCH();
2856 }
2857
2858 case TARGET(DELETE_ATTR): {
2859 PyObject *name = GETITEM(names, oparg);
2860 PyObject *owner = POP();
2861 int err;
2862 err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
2863 Py_DECREF(owner);
2864 if (err != 0)
2865 goto error;
2866 DISPATCH();
2867 }
2868
2869 case TARGET(STORE_GLOBAL): {
2870 PyObject *name = GETITEM(names, oparg);
2871 PyObject *v = POP();
2872 int err;
2873 err = PyDict_SetItem(f->f_globals, name, v);
2874 Py_DECREF(v);
2875 if (err != 0)
2876 goto error;
2877 DISPATCH();
2878 }
2879
2880 case TARGET(DELETE_GLOBAL): {
2881 PyObject *name = GETITEM(names, oparg);
2882 int err;
2883 err = PyDict_DelItem(f->f_globals, name);
2884 if (err != 0) {
2885 if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
2886 format_exc_check_arg(tstate, PyExc_NameError,
2887 NAME_ERROR_MSG, name);
2888 }
2889 goto error;
2890 }
2891 DISPATCH();
2892 }
2893
2894 case TARGET(LOAD_NAME): {
2895 PyObject *name = GETITEM(names, oparg);
2896 PyObject *locals = f->f_locals;
2897 PyObject *v;
2898 if (locals == NULL) {
2899 _PyErr_Format(tstate, PyExc_SystemError,
2900 "no locals when loading %R", name);
2901 goto error;
2902 }
2903 if (PyDict_CheckExact(locals)) {
2904 v = PyDict_GetItemWithError(locals, name);
2905 if (v != NULL) {
2906 Py_INCREF(v);
2907 }
2908 else if (_PyErr_Occurred(tstate)) {
2909 goto error;
2910 }
2911 }
2912 else {
2913 v = PyObject_GetItem(locals, name);
2914 if (v == NULL) {
2915 if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
2916 goto error;
2917 _PyErr_Clear(tstate);
2918 }
2919 }
2920 if (v == NULL) {
2921 v = PyDict_GetItemWithError(f->f_globals, name);
2922 if (v != NULL) {
2923 Py_INCREF(v);
2924 }
2925 else if (_PyErr_Occurred(tstate)) {
2926 goto error;
2927 }
2928 else {
2929 if (PyDict_CheckExact(f->f_builtins)) {
2930 v = PyDict_GetItemWithError(f->f_builtins, name);
2931 if (v == NULL) {
2932 if (!_PyErr_Occurred(tstate)) {
2933 format_exc_check_arg(
2934 tstate, PyExc_NameError,
2935 NAME_ERROR_MSG, name);
2936 }
2937 goto error;
2938 }
2939 Py_INCREF(v);
2940 }
2941 else {
2942 v = PyObject_GetItem(f->f_builtins, name);
2943 if (v == NULL) {
2944 if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
2945 format_exc_check_arg(
2946 tstate, PyExc_NameError,
2947 NAME_ERROR_MSG, name);
2948 }
2949 goto error;
2950 }
2951 }
2952 }
2953 }
2954 PUSH(v);
2955 DISPATCH();
2956 }
2957
2958 case TARGET(LOAD_GLOBAL): {
2959 PyObject *name;
2960 PyObject *v;
2961 if (PyDict_CheckExact(f->f_globals)
2962 && PyDict_CheckExact(f->f_builtins))
2963 {
2964 OPCACHE_CHECK();
2965 if (co_opcache != NULL && co_opcache->optimized > 0) {
2966 _PyOpcache_LoadGlobal *lg = &co_opcache->u.lg;
2967
2968 if (lg->globals_ver ==
2969 ((PyDictObject *)f->f_globals)->ma_version_tag
2970 && lg->builtins_ver ==
2971 ((PyDictObject *)f->f_builtins)->ma_version_tag)
2972 {
2973 PyObject *ptr = lg->ptr;
2974 OPCACHE_STAT_GLOBAL_HIT();
2975 assert(ptr != NULL);
2976 Py_INCREF(ptr);
2977 PUSH(ptr);
2978 DISPATCH();
2979 }
2980 }
2981
2982 name = GETITEM(names, oparg);
2983 v = _PyDict_LoadGlobal((PyDictObject *)f->f_globals,
2984 (PyDictObject *)f->f_builtins,
2985 name);
2986 if (v == NULL) {
2987 if (!_PyErr_Occurred(tstate)) {
2988 /* _PyDict_LoadGlobal() returns NULL without raising
2989 * an exception if the key doesn't exist */
2990 format_exc_check_arg(tstate, PyExc_NameError,
2991 NAME_ERROR_MSG, name);
2992 }
2993 goto error;
2994 }
2995
2996 if (co_opcache != NULL) {
2997 _PyOpcache_LoadGlobal *lg = &co_opcache->u.lg;
2998
2999 if (co_opcache->optimized == 0) {
3000 /* Wasn't optimized before. */
3001 OPCACHE_STAT_GLOBAL_OPT();
3002 } else {
3003 OPCACHE_STAT_GLOBAL_MISS();
3004 }
3005
3006 co_opcache->optimized = 1;
3007 lg->globals_ver =
3008 ((PyDictObject *)f->f_globals)->ma_version_tag;
3009 lg->builtins_ver =
3010 ((PyDictObject *)f->f_builtins)->ma_version_tag;
3011 lg->ptr = v; /* borrowed */
3012 }
3013
3014 Py_INCREF(v);
3015 }
3016 else {
3017 /* Slow-path if globals or builtins is not a dict */
3018
3019 /* namespace 1: globals */
3020 name = GETITEM(names, oparg);
3021 v = PyObject_GetItem(f->f_globals, name);
3022 if (v == NULL) {
3023 if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
3024 goto error;
3025 }
3026 _PyErr_Clear(tstate);
3027
3028 /* namespace 2: builtins */
3029 v = PyObject_GetItem(f->f_builtins, name);
3030 if (v == NULL) {
3031 if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
3032 format_exc_check_arg(
3033 tstate, PyExc_NameError,
3034 NAME_ERROR_MSG, name);
3035 }
3036 goto error;
3037 }
3038 }
3039 }
3040 PUSH(v);
3041 DISPATCH();
3042 }
3043
3044 case TARGET(DELETE_FAST): {
3045 PyObject *v = GETLOCAL(oparg);
3046 if (v != NULL) {
3047 SETLOCAL(oparg, NULL);
3048 DISPATCH();
3049 }
3050 format_exc_check_arg(
3051 tstate, PyExc_UnboundLocalError,
3052 UNBOUNDLOCAL_ERROR_MSG,
3053 PyTuple_GetItem(co->co_varnames, oparg)
3054 );
3055 goto error;
3056 }
3057
3058 case TARGET(DELETE_DEREF): {
3059 PyObject *cell = freevars[oparg];
3060 PyObject *oldobj = PyCell_GET(cell);
3061 if (oldobj != NULL) {
3062 PyCell_SET(cell, NULL);
3063 Py_DECREF(oldobj);
3064 DISPATCH();
3065 }
3066 format_exc_unbound(tstate, co, oparg);
3067 goto error;
3068 }
3069
3070 case TARGET(LOAD_CLOSURE): {
3071 PyObject *cell = freevars[oparg];
3072 Py_INCREF(cell);
3073 PUSH(cell);
3074 DISPATCH();
3075 }
3076
3077 case TARGET(LOAD_CLASSDEREF): {
3078 PyObject *name, *value, *locals = f->f_locals;
3079 Py_ssize_t idx;
3080 assert(locals);
3081 assert(oparg >= PyTuple_GET_SIZE(co->co_cellvars));
3082 idx = oparg - PyTuple_GET_SIZE(co->co_cellvars);
3083 assert(idx >= 0 && idx < PyTuple_GET_SIZE(co->co_freevars));
3084 name = PyTuple_GET_ITEM(co->co_freevars, idx);
3085 if (PyDict_CheckExact(locals)) {
3086 value = PyDict_GetItemWithError(locals, name);
3087 if (value != NULL) {
3088 Py_INCREF(value);
3089 }
3090 else if (_PyErr_Occurred(tstate)) {
3091 goto error;
3092 }
3093 }
3094 else {
3095 value = PyObject_GetItem(locals, name);
3096 if (value == NULL) {
3097 if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
3098 goto error;
3099 }
3100 _PyErr_Clear(tstate);
3101 }
3102 }
3103 if (!value) {
3104 PyObject *cell = freevars[oparg];
3105 value = PyCell_GET(cell);
3106 if (value == NULL) {
3107 format_exc_unbound(tstate, co, oparg);
3108 goto error;
3109 }
3110 Py_INCREF(value);
3111 }
3112 PUSH(value);
3113 DISPATCH();
3114 }
3115
3116 case TARGET(LOAD_DEREF): {
3117 PyObject *cell = freevars[oparg];
3118 PyObject *value = PyCell_GET(cell);
3119 if (value == NULL) {
3120 format_exc_unbound(tstate, co, oparg);
3121 goto error;
3122 }
3123 Py_INCREF(value);
3124 PUSH(value);
3125 DISPATCH();
3126 }
3127
3128 case TARGET(STORE_DEREF): {
3129 PyObject *v = POP();
3130 PyObject *cell = freevars[oparg];
3131 PyObject *oldobj = PyCell_GET(cell);
3132 PyCell_SET(cell, v);
3133 Py_XDECREF(oldobj);
3134 DISPATCH();
3135 }
3136
3137 case TARGET(BUILD_STRING): {
3138 PyObject *str;
3139 PyObject *empty = PyUnicode_New(0, 0);
3140 if (empty == NULL) {
3141 goto error;
3142 }
3143 str = _PyUnicode_JoinArray(empty, stack_pointer - oparg, oparg);
3144 Py_DECREF(empty);
3145 if (str == NULL)
3146 goto error;
3147 while (--oparg >= 0) {
3148 PyObject *item = POP();
3149 Py_DECREF(item);
3150 }
3151 PUSH(str);
3152 DISPATCH();
3153 }
3154
3155 case TARGET(BUILD_TUPLE): {
3156 PyObject *tup = PyTuple_New(oparg);
3157 if (tup == NULL)
3158 goto error;
3159 while (--oparg >= 0) {
3160 PyObject *item = POP();
3161 PyTuple_SET_ITEM(tup, oparg, item);
3162 }
3163 PUSH(tup);
3164 DISPATCH();
3165 }
3166
3167 case TARGET(BUILD_LIST): {
3168 PyObject *list = PyList_New(oparg);
3169 if (list == NULL)
3170 goto error;
3171 while (--oparg >= 0) {
3172 PyObject *item = POP();
3173 PyList_SET_ITEM(list, oparg, item);
3174 }
3175 PUSH(list);
3176 DISPATCH();
3177 }
3178
3179 case TARGET(LIST_TO_TUPLE): {
3180 PyObject *list = POP();
3181 PyObject *tuple = PyList_AsTuple(list);
3182 Py_DECREF(list);
3183 if (tuple == NULL) {
3184 goto error;
3185 }
3186 PUSH(tuple);
3187 DISPATCH();
3188 }
3189
3190 case TARGET(LIST_EXTEND): {
3191 PyObject *iterable = POP();
3192 PyObject *list = PEEK(oparg);
3193 PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
3194 if (none_val == NULL) {
3195 if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
3196 (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
3197 {
3198 _PyErr_Clear(tstate);
3199 _PyErr_Format(tstate, PyExc_TypeError,
3200 "Value after * must be an iterable, not %.200s",
3201 Py_TYPE(iterable)->tp_name);
3202 }
3203 Py_DECREF(iterable);
3204 goto error;
3205 }
3206 Py_DECREF(none_val);
3207 Py_DECREF(iterable);
3208 DISPATCH();
3209 }
3210
3211 case TARGET(SET_UPDATE): {
3212 PyObject *iterable = POP();
3213 PyObject *set = PEEK(oparg);
3214 int err = _PySet_Update(set, iterable);
3215 Py_DECREF(iterable);
3216 if (err < 0) {
3217 goto error;
3218 }
3219 DISPATCH();
3220 }
3221
3222 case TARGET(BUILD_SET): {
3223 PyObject *set = PySet_New(NULL);
3224 int err = 0;
3225 int i;
3226 if (set == NULL)
3227 goto error;
3228 for (i = oparg; i > 0; i--) {
3229 PyObject *item = PEEK(i);
3230 if (err == 0)
3231 err = PySet_Add(set, item);
3232 Py_DECREF(item);
3233 }
3234 STACK_SHRINK(oparg);
3235 if (err != 0) {
3236 Py_DECREF(set);
3237 goto error;
3238 }
3239 PUSH(set);
3240 DISPATCH();
3241 }
3242
3243 case TARGET(BUILD_MAP): {
3244 Py_ssize_t i;
3245 PyObject *map = _PyDict_NewPresized((Py_ssize_t)oparg);
3246 if (map == NULL)
3247 goto error;
3248 for (i = oparg; i > 0; i--) {
3249 int err;
3250 PyObject *key = PEEK(2*i);
3251 PyObject *value = PEEK(2*i - 1);
3252 err = PyDict_SetItem(map, key, value);
3253 if (err != 0) {
3254 Py_DECREF(map);
3255 goto error;
3256 }
3257 }
3258
3259 while (oparg--) {
3260 Py_DECREF(POP());
3261 Py_DECREF(POP());
3262 }
3263 PUSH(map);
3264 DISPATCH();
3265 }
3266
3267 case TARGET(SETUP_ANNOTATIONS): {
3268 _Py_IDENTIFIER(__annotations__);
3269 int err;
3270 PyObject *ann_dict;
3271 if (f->f_locals == NULL) {
3272 _PyErr_Format(tstate, PyExc_SystemError,
3273 "no locals found when setting up annotations");
3274 goto error;
3275 }
3276 /* check if __annotations__ in locals()... */
3277 if (PyDict_CheckExact(f->f_locals)) {
3278 ann_dict = _PyDict_GetItemIdWithError(f->f_locals,
3279 &PyId___annotations__);
3280 if (ann_dict == NULL) {
3281 if (_PyErr_Occurred(tstate)) {
3282 goto error;
3283 }
3284 /* ...if not, create a new one */
3285 ann_dict = PyDict_New();
3286 if (ann_dict == NULL) {
3287 goto error;
3288 }
3289 err = _PyDict_SetItemId(f->f_locals,
3290 &PyId___annotations__, ann_dict);
3291 Py_DECREF(ann_dict);
3292 if (err != 0) {
3293 goto error;
3294 }
3295 }
3296 }
3297 else {
3298 /* do the same if locals() is not a dict */
3299 PyObject *ann_str = _PyUnicode_FromId(&PyId___annotations__);
3300 if (ann_str == NULL) {
3301 goto error;
3302 }
3303 ann_dict = PyObject_GetItem(f->f_locals, ann_str);
3304 if (ann_dict == NULL) {
3305 if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
3306 goto error;
3307 }
3308 _PyErr_Clear(tstate);
3309 ann_dict = PyDict_New();
3310 if (ann_dict == NULL) {
3311 goto error;
3312 }
3313 err = PyObject_SetItem(f->f_locals, ann_str, ann_dict);
3314 Py_DECREF(ann_dict);
3315 if (err != 0) {
3316 goto error;
3317 }
3318 }
3319 else {
3320 Py_DECREF(ann_dict);
3321 }
3322 }
3323 DISPATCH();
3324 }
3325
3326 case TARGET(BUILD_CONST_KEY_MAP): {
3327 Py_ssize_t i;
3328 PyObject *map;
3329 PyObject *keys = TOP();
3330 if (!PyTuple_CheckExact(keys) ||
3331 PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
3332 _PyErr_SetString(tstate, PyExc_SystemError,
3333 "bad BUILD_CONST_KEY_MAP keys argument");
3334 goto error;
3335 }
3336 map = _PyDict_NewPresized((Py_ssize_t)oparg);
3337 if (map == NULL) {
3338 goto error;
3339 }
3340 for (i = oparg; i > 0; i--) {
3341 int err;
3342 PyObject *key = PyTuple_GET_ITEM(keys, oparg - i);
3343 PyObject *value = PEEK(i + 1);
3344 err = PyDict_SetItem(map, key, value);
3345 if (err != 0) {
3346 Py_DECREF(map);
3347 goto error;
3348 }
3349 }
3350
3351 Py_DECREF(POP());
3352 while (oparg--) {
3353 Py_DECREF(POP());
3354 }
3355 PUSH(map);
3356 DISPATCH();
3357 }
3358
3359 case TARGET(DICT_UPDATE): {
3360 PyObject *update = POP();
3361 PyObject *dict = PEEK(oparg);
3362 if (PyDict_Update(dict, update) < 0) {
3363 if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
3364 _PyErr_Format(tstate, PyExc_TypeError,
3365 "'%.200s' object is not a mapping",
3366 Py_TYPE(update)->tp_name);
3367 }
3368 Py_DECREF(update);
3369 goto error;
3370 }
3371 Py_DECREF(update);
3372 DISPATCH();
3373 }
3374
3375 case TARGET(DICT_MERGE): {
3376 PyObject *update = POP();
3377 PyObject *dict = PEEK(oparg);
3378
3379 if (_PyDict_MergeEx(dict, update, 2) < 0) {
3380 format_kwargs_error(tstate, PEEK(2 + oparg), update);
3381 Py_DECREF(update);
3382 goto error;
3383 }
3384 Py_DECREF(update);
3385 PREDICT(CALL_FUNCTION_EX);
3386 DISPATCH();
3387 }
3388
3389 case TARGET(MAP_ADD): {
3390 PyObject *value = TOP();
3391 PyObject *key = SECOND();
3392 PyObject *map;
3393 int err;
3394 STACK_SHRINK(2);
3395 map = PEEK(oparg); /* dict */
3396 assert(PyDict_CheckExact(map));
3397 err = PyDict_SetItem(map, key, value); /* map[key] = value */
3398 Py_DECREF(value);
3399 Py_DECREF(key);
3400 if (err != 0)
3401 goto error;
3402 PREDICT(JUMP_ABSOLUTE);
3403 DISPATCH();
3404 }
3405
3406 case TARGET(LOAD_ATTR): {
3407 PyObject *name = GETITEM(names, oparg);
3408 PyObject *owner = TOP();
3409
3410 PyTypeObject *type = Py_TYPE(owner);
3411 PyObject *res;
3412 PyObject **dictptr;
3413 PyObject *dict;
3414 _PyOpCodeOpt_LoadAttr *la;
3415
3416 OPCACHE_STAT_ATTR_TOTAL();
3417
3418 OPCACHE_CHECK();
3419 if (co_opcache != NULL && PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG))
3420 {
3421 if (co_opcache->optimized > 0) {
3422 // Fast path -- cache hit makes LOAD_ATTR ~30% faster.
3423 la = &co_opcache->u.la;
3424 if (la->type == type && la->tp_version_tag == type->tp_version_tag)
3425 {
3426 // Hint >= 0 is a dict index; hint == -1 is a dict miss.
3427 // Hint < -1 is an inverted slot offset: offset is strictly > 0,
3428 // so ~offset is strictly < -1 (assuming 2's complement).
3429 if (la->hint < -1) {
3430 // Even faster path -- slot hint.
3431 Py_ssize_t offset = ~la->hint;
3432 // fprintf(stderr, "Using hint for offset %zd\n", offset);
3433 char *addr = (char *)owner + offset;
3434 res = *(PyObject **)addr;
3435 if (res != NULL) {
3436 Py_INCREF(res);
3437 SET_TOP(res);
3438 Py_DECREF(owner);
3439 DISPATCH();
3440 }
3441 // Else slot is NULL. Fall through to slow path to raise AttributeError(name).
3442 // Don't DEOPT, since the slot is still there.
3443 } else {
3444 // Fast path for dict.
3445 assert(type->tp_dict != NULL);
3446 assert(type->tp_dictoffset > 0);
3447
3448 dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
3449 dict = *dictptr;
3450 if (dict != NULL && PyDict_CheckExact(dict)) {
3451 Py_ssize_t hint = la->hint;
3452 Py_INCREF(dict);
3453 res = NULL;
3454 assert(!_PyErr_Occurred(tstate));
3455 la->hint = _PyDict_GetItemHint((PyDictObject*)dict, name, hint, &res);
3456 if (res != NULL) {
3457 assert(la->hint >= 0);
3458 if (la->hint == hint && hint >= 0) {
3459 // Our hint has helped -- cache hit.
3460 OPCACHE_STAT_ATTR_HIT();
3461 } else {
3462 // The hint we provided didn't work.
3463 // Maybe next time?
3464 OPCACHE_MAYBE_DEOPT_LOAD_ATTR();
3465 }
3466
3467 Py_INCREF(res);
3468 SET_TOP(res);
3469 Py_DECREF(owner);
3470 Py_DECREF(dict);
3471 DISPATCH();
3472 }
3473 else {
3474 _PyErr_Clear(tstate);
3475 // This attribute can be missing sometimes;
3476 // we don't want to optimize this lookup.
3477 OPCACHE_DEOPT_LOAD_ATTR();
3478 Py_DECREF(dict);
3479 }
3480 }
3481 else {
3482 // There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact.
3483 OPCACHE_DEOPT_LOAD_ATTR();
3484 }
3485 }
3486 }
3487 else {
3488 // The type of the object has either been updated,
3489 // or is different. Maybe it will stabilize?
3490 OPCACHE_MAYBE_DEOPT_LOAD_ATTR();
3491 }
3492 OPCACHE_STAT_ATTR_MISS();
3493 }
3494
3495 if (co_opcache != NULL && // co_opcache can be NULL after a DEOPT() call.
3496 type->tp_getattro == PyObject_GenericGetAttr)
3497 {
3498 if (type->tp_dict == NULL) {
3499 if (PyType_Ready(type) < 0) {
3500 Py_DECREF(owner);
3501 SET_TOP(NULL);
3502 goto error;
3503 }
3504 }
3505 PyObject *descr = _PyType_Lookup(type, name);
3506 if (descr != NULL) {
3507 // We found an attribute with a data-like descriptor.
3508 PyTypeObject *dtype = Py_TYPE(descr);
3509 if (dtype == &PyMemberDescr_Type) { // It's a slot
3510 PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
3511 struct PyMemberDef *dmem = member->d_member;
3512 if (dmem->type == T_OBJECT_EX) {
3513 Py_ssize_t offset = dmem->offset;
3514 assert(offset > 0); // 0 would be confused with dict hint == -1 (miss).
3515
3516 if (co_opcache->optimized == 0) {
3517 // First time we optimize this opcode.
3518 OPCACHE_STAT_ATTR_OPT();
3519 co_opcache->optimized = OPCODE_CACHE_MAX_TRIES;
3520 // fprintf(stderr, "Setting hint for %s, offset %zd\n", dmem->name, offset);
3521 }
3522
3523 la = &co_opcache->u.la;
3524 la->type = type;
3525 la->tp_version_tag = type->tp_version_tag;
3526 la->hint = ~offset;
3527
3528 char *addr = (char *)owner + offset;
3529 res = *(PyObject **)addr;
3530 if (res != NULL) {
3531 Py_INCREF(res);
3532 Py_DECREF(owner);
3533 SET_TOP(res);
3534
3535 DISPATCH();
3536 }
3537 // Else slot is NULL. Fall through to slow path to raise AttributeError(name).
3538 }
3539 // Else it's a slot of a different type. We don't handle those.
3540 }
3541 // Else it's some other kind of descriptor that we don't handle.
3542 OPCACHE_DEOPT_LOAD_ATTR();
3543 }
3544 else if (type->tp_dictoffset > 0) {
3545 // We found an instance with a __dict__.
3546 dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
3547 dict = *dictptr;
3548
3549 if (dict != NULL && PyDict_CheckExact(dict)) {
3550 Py_INCREF(dict);
3551 res = NULL;
3552 assert(!_PyErr_Occurred(tstate));
3553 Py_ssize_t hint = _PyDict_GetItemHint((PyDictObject*)dict, name, -1, &res);
3554 if (res != NULL) {
3555 Py_INCREF(res);
3556 Py_DECREF(dict);
3557 Py_DECREF(owner);
3558 SET_TOP(res);
3559
3560 if (co_opcache->optimized == 0) {
3561 // First time we optimize this opcode.
3562 OPCACHE_STAT_ATTR_OPT();
3563 co_opcache->optimized = OPCODE_CACHE_MAX_TRIES;
3564 }
3565
3566 la = &co_opcache->u.la;
3567 la->type = type;
3568 la->tp_version_tag = type->tp_version_tag;
3569 assert(hint >= 0);
3570 la->hint = hint;
3571
3572 DISPATCH();
3573 }
3574 else {
3575 _PyErr_Clear(tstate);
3576 }
3577 Py_DECREF(dict);
3578 } else {
3579 // There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact.
3580 OPCACHE_DEOPT_LOAD_ATTR();
3581 }
3582 } else {
3583 // The object's class does not have a tp_dictoffset we can use.
3584 OPCACHE_DEOPT_LOAD_ATTR();
3585 }
3586 } else if (type->tp_getattro != PyObject_GenericGetAttr) {
3587 OPCACHE_DEOPT_LOAD_ATTR();
3588 }
3589 }
3590
3591 // Slow path.
3592 res = PyObject_GetAttr(owner, name);
3593 Py_DECREF(owner);
3594 SET_TOP(res);
3595 if (res == NULL)
3596 goto error;
3597 DISPATCH();
3598 }
3599
3600 case TARGET(COMPARE_OP): {
3601 assert(oparg <= Py_GE);
3602 PyObject *right = POP();
3603 PyObject *left = TOP();
3604 PyObject *res = PyObject_RichCompare(left, right, oparg);
3605 SET_TOP(res);
3606 Py_DECREF(left);
3607 Py_DECREF(right);
3608 if (res == NULL)
3609 goto error;
3610 PREDICT(POP_JUMP_IF_FALSE);
3611 PREDICT(POP_JUMP_IF_TRUE);
3612 DISPATCH();
3613 }
3614
3615 case TARGET(IS_OP): {
3616 PyObject *right = POP();
3617 PyObject *left = TOP();
3618 int res = Py_Is(left, right) ^ oparg;
3619 PyObject *b = res ? Py_True : Py_False;
3620 Py_INCREF(b);
3621 SET_TOP(b);
3622 Py_DECREF(left);
3623 Py_DECREF(right);
3624 PREDICT(POP_JUMP_IF_FALSE);
3625 PREDICT(POP_JUMP_IF_TRUE);
3626 DISPATCH();
3627 }
3628
3629 case TARGET(CONTAINS_OP): {
3630 PyObject *right = POP();
3631 PyObject *left = POP();
3632 int res = PySequence_Contains(right, left);
3633 Py_DECREF(left);
3634 Py_DECREF(right);
3635 if (res < 0) {
3636 goto error;
3637 }
3638 PyObject *b = (res^oparg) ? Py_True : Py_False;
3639 Py_INCREF(b);
3640 PUSH(b);
3641 PREDICT(POP_JUMP_IF_FALSE);
3642 PREDICT(POP_JUMP_IF_TRUE);
3643 DISPATCH();
3644 }
3645
3646 #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\
3647 "BaseException is not allowed"
3648
3649 case TARGET(JUMP_IF_NOT_EXC_MATCH): {
3650 PyObject *right = POP();
3651 PyObject *left = POP();
3652 if (PyTuple_Check(right)) {
3653 Py_ssize_t i, length;
3654 length = PyTuple_GET_SIZE(right);
3655 for (i = 0; i < length; i++) {
3656 PyObject *exc = PyTuple_GET_ITEM(right, i);
3657 if (!PyExceptionClass_Check(exc)) {
3658 _PyErr_SetString(tstate, PyExc_TypeError,
3659 CANNOT_CATCH_MSG);
3660 Py_DECREF(left);
3661 Py_DECREF(right);
3662 goto error;
3663 }
3664 }
3665 }
3666 else {
3667 if (!PyExceptionClass_Check(right)) {
3668 _PyErr_SetString(tstate, PyExc_TypeError,
3669 CANNOT_CATCH_MSG);
3670 Py_DECREF(left);
3671 Py_DECREF(right);
3672 goto error;
3673 }
3674 }
3675 int res = PyErr_GivenExceptionMatches(left, right);
3676 Py_DECREF(left);
3677 Py_DECREF(right);
3678 if (res > 0) {
3679 /* Exception matches -- Do nothing */;
3680 }
3681 else if (res == 0) {
3682 JUMPTO(oparg);
3683 }
3684 else {
3685 goto error;
3686 }
3687 DISPATCH();
3688 }
3689
3690 case TARGET(IMPORT_NAME): {
3691 PyObject *name = GETITEM(names, oparg);
3692 PyObject *fromlist = POP();
3693 PyObject *level = TOP();
3694 PyObject *res;
3695 res = import_name(tstate, f, name, fromlist, level);
3696 Py_DECREF(level);
3697 Py_DECREF(fromlist);
3698 SET_TOP(res);
3699 if (res == NULL)
3700 goto error;
3701 DISPATCH();
3702 }
3703
3704 case TARGET(IMPORT_STAR): {
3705 PyObject *from = POP(), *locals;
3706 int err;
3707 if (PyFrame_FastToLocalsWithError(f) < 0) {
3708 Py_DECREF(from);
3709 goto error;
3710 }
3711
3712 locals = f->f_locals;
3713 if (locals == NULL) {
3714 _PyErr_SetString(tstate, PyExc_SystemError,
3715 "no locals found during 'import *'");
3716 Py_DECREF(from);
3717 goto error;
3718 }
3719 err = import_all_from(tstate, locals, from);
3720 PyFrame_LocalsToFast(f, 0);
3721 Py_DECREF(from);
3722 if (err != 0)
3723 goto error;
3724 DISPATCH();
3725 }
3726
3727 case TARGET(IMPORT_FROM): {
3728 PyObject *name = GETITEM(names, oparg);
3729 PyObject *from = TOP();
3730 PyObject *res;
3731 res = import_from(tstate, from, name);
3732 PUSH(res);
3733 if (res == NULL)
3734 goto error;
3735 DISPATCH();
3736 }
3737
3738 case TARGET(JUMP_FORWARD): {
3739 JUMPBY(oparg);
3740 DISPATCH();
3741 }
3742
3743 case TARGET(POP_JUMP_IF_FALSE): {
3744 PREDICTED(POP_JUMP_IF_FALSE);
3745 PyObject *cond = POP();
3746 int err;
3747 if (Py_IsTrue(cond)) {
3748 Py_DECREF(cond);
3749 DISPATCH();
3750 }
3751 if (Py_IsFalse(cond)) {
3752 Py_DECREF(cond);
3753 JUMPTO(oparg);
3754 CHECK_EVAL_BREAKER();
3755 DISPATCH();
3756 }
3757 err = PyObject_IsTrue(cond);
3758 Py_DECREF(cond);
3759 if (err > 0)
3760 ;
3761 else if (err == 0) {
3762 JUMPTO(oparg);
3763 CHECK_EVAL_BREAKER();
3764 }
3765 else
3766 goto error;
3767 DISPATCH();
3768 }
3769
3770 case TARGET(POP_JUMP_IF_TRUE): {
3771 PREDICTED(POP_JUMP_IF_TRUE);
3772 PyObject *cond = POP();
3773 int err;
3774 if (Py_IsFalse(cond)) {
3775 Py_DECREF(cond);
3776 DISPATCH();
3777 }
3778 if (Py_IsTrue(cond)) {
3779 Py_DECREF(cond);
3780 JUMPTO(oparg);
3781 CHECK_EVAL_BREAKER();
3782 DISPATCH();
3783 }
3784 err = PyObject_IsTrue(cond);
3785 Py_DECREF(cond);
3786 if (err > 0) {
3787 JUMPTO(oparg);
3788 CHECK_EVAL_BREAKER();
3789 }
3790 else if (err == 0)
3791 ;
3792 else
3793 goto error;
3794 DISPATCH();
3795 }
3796
3797 case TARGET(JUMP_IF_FALSE_OR_POP): {
3798 PyObject *cond = TOP();
3799 int err;
3800 if (Py_IsTrue(cond)) {
3801 STACK_SHRINK(1);
3802 Py_DECREF(cond);
3803 DISPATCH();
3804 }
3805 if (Py_IsFalse(cond)) {
3806 JUMPTO(oparg);
3807 DISPATCH();
3808 }
3809 err = PyObject_IsTrue(cond);
3810 if (err > 0) {
3811 STACK_SHRINK(1);
3812 Py_DECREF(cond);
3813 }
3814 else if (err == 0)
3815 JUMPTO(oparg);
3816 else
3817 goto error;
3818 DISPATCH();
3819 }
3820
3821 case TARGET(JUMP_IF_TRUE_OR_POP): {
3822 PyObject *cond = TOP();
3823 int err;
3824 if (Py_IsFalse(cond)) {
3825 STACK_SHRINK(1);
3826 Py_DECREF(cond);
3827 DISPATCH();
3828 }
3829 if (Py_IsTrue(cond)) {
3830 JUMPTO(oparg);
3831 DISPATCH();
3832 }
3833 err = PyObject_IsTrue(cond);
3834 if (err > 0) {
3835 JUMPTO(oparg);
3836 }
3837 else if (err == 0) {
3838 STACK_SHRINK(1);
3839 Py_DECREF(cond);
3840 }
3841 else
3842 goto error;
3843 DISPATCH();
3844 }
3845
3846 case TARGET(JUMP_ABSOLUTE): {
3847 PREDICTED(JUMP_ABSOLUTE);
3848 JUMPTO(oparg);
3849 CHECK_EVAL_BREAKER();
3850 DISPATCH();
3851 }
3852
3853 case TARGET(GET_LEN): {
3854 // PUSH(len(TOS))
3855 Py_ssize_t len_i = PyObject_Length(TOP());
3856 if (len_i < 0) {
3857 goto error;
3858 }
3859 PyObject *len_o = PyLong_FromSsize_t(len_i);
3860 if (len_o == NULL) {
3861 goto error;
3862 }
3863 PUSH(len_o);
3864 DISPATCH();
3865 }
3866
3867 case TARGET(MATCH_CLASS): {
3868 // Pop TOS. On success, set TOS to True and TOS1 to a tuple of
3869 // attributes. On failure, set TOS to False.
3870 PyObject *names = POP();
3871 PyObject *type = TOP();
3872 PyObject *subject = SECOND();
3873 assert(PyTuple_CheckExact(names));
3874 PyObject *attrs = match_class(tstate, subject, type, oparg, names);
3875 Py_DECREF(names);
3876 if (attrs) {
3877 // Success!
3878 assert(PyTuple_CheckExact(attrs));
3879 Py_DECREF(subject);
3880 SET_SECOND(attrs);
3881 }
3882 else if (_PyErr_Occurred(tstate)) {
3883 goto error;
3884 }
3885 Py_DECREF(type);
3886 SET_TOP(PyBool_FromLong(!!attrs));
3887 DISPATCH();
3888 }
3889
3890 case TARGET(MATCH_MAPPING): {
3891 PyObject *subject = TOP();
3892 int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
3893 PyObject *res = match ? Py_True : Py_False;
3894 Py_INCREF(res);
3895 PUSH(res);
3896 DISPATCH();
3897 }
3898
3899 case TARGET(MATCH_SEQUENCE): {
3900 PyObject *subject = TOP();
3901 int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
3902 PyObject *res = match ? Py_True : Py_False;
3903 Py_INCREF(res);
3904 PUSH(res);
3905 DISPATCH();
3906 }
3907
3908 case TARGET(MATCH_KEYS): {
3909 // On successful match for all keys, PUSH(values) and PUSH(True).
3910 // Otherwise, PUSH(None) and PUSH(False).
3911 PyObject *keys = TOP();
3912 PyObject *subject = SECOND();
3913 PyObject *values_or_none = match_keys(tstate, subject, keys);
3914 if (values_or_none == NULL) {
3915 goto error;
3916 }
3917 PUSH(values_or_none);
3918 if (Py_IsNone(values_or_none)) {
3919 Py_INCREF(Py_False);
3920 PUSH(Py_False);
3921 DISPATCH();
3922 }
3923 assert(PyTuple_CheckExact(values_or_none));
3924 Py_INCREF(Py_True);
3925 PUSH(Py_True);
3926 DISPATCH();
3927 }
3928
3929 case TARGET(COPY_DICT_WITHOUT_KEYS): {
3930 // rest = dict(TOS1)
3931 // for key in TOS:
3932 // del rest[key]
3933 // SET_TOP(rest)
3934 PyObject *keys = TOP();
3935 PyObject *subject = SECOND();
3936 PyObject *rest = PyDict_New();
3937 if (rest == NULL || PyDict_Update(rest, subject)) {
3938 Py_XDECREF(rest);
3939 goto error;
3940 }
3941 // This may seem a bit inefficient, but keys is rarely big enough to
3942 // actually impact runtime.
3943 assert(PyTuple_CheckExact(keys));
3944 for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(keys); i++) {
3945 if (PyDict_DelItem(rest, PyTuple_GET_ITEM(keys, i))) {
3946 Py_DECREF(rest);
3947 goto error;
3948 }
3949 }
3950 Py_DECREF(keys);
3951 SET_TOP(rest);
3952 DISPATCH();
3953 }
3954
3955 case TARGET(GET_ITER): {
3956 /* before: [obj]; after [getiter(obj)] */
3957 PyObject *iterable = TOP();
3958 PyObject *iter = PyObject_GetIter(iterable);
3959 Py_DECREF(iterable);
3960 SET_TOP(iter);
3961 if (iter == NULL)
3962 goto error;
3963 PREDICT(FOR_ITER);
3964 PREDICT(CALL_FUNCTION);
3965 DISPATCH();
3966 }
3967
3968 case TARGET(GET_YIELD_FROM_ITER): {
3969 /* before: [obj]; after [getiter(obj)] */
3970 PyObject *iterable = TOP();
3971 PyObject *iter;
3972 if (PyCoro_CheckExact(iterable)) {
3973 /* `iterable` is a coroutine */
3974 if (!(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
3975 /* and it is used in a 'yield from' expression of a
3976 regular generator. */
3977 Py_DECREF(iterable);
3978 SET_TOP(NULL);
3979 _PyErr_SetString(tstate, PyExc_TypeError,
3980 "cannot 'yield from' a coroutine object "
3981 "in a non-coroutine generator");
3982 goto error;
3983 }
3984 }
3985 else if (!PyGen_CheckExact(iterable)) {
3986 /* `iterable` is not a generator. */
3987 iter = PyObject_GetIter(iterable);
3988 Py_DECREF(iterable);
3989 SET_TOP(iter);
3990 if (iter == NULL)
3991 goto error;
3992 }
3993 PREDICT(LOAD_CONST);
3994 DISPATCH();
3995 }
3996
3997 case TARGET(FOR_ITER): {
3998 PREDICTED(FOR_ITER);
3999 /* before: [iter]; after: [iter, iter()] *or* [] */
4000 PyObject *iter = TOP();
4001 PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
4002 if (next != NULL) {
4003 PUSH(next);
4004 PREDICT(STORE_FAST);
4005 PREDICT(UNPACK_SEQUENCE);
4006 DISPATCH();
4007 }
4008 if (_PyErr_Occurred(tstate)) {
4009 if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
4010 goto error;
4011 }
4012 else if (tstate->c_tracefunc != NULL) {
4013 call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f, &trace_info);
4014 }
4015 _PyErr_Clear(tstate);
4016 }
4017 /* iterator ended normally */
4018 STACK_SHRINK(1);
4019 Py_DECREF(iter);
4020 JUMPBY(oparg);
4021 DISPATCH();
4022 }
4023
4024 case TARGET(SETUP_FINALLY): {
4025 PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg,
4026 STACK_LEVEL());
4027 DISPATCH();
4028 }
4029
4030 case TARGET(BEFORE_ASYNC_WITH): {
4031 _Py_IDENTIFIER(__aenter__);
4032 _Py_IDENTIFIER(__aexit__);
4033 PyObject *mgr = TOP();
4034 PyObject *enter = special_lookup(tstate, mgr, &PyId___aenter__);
4035 PyObject *res;
4036 if (enter == NULL) {
4037 goto error;
4038 }
4039 PyObject *exit = special_lookup(tstate, mgr, &PyId___aexit__);
4040 if (exit == NULL) {
4041 Py_DECREF(enter);
4042 goto error;
4043 }
4044 SET_TOP(exit);
4045 Py_DECREF(mgr);
4046 res = _PyObject_CallNoArg(enter);
4047 Py_DECREF(enter);
4048 if (res == NULL)
4049 goto error;
4050 PUSH(res);
4051 PREDICT(GET_AWAITABLE);
4052 DISPATCH();
4053 }
4054
4055 case TARGET(SETUP_ASYNC_WITH): {
4056 PyObject *res = POP();
4057 /* Setup the finally block before pushing the result
4058 of __aenter__ on the stack. */
4059 PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg,
4060 STACK_LEVEL());
4061 PUSH(res);
4062 DISPATCH();
4063 }
4064
4065 case TARGET(SETUP_WITH): {
4066 _Py_IDENTIFIER(__enter__);
4067 _Py_IDENTIFIER(__exit__);
4068 PyObject *mgr = TOP();
4069 PyObject *enter = special_lookup(tstate, mgr, &PyId___enter__);
4070 PyObject *res;
4071 if (enter == NULL) {
4072 goto error;
4073 }
4074 PyObject *exit = special_lookup(tstate, mgr, &PyId___exit__);
4075 if (exit == NULL) {
4076 Py_DECREF(enter);
4077 goto error;
4078 }
4079 SET_TOP(exit);
4080 Py_DECREF(mgr);
4081 res = _PyObject_CallNoArg(enter);
4082 Py_DECREF(enter);
4083 if (res == NULL)
4084 goto error;
4085 /* Setup the finally block before pushing the result
4086 of __enter__ on the stack. */
4087 PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg,
4088 STACK_LEVEL());
4089
4090 PUSH(res);
4091 DISPATCH();
4092 }
4093
4094 case TARGET(WITH_EXCEPT_START): {
4095 /* At the top of the stack are 7 values:
4096 - (TOP, SECOND, THIRD) = exc_info()
4097 - (FOURTH, FIFTH, SIXTH) = previous exception for EXCEPT_HANDLER
4098 - SEVENTH: the context.__exit__ bound method
4099 We call SEVENTH(TOP, SECOND, THIRD).
4100 Then we push again the TOP exception and the __exit__
4101 return value.
4102 */
4103 PyObject *exit_func;
4104 PyObject *exc, *val, *tb, *res;
4105
4106 exc = TOP();
4107 val = SECOND();
4108 tb = THIRD();
4109 assert(!Py_IsNone(exc));
4110 assert(!PyLong_Check(exc));
4111 exit_func = PEEK(7);
4112 PyObject *stack[4] = {NULL, exc, val, tb};
4113 res = PyObject_Vectorcall(exit_func, stack + 1,
4114 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
4115 if (res == NULL)
4116 goto error;
4117
4118 PUSH(res);
4119 DISPATCH();
4120 }
4121
4122 case TARGET(LOAD_METHOD): {
4123 /* Designed to work in tandem with CALL_METHOD. */
4124 PyObject *name = GETITEM(names, oparg);
4125 PyObject *obj = TOP();
4126 PyObject *meth = NULL;
4127
4128 int meth_found = _PyObject_GetMethod(obj, name, &meth);
4129
4130 if (meth == NULL) {
4131 /* Most likely attribute wasn't found. */
4132 goto error;
4133 }
4134
4135 if (meth_found) {
4136 /* We can bypass temporary bound method object.
4137 meth is unbound method and obj is self.
4138
4139 meth | self | arg1 | ... | argN
4140 */
4141 SET_TOP(meth);
4142 PUSH(obj); // self
4143 }
4144 else {
4145 /* meth is not an unbound method (but a regular attr, or
4146 something was returned by a descriptor protocol). Set
4147 the second element of the stack to NULL, to signal
4148 CALL_METHOD that it's not a method call.
4149
4150 NULL | meth | arg1 | ... | argN
4151 */
4152 SET_TOP(NULL);
4153 Py_DECREF(obj);
4154 PUSH(meth);
4155 }
4156 DISPATCH();
4157 }
4158
4159 case TARGET(CALL_METHOD): {
4160 /* Designed to work in tamdem with LOAD_METHOD. */
4161 PyObject **sp, *res, *meth;
4162
4163 sp = stack_pointer;
4164
4165 meth = PEEK(oparg + 2);
4166 if (meth == NULL) {
4167 /* `meth` is NULL when LOAD_METHOD thinks that it's not
4168 a method call.
4169
4170 Stack layout:
4171
4172 ... | NULL | callable | arg1 | ... | argN
4173 ^- TOP()
4174 ^- (-oparg)
4175 ^- (-oparg-1)
4176 ^- (-oparg-2)
4177
4178 `callable` will be POPed by call_function.
4179 NULL will will be POPed manually later.
4180 */
4181 res = call_function(tstate, &trace_info, &sp, oparg, NULL);
4182 stack_pointer = sp;
4183 (void)POP(); /* POP the NULL. */
4184 }
4185 else {
4186 /* This is a method call. Stack layout:
4187
4188 ... | method | self | arg1 | ... | argN
4189 ^- TOP()
4190 ^- (-oparg)
4191 ^- (-oparg-1)
4192 ^- (-oparg-2)
4193
4194 `self` and `method` will be POPed by call_function.
4195 We'll be passing `oparg + 1` to call_function, to
4196 make it accept the `self` as a first argument.
4197 */
4198 res = call_function(tstate, &trace_info, &sp, oparg + 1, NULL);
4199 stack_pointer = sp;
4200 }
4201
4202 PUSH(res);
4203 if (res == NULL)
4204 goto error;
4205 CHECK_EVAL_BREAKER();
4206 DISPATCH();
4207 }
4208
4209 case TARGET(CALL_FUNCTION): {
4210 PREDICTED(CALL_FUNCTION);
4211 PyObject **sp, *res;
4212 sp = stack_pointer;
4213 res = call_function(tstate, &trace_info, &sp, oparg, NULL);
4214 stack_pointer = sp;
4215 PUSH(res);
4216 if (res == NULL) {
4217 goto error;
4218 }
4219 CHECK_EVAL_BREAKER();
4220 DISPATCH();
4221 }
4222
4223 case TARGET(CALL_FUNCTION_KW): {
4224 PyObject **sp, *res, *names;
4225
4226 names = POP();
4227 assert(PyTuple_Check(names));
4228 assert(PyTuple_GET_SIZE(names) <= oparg);
4229 /* We assume without checking that names contains only strings */
4230 sp = stack_pointer;
4231 res = call_function(tstate, &trace_info, &sp, oparg, names);
4232 stack_pointer = sp;
4233 PUSH(res);
4234 Py_DECREF(names);
4235
4236 if (res == NULL) {
4237 goto error;
4238 }
4239 CHECK_EVAL_BREAKER();
4240 DISPATCH();
4241 }
4242
4243 case TARGET(CALL_FUNCTION_EX): {
4244 PREDICTED(CALL_FUNCTION_EX);
4245 PyObject *func, *callargs, *kwargs = NULL, *result;
4246 if (oparg & 0x01) {
4247 kwargs = POP();
4248 if (!PyDict_CheckExact(kwargs)) {
4249 PyObject *d = PyDict_New();
4250 if (d == NULL)
4251 goto error;
4252 if (_PyDict_MergeEx(d, kwargs, 2) < 0) {
4253 Py_DECREF(d);
4254 format_kwargs_error(tstate, SECOND(), kwargs);
4255 Py_DECREF(kwargs);
4256 goto error;
4257 }
4258 Py_DECREF(kwargs);
4259 kwargs = d;
4260 }
4261 assert(PyDict_CheckExact(kwargs));
4262 }
4263 callargs = POP();
4264 func = TOP();
4265 if (!PyTuple_CheckExact(callargs)) {
4266 if (check_args_iterable(tstate, func, callargs) < 0) {
4267 Py_DECREF(callargs);
4268 goto error;
4269 }
4270 Py_SETREF(callargs, PySequence_Tuple(callargs));
4271 if (callargs == NULL) {
4272 goto error;
4273 }
4274 }
4275 assert(PyTuple_CheckExact(callargs));
4276
4277 result = do_call_core(tstate, &trace_info, func, callargs, kwargs);
4278 Py_DECREF(func);
4279 Py_DECREF(callargs);
4280 Py_XDECREF(kwargs);
4281
4282 SET_TOP(result);
4283 if (result == NULL) {
4284 goto error;
4285 }
4286 CHECK_EVAL_BREAKER();
4287 DISPATCH();
4288 }
4289
4290 case TARGET(MAKE_FUNCTION): {
4291 PyObject *qualname = POP();
4292 PyObject *codeobj = POP();
4293 PyFunctionObject *func = (PyFunctionObject *)
4294 PyFunction_NewWithQualName(codeobj, f->f_globals, qualname);
4295
4296 Py_DECREF(codeobj);
4297 Py_DECREF(qualname);
4298 if (func == NULL) {
4299 goto error;
4300 }
4301
4302 if (oparg & 0x08) {
4303 assert(PyTuple_CheckExact(TOP()));
4304 func->func_closure = POP();
4305 }
4306 if (oparg & 0x04) {
4307 assert(PyTuple_CheckExact(TOP()));
4308 func->func_annotations = POP();
4309 }
4310 if (oparg & 0x02) {
4311 assert(PyDict_CheckExact(TOP()));
4312 func->func_kwdefaults = POP();
4313 }
4314 if (oparg & 0x01) {
4315 assert(PyTuple_CheckExact(TOP()));
4316 func->func_defaults = POP();
4317 }
4318
4319 PUSH((PyObject *)func);
4320 DISPATCH();
4321 }
4322
4323 case TARGET(BUILD_SLICE): {
4324 PyObject *start, *stop, *step, *slice;
4325 if (oparg == 3)
4326 step = POP();
4327 else
4328 step = NULL;
4329 stop = POP();
4330 start = TOP();
4331 slice = PySlice_New(start, stop, step);
4332 Py_DECREF(start);
4333 Py_DECREF(stop);
4334 Py_XDECREF(step);
4335 SET_TOP(slice);
4336 if (slice == NULL)
4337 goto error;
4338 DISPATCH();
4339 }
4340
4341 case TARGET(FORMAT_VALUE): {
4342 /* Handles f-string value formatting. */
4343 PyObject *result;
4344 PyObject *fmt_spec;
4345 PyObject *value;
4346 PyObject *(*conv_fn)(PyObject *);
4347 int which_conversion = oparg & FVC_MASK;
4348 int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC;
4349
4350 fmt_spec = have_fmt_spec ? POP() : NULL;
4351 value = POP();
4352
4353 /* See if any conversion is specified. */
4354 switch (which_conversion) {
4355 case FVC_NONE: conv_fn = NULL; break;
4356 case FVC_STR: conv_fn = PyObject_Str; break;
4357 case FVC_REPR: conv_fn = PyObject_Repr; break;
4358 case FVC_ASCII: conv_fn = PyObject_ASCII; break;
4359 default:
4360 _PyErr_Format(tstate, PyExc_SystemError,
4361 "unexpected conversion flag %d",
4362 which_conversion);
4363 goto error;
4364 }
4365
4366 /* If there's a conversion function, call it and replace
4367 value with that result. Otherwise, just use value,
4368 without conversion. */
4369 if (conv_fn != NULL) {
4370 result = conv_fn(value);
4371 Py_DECREF(value);
4372 if (result == NULL) {
4373 Py_XDECREF(fmt_spec);
4374 goto error;
4375 }
4376 value = result;
4377 }
4378
4379 /* If value is a unicode object, and there's no fmt_spec,
4380 then we know the result of format(value) is value
4381 itself. In that case, skip calling format(). I plan to
4382 move this optimization in to PyObject_Format()
4383 itself. */
4384 if (PyUnicode_CheckExact(value) && fmt_spec == NULL) {
4385 /* Do nothing, just transfer ownership to result. */
4386 result = value;
4387 } else {
4388 /* Actually call format(). */
4389 result = PyObject_Format(value, fmt_spec);
4390 Py_DECREF(value);
4391 Py_XDECREF(fmt_spec);
4392 if (result == NULL) {
4393 goto error;
4394 }
4395 }
4396
4397 PUSH(result);
4398 DISPATCH();
4399 }
4400
4401 case TARGET(ROT_N): {
4402 PyObject *top = TOP();
4403 memmove(&PEEK(oparg - 1), &PEEK(oparg),
4404 sizeof(PyObject*) * (oparg - 1));
4405 PEEK(oparg) = top;
4406 DISPATCH();
4407 }
4408
4409 case TARGET(EXTENDED_ARG): {
4410 int oldoparg = oparg;
4411 NEXTOPARG();
4412 oparg |= oldoparg << 8;
4413 goto dispatch_opcode;
4414 }
4415
4416
4417 #if USE_COMPUTED_GOTOS
4418 _unknown_opcode:
4419 #endif
4420 default:
4421 fprintf(stderr,
4422 "XXX lineno: %d, opcode: %d\n",
4423 PyFrame_GetLineNumber(f),
4424 opcode);
4425 _PyErr_SetString(tstate, PyExc_SystemError, "unknown opcode");
4426 goto error;
4427
4428 } /* switch */
4429
4430 /* This should never be reached. Every opcode should end with DISPATCH()
4431 or goto error. */
4432 Py_UNREACHABLE();
4433
4434 error:
4435 /* Double-check exception status. */
4436 #ifdef NDEBUG
4437 if (!_PyErr_Occurred(tstate)) {
4438 _PyErr_SetString(tstate, PyExc_SystemError,
4439 "error return without exception set");
4440 }
4441 #else
4442 assert(_PyErr_Occurred(tstate));
4443 #endif
4444
4445 /* Log traceback info. */
4446 PyTraceBack_Here(f);
4447
4448 if (tstate->c_tracefunc != NULL) {
4449 /* Make sure state is set to FRAME_EXECUTING for tracing */
4450 assert(f->f_state == FRAME_EXECUTING);
4451 f->f_state = FRAME_UNWINDING;
4452 call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj,
4453 tstate, f, &trace_info);
4454 }
4455 exception_unwind:
4456 f->f_state = FRAME_UNWINDING;
4457 /* Unwind stacks if an exception occurred */
4458 while (f->f_iblock > 0) {
4459 /* Pop the current block. */
4460 PyTryBlock *b = &f->f_blockstack[--f->f_iblock];
4461
4462 if (b->b_type == EXCEPT_HANDLER) {
4463 UNWIND_EXCEPT_HANDLER(b);
4464 continue;
4465 }
4466 UNWIND_BLOCK(b);
4467 if (b->b_type == SETUP_FINALLY) {
4468 PyObject *exc, *val, *tb;
4469 int handler = b->b_handler;
4470 _PyErr_StackItem *exc_info = tstate->exc_info;
4471 /* Beware, this invalidates all b->b_* fields */
4472 PyFrame_BlockSetup(f, EXCEPT_HANDLER, f->f_lasti, STACK_LEVEL());
4473 PUSH(exc_info->exc_traceback);
4474 PUSH(exc_info->exc_value);
4475 if (exc_info->exc_type != NULL) {
4476 PUSH(exc_info->exc_type);
4477 }
4478 else {
4479 Py_INCREF(Py_None);
4480 PUSH(Py_None);
4481 }
4482 _PyErr_Fetch(tstate, &exc, &val, &tb);
4483 /* Make the raw exception data
4484 available to the handler,
4485 so a program can emulate the
4486 Python main loop. */
4487 _PyErr_NormalizeException(tstate, &exc, &val, &tb);
4488 if (tb != NULL)
4489 PyException_SetTraceback(val, tb);
4490 else
4491 PyException_SetTraceback(val, Py_None);
4492 Py_INCREF(exc);
4493 exc_info->exc_type = exc;
4494 Py_INCREF(val);
4495 exc_info->exc_value = val;
4496 exc_info->exc_traceback = tb;
4497 if (tb == NULL)
4498 tb = Py_None;
4499 Py_INCREF(tb);
4500 PUSH(tb);
4501 PUSH(val);
4502 PUSH(exc);
4503 JUMPTO(handler);
4504 /* Resume normal execution */
4505 f->f_state = FRAME_EXECUTING;
4506 goto main_loop;
4507 }
4508 } /* unwind stack */
4509
4510 /* End the loop as we still have an error */
4511 break;
4512 } /* main loop */
4513
4514 assert(retval == NULL);
4515 assert(_PyErr_Occurred(tstate));
4516
4517 /* Pop remaining stack entries. */
4518 while (!EMPTY()) {
4519 PyObject *o = POP();
4520 Py_XDECREF(o);
4521 }
4522 f->f_stackdepth = 0;
4523 f->f_state = FRAME_RAISED;
4524 exiting:
4525 if (trace_info.cframe.use_tracing) {
4526 if (tstate->c_tracefunc) {
4527 if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,
4528 tstate, f, &trace_info, PyTrace_RETURN, retval)) {
4529 Py_CLEAR(retval);
4530 }
4531 }
4532 if (tstate->c_profilefunc) {
4533 if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,
4534 tstate, f, &trace_info, PyTrace_RETURN, retval)) {
4535 Py_CLEAR(retval);
4536 }
4537 }
4538 }
4539
4540 /* pop frame */
4541 exit_eval_frame:
4542 /* Restore previous cframe */
4543 tstate->cframe = trace_info.cframe.previous;
4544 tstate->cframe->use_tracing = trace_info.cframe.use_tracing;
4545
4546 if (PyDTrace_FUNCTION_RETURN_ENABLED())
4547 dtrace_function_return(f);
4548 _Py_LeaveRecursiveCall(tstate);
4549 tstate->frame = f->f_back;
4550
4551 return _Py_CheckFunctionResult(tstate, NULL, retval, __func__);
4552 }
4553
4554 static void
format_missing(PyThreadState * tstate,const char * kind,PyCodeObject * co,PyObject * names,PyObject * qualname)4555 format_missing(PyThreadState *tstate, const char *kind,
4556 PyCodeObject *co, PyObject *names, PyObject *qualname)
4557 {
4558 int err;
4559 Py_ssize_t len = PyList_GET_SIZE(names);
4560 PyObject *name_str, *comma, *tail, *tmp;
4561
4562 assert(PyList_CheckExact(names));
4563 assert(len >= 1);
4564 /* Deal with the joys of natural language. */
4565 switch (len) {
4566 case 1:
4567 name_str = PyList_GET_ITEM(names, 0);
4568 Py_INCREF(name_str);
4569 break;
4570 case 2:
4571 name_str = PyUnicode_FromFormat("%U and %U",
4572 PyList_GET_ITEM(names, len - 2),
4573 PyList_GET_ITEM(names, len - 1));
4574 break;
4575 default:
4576 tail = PyUnicode_FromFormat(", %U, and %U",
4577 PyList_GET_ITEM(names, len - 2),
4578 PyList_GET_ITEM(names, len - 1));
4579 if (tail == NULL)
4580 return;
4581 /* Chop off the last two objects in the list. This shouldn't actually
4582 fail, but we can't be too careful. */
4583 err = PyList_SetSlice(names, len - 2, len, NULL);
4584 if (err == -1) {
4585 Py_DECREF(tail);
4586 return;
4587 }
4588 /* Stitch everything up into a nice comma-separated list. */
4589 comma = PyUnicode_FromString(", ");
4590 if (comma == NULL) {
4591 Py_DECREF(tail);
4592 return;
4593 }
4594 tmp = PyUnicode_Join(comma, names);
4595 Py_DECREF(comma);
4596 if (tmp == NULL) {
4597 Py_DECREF(tail);
4598 return;
4599 }
4600 name_str = PyUnicode_Concat(tmp, tail);
4601 Py_DECREF(tmp);
4602 Py_DECREF(tail);
4603 break;
4604 }
4605 if (name_str == NULL)
4606 return;
4607 _PyErr_Format(tstate, PyExc_TypeError,
4608 "%U() missing %i required %s argument%s: %U",
4609 qualname,
4610 len,
4611 kind,
4612 len == 1 ? "" : "s",
4613 name_str);
4614 Py_DECREF(name_str);
4615 }
4616
4617 static void
missing_arguments(PyThreadState * tstate,PyCodeObject * co,Py_ssize_t missing,Py_ssize_t defcount,PyObject ** fastlocals,PyObject * qualname)4618 missing_arguments(PyThreadState *tstate, PyCodeObject *co,
4619 Py_ssize_t missing, Py_ssize_t defcount,
4620 PyObject **fastlocals, PyObject *qualname)
4621 {
4622 Py_ssize_t i, j = 0;
4623 Py_ssize_t start, end;
4624 int positional = (defcount != -1);
4625 const char *kind = positional ? "positional" : "keyword-only";
4626 PyObject *missing_names;
4627
4628 /* Compute the names of the arguments that are missing. */
4629 missing_names = PyList_New(missing);
4630 if (missing_names == NULL)
4631 return;
4632 if (positional) {
4633 start = 0;
4634 end = co->co_argcount - defcount;
4635 }
4636 else {
4637 start = co->co_argcount;
4638 end = start + co->co_kwonlyargcount;
4639 }
4640 for (i = start; i < end; i++) {
4641 if (GETLOCAL(i) == NULL) {
4642 PyObject *raw = PyTuple_GET_ITEM(co->co_varnames, i);
4643 PyObject *name = PyObject_Repr(raw);
4644 if (name == NULL) {
4645 Py_DECREF(missing_names);
4646 return;
4647 }
4648 PyList_SET_ITEM(missing_names, j++, name);
4649 }
4650 }
4651 assert(j == missing);
4652 format_missing(tstate, kind, co, missing_names, qualname);
4653 Py_DECREF(missing_names);
4654 }
4655
4656 static void
too_many_positional(PyThreadState * tstate,PyCodeObject * co,Py_ssize_t given,PyObject * defaults,PyObject ** fastlocals,PyObject * qualname)4657 too_many_positional(PyThreadState *tstate, PyCodeObject *co,
4658 Py_ssize_t given, PyObject *defaults,
4659 PyObject **fastlocals, PyObject *qualname)
4660 {
4661 int plural;
4662 Py_ssize_t kwonly_given = 0;
4663 Py_ssize_t i;
4664 PyObject *sig, *kwonly_sig;
4665 Py_ssize_t co_argcount = co->co_argcount;
4666
4667 assert((co->co_flags & CO_VARARGS) == 0);
4668 /* Count missing keyword-only args. */
4669 for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) {
4670 if (GETLOCAL(i) != NULL) {
4671 kwonly_given++;
4672 }
4673 }
4674 Py_ssize_t defcount = defaults == NULL ? 0 : PyTuple_GET_SIZE(defaults);
4675 if (defcount) {
4676 Py_ssize_t atleast = co_argcount - defcount;
4677 plural = 1;
4678 sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount);
4679 }
4680 else {
4681 plural = (co_argcount != 1);
4682 sig = PyUnicode_FromFormat("%zd", co_argcount);
4683 }
4684 if (sig == NULL)
4685 return;
4686 if (kwonly_given) {
4687 const char *format = " positional argument%s (and %zd keyword-only argument%s)";
4688 kwonly_sig = PyUnicode_FromFormat(format,
4689 given != 1 ? "s" : "",
4690 kwonly_given,
4691 kwonly_given != 1 ? "s" : "");
4692 if (kwonly_sig == NULL) {
4693 Py_DECREF(sig);
4694 return;
4695 }
4696 }
4697 else {
4698 /* This will not fail. */
4699 kwonly_sig = PyUnicode_FromString("");
4700 assert(kwonly_sig != NULL);
4701 }
4702 _PyErr_Format(tstate, PyExc_TypeError,
4703 "%U() takes %U positional argument%s but %zd%U %s given",
4704 qualname,
4705 sig,
4706 plural ? "s" : "",
4707 given,
4708 kwonly_sig,
4709 given == 1 && !kwonly_given ? "was" : "were");
4710 Py_DECREF(sig);
4711 Py_DECREF(kwonly_sig);
4712 }
4713
4714 static int
positional_only_passed_as_keyword(PyThreadState * tstate,PyCodeObject * co,Py_ssize_t kwcount,PyObject * kwnames,PyObject * qualname)4715 positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co,
4716 Py_ssize_t kwcount, PyObject* kwnames,
4717 PyObject *qualname)
4718 {
4719 int posonly_conflicts = 0;
4720 PyObject* posonly_names = PyList_New(0);
4721
4722 for(int k=0; k < co->co_posonlyargcount; k++){
4723 PyObject* posonly_name = PyTuple_GET_ITEM(co->co_varnames, k);
4724
4725 for (int k2=0; k2<kwcount; k2++){
4726 /* Compare the pointers first and fallback to PyObject_RichCompareBool*/
4727 PyObject* kwname = PyTuple_GET_ITEM(kwnames, k2);
4728 if (kwname == posonly_name){
4729 if(PyList_Append(posonly_names, kwname) != 0) {
4730 goto fail;
4731 }
4732 posonly_conflicts++;
4733 continue;
4734 }
4735
4736 int cmp = PyObject_RichCompareBool(posonly_name, kwname, Py_EQ);
4737
4738 if ( cmp > 0) {
4739 if(PyList_Append(posonly_names, kwname) != 0) {
4740 goto fail;
4741 }
4742 posonly_conflicts++;
4743 } else if (cmp < 0) {
4744 goto fail;
4745 }
4746
4747 }
4748 }
4749 if (posonly_conflicts) {
4750 PyObject* comma = PyUnicode_FromString(", ");
4751 if (comma == NULL) {
4752 goto fail;
4753 }
4754 PyObject* error_names = PyUnicode_Join(comma, posonly_names);
4755 Py_DECREF(comma);
4756 if (error_names == NULL) {
4757 goto fail;
4758 }
4759 _PyErr_Format(tstate, PyExc_TypeError,
4760 "%U() got some positional-only arguments passed"
4761 " as keyword arguments: '%U'",
4762 qualname, error_names);
4763 Py_DECREF(error_names);
4764 goto fail;
4765 }
4766
4767 Py_DECREF(posonly_names);
4768 return 0;
4769
4770 fail:
4771 Py_XDECREF(posonly_names);
4772 return 1;
4773
4774 }
4775
4776
4777 PyFrameObject *
_PyEval_MakeFrameVector(PyThreadState * tstate,PyFrameConstructor * con,PyObject * locals,PyObject * const * args,Py_ssize_t argcount,PyObject * kwnames)4778 _PyEval_MakeFrameVector(PyThreadState *tstate,
4779 PyFrameConstructor *con, PyObject *locals,
4780 PyObject *const *args, Py_ssize_t argcount,
4781 PyObject *kwnames)
4782 {
4783 assert(is_tstate_valid(tstate));
4784
4785 PyCodeObject *co = (PyCodeObject*)con->fc_code;
4786 assert(con->fc_defaults == NULL || PyTuple_CheckExact(con->fc_defaults));
4787 const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount;
4788
4789 /* Create the frame */
4790 PyFrameObject *f = _PyFrame_New_NoTrack(tstate, con, locals);
4791 if (f == NULL) {
4792 return NULL;
4793 }
4794 PyObject **fastlocals = f->f_localsplus;
4795 PyObject **freevars = f->f_localsplus + co->co_nlocals;
4796
4797 /* Create a dictionary for keyword parameters (**kwags) */
4798 PyObject *kwdict;
4799 Py_ssize_t i;
4800 if (co->co_flags & CO_VARKEYWORDS) {
4801 kwdict = PyDict_New();
4802 if (kwdict == NULL)
4803 goto fail;
4804 i = total_args;
4805 if (co->co_flags & CO_VARARGS) {
4806 i++;
4807 }
4808 SETLOCAL(i, kwdict);
4809 }
4810 else {
4811 kwdict = NULL;
4812 }
4813
4814 /* Copy all positional arguments into local variables */
4815 Py_ssize_t j, n;
4816 if (argcount > co->co_argcount) {
4817 n = co->co_argcount;
4818 }
4819 else {
4820 n = argcount;
4821 }
4822 for (j = 0; j < n; j++) {
4823 PyObject *x = args[j];
4824 Py_INCREF(x);
4825 SETLOCAL(j, x);
4826 }
4827
4828 /* Pack other positional arguments into the *args argument */
4829 if (co->co_flags & CO_VARARGS) {
4830 PyObject *u = _PyTuple_FromArray(args + n, argcount - n);
4831 if (u == NULL) {
4832 goto fail;
4833 }
4834 SETLOCAL(total_args, u);
4835 }
4836
4837 /* Handle keyword arguments */
4838 if (kwnames != NULL) {
4839 Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
4840 for (i = 0; i < kwcount; i++) {
4841 PyObject **co_varnames;
4842 PyObject *keyword = PyTuple_GET_ITEM(kwnames, i);
4843 PyObject *value = args[i+argcount];
4844 Py_ssize_t j;
4845
4846 if (keyword == NULL || !PyUnicode_Check(keyword)) {
4847 _PyErr_Format(tstate, PyExc_TypeError,
4848 "%U() keywords must be strings",
4849 con->fc_qualname);
4850 goto fail;
4851 }
4852
4853 /* Speed hack: do raw pointer compares. As names are
4854 normally interned this should almost always hit. */
4855 co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item;
4856 for (j = co->co_posonlyargcount; j < total_args; j++) {
4857 PyObject *varname = co_varnames[j];
4858 if (varname == keyword) {
4859 goto kw_found;
4860 }
4861 }
4862
4863 /* Slow fallback, just in case */
4864 for (j = co->co_posonlyargcount; j < total_args; j++) {
4865 PyObject *varname = co_varnames[j];
4866 int cmp = PyObject_RichCompareBool( keyword, varname, Py_EQ);
4867 if (cmp > 0) {
4868 goto kw_found;
4869 }
4870 else if (cmp < 0) {
4871 goto fail;
4872 }
4873 }
4874
4875 assert(j >= total_args);
4876 if (kwdict == NULL) {
4877
4878 if (co->co_posonlyargcount
4879 && positional_only_passed_as_keyword(tstate, co,
4880 kwcount, kwnames,
4881 con->fc_qualname))
4882 {
4883 goto fail;
4884 }
4885
4886 _PyErr_Format(tstate, PyExc_TypeError,
4887 "%U() got an unexpected keyword argument '%S'",
4888 con->fc_qualname, keyword);
4889 goto fail;
4890 }
4891
4892 if (PyDict_SetItem(kwdict, keyword, value) == -1) {
4893 goto fail;
4894 }
4895 continue;
4896
4897 kw_found:
4898 if (GETLOCAL(j) != NULL) {
4899 _PyErr_Format(tstate, PyExc_TypeError,
4900 "%U() got multiple values for argument '%S'",
4901 con->fc_qualname, keyword);
4902 goto fail;
4903 }
4904 Py_INCREF(value);
4905 SETLOCAL(j, value);
4906 }
4907 }
4908
4909 /* Check the number of positional arguments */
4910 if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) {
4911 too_many_positional(tstate, co, argcount, con->fc_defaults, fastlocals,
4912 con->fc_qualname);
4913 goto fail;
4914 }
4915
4916 /* Add missing positional arguments (copy default values from defs) */
4917 if (argcount < co->co_argcount) {
4918 Py_ssize_t defcount = con->fc_defaults == NULL ? 0 : PyTuple_GET_SIZE(con->fc_defaults);
4919 Py_ssize_t m = co->co_argcount - defcount;
4920 Py_ssize_t missing = 0;
4921 for (i = argcount; i < m; i++) {
4922 if (GETLOCAL(i) == NULL) {
4923 missing++;
4924 }
4925 }
4926 if (missing) {
4927 missing_arguments(tstate, co, missing, defcount, fastlocals,
4928 con->fc_qualname);
4929 goto fail;
4930 }
4931 if (n > m)
4932 i = n - m;
4933 else
4934 i = 0;
4935 if (defcount) {
4936 PyObject **defs = &PyTuple_GET_ITEM(con->fc_defaults, 0);
4937 for (; i < defcount; i++) {
4938 if (GETLOCAL(m+i) == NULL) {
4939 PyObject *def = defs[i];
4940 Py_INCREF(def);
4941 SETLOCAL(m+i, def);
4942 }
4943 }
4944 }
4945 }
4946
4947 /* Add missing keyword arguments (copy default values from kwdefs) */
4948 if (co->co_kwonlyargcount > 0) {
4949 Py_ssize_t missing = 0;
4950 for (i = co->co_argcount; i < total_args; i++) {
4951 if (GETLOCAL(i) != NULL)
4952 continue;
4953 PyObject *varname = PyTuple_GET_ITEM(co->co_varnames, i);
4954 if (con->fc_kwdefaults != NULL) {
4955 PyObject *def = PyDict_GetItemWithError(con->fc_kwdefaults, varname);
4956 if (def) {
4957 Py_INCREF(def);
4958 SETLOCAL(i, def);
4959 continue;
4960 }
4961 else if (_PyErr_Occurred(tstate)) {
4962 goto fail;
4963 }
4964 }
4965 missing++;
4966 }
4967 if (missing) {
4968 missing_arguments(tstate, co, missing, -1, fastlocals,
4969 con->fc_qualname);
4970 goto fail;
4971 }
4972 }
4973
4974 /* Allocate and initialize storage for cell vars, and copy free
4975 vars into frame. */
4976 for (i = 0; i < PyTuple_GET_SIZE(co->co_cellvars); ++i) {
4977 PyObject *c;
4978 Py_ssize_t arg;
4979 /* Possibly account for the cell variable being an argument. */
4980 if (co->co_cell2arg != NULL &&
4981 (arg = co->co_cell2arg[i]) != CO_CELL_NOT_AN_ARG) {
4982 c = PyCell_New(GETLOCAL(arg));
4983 /* Clear the local copy. */
4984 SETLOCAL(arg, NULL);
4985 }
4986 else {
4987 c = PyCell_New(NULL);
4988 }
4989 if (c == NULL)
4990 goto fail;
4991 SETLOCAL(co->co_nlocals + i, c);
4992 }
4993
4994 /* Copy closure variables to free variables */
4995 for (i = 0; i < PyTuple_GET_SIZE(co->co_freevars); ++i) {
4996 PyObject *o = PyTuple_GET_ITEM(con->fc_closure, i);
4997 Py_INCREF(o);
4998 freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o;
4999 }
5000
5001 return f;
5002
5003 fail: /* Jump here from prelude on failure */
5004
5005 /* decref'ing the frame can cause __del__ methods to get invoked,
5006 which can call back into Python. While we're done with the
5007 current Python frame (f), the associated C stack is still in use,
5008 so recursion_depth must be boosted for the duration.
5009 */
5010 if (Py_REFCNT(f) > 1) {
5011 Py_DECREF(f);
5012 _PyObject_GC_TRACK(f);
5013 }
5014 else {
5015 ++tstate->recursion_depth;
5016 Py_DECREF(f);
5017 --tstate->recursion_depth;
5018 }
5019 return NULL;
5020 }
5021
5022 static PyObject *
make_coro(PyFrameConstructor * con,PyFrameObject * f)5023 make_coro(PyFrameConstructor *con, PyFrameObject *f)
5024 {
5025 assert (((PyCodeObject *)con->fc_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR));
5026 PyObject *gen;
5027 int is_coro = ((PyCodeObject *)con->fc_code)->co_flags & CO_COROUTINE;
5028
5029 /* Don't need to keep the reference to f_back, it will be set
5030 * when the generator is resumed. */
5031 Py_CLEAR(f->f_back);
5032
5033 /* Create a new generator that owns the ready to run frame
5034 * and return that as the value. */
5035 if (is_coro) {
5036 gen = PyCoro_New(f, con->fc_name, con->fc_qualname);
5037 } else if (((PyCodeObject *)con->fc_code)->co_flags & CO_ASYNC_GENERATOR) {
5038 gen = PyAsyncGen_New(f, con->fc_name, con->fc_qualname);
5039 } else {
5040 gen = PyGen_NewWithQualName(f, con->fc_name, con->fc_qualname);
5041 }
5042 if (gen == NULL) {
5043 return NULL;
5044 }
5045
5046 _PyObject_GC_TRACK(f);
5047
5048 return gen;
5049 }
5050
5051 PyObject *
_PyEval_Vector(PyThreadState * tstate,PyFrameConstructor * con,PyObject * locals,PyObject * const * args,size_t argcount,PyObject * kwnames)5052 _PyEval_Vector(PyThreadState *tstate, PyFrameConstructor *con,
5053 PyObject *locals,
5054 PyObject* const* args, size_t argcount,
5055 PyObject *kwnames)
5056 {
5057 PyFrameObject *f = _PyEval_MakeFrameVector(
5058 tstate, con, locals, args, argcount, kwnames);
5059 if (f == NULL) {
5060 return NULL;
5061 }
5062 if (((PyCodeObject *)con->fc_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
5063 return make_coro(con, f);
5064 }
5065 PyObject *retval = _PyEval_EvalFrame(tstate, f, 0);
5066
5067 /* decref'ing the frame can cause __del__ methods to get invoked,
5068 which can call back into Python. While we're done with the
5069 current Python frame (f), the associated C stack is still in use,
5070 so recursion_depth must be boosted for the duration.
5071 */
5072 if (Py_REFCNT(f) > 1) {
5073 Py_DECREF(f);
5074 _PyObject_GC_TRACK(f);
5075 }
5076 else {
5077 ++tstate->recursion_depth;
5078 Py_DECREF(f);
5079 --tstate->recursion_depth;
5080 }
5081 return retval;
5082 }
5083
5084 /* Legacy API */
5085 PyObject *
PyEval_EvalCodeEx(PyObject * _co,PyObject * globals,PyObject * locals,PyObject * const * args,int argcount,PyObject * const * kws,int kwcount,PyObject * const * defs,int defcount,PyObject * kwdefs,PyObject * closure)5086 PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals,
5087 PyObject *const *args, int argcount,
5088 PyObject *const *kws, int kwcount,
5089 PyObject *const *defs, int defcount,
5090 PyObject *kwdefs, PyObject *closure)
5091 {
5092 PyThreadState *tstate = _PyThreadState_GET();
5093 PyObject *res = NULL;
5094 PyObject *defaults = _PyTuple_FromArray(defs, defcount);
5095 if (defaults == NULL) {
5096 return NULL;
5097 }
5098 PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref
5099 if (builtins == NULL) {
5100 Py_DECREF(defaults);
5101 return NULL;
5102 }
5103 if (locals == NULL) {
5104 locals = globals;
5105 }
5106 PyObject *kwnames = NULL;
5107 PyObject *const *allargs;
5108 PyObject **newargs = NULL;
5109 if (kwcount == 0) {
5110 allargs = args;
5111 }
5112 else {
5113 kwnames = PyTuple_New(kwcount);
5114 if (kwnames == NULL) {
5115 goto fail;
5116 }
5117 newargs = PyMem_Malloc(sizeof(PyObject *)*(kwcount+argcount));
5118 if (newargs == NULL) {
5119 goto fail;
5120 }
5121 for (int i = 0; i < argcount; i++) {
5122 newargs[i] = args[i];
5123 }
5124 for (int i = 0; i < kwcount; i++) {
5125 Py_INCREF(kws[2*i]);
5126 PyTuple_SET_ITEM(kwnames, i, kws[2*i]);
5127 newargs[argcount+i] = kws[2*i+1];
5128 }
5129 allargs = newargs;
5130 }
5131 for (int i = 0; i < kwcount; i++) {
5132 Py_INCREF(kws[2*i]);
5133 PyTuple_SET_ITEM(kwnames, i, kws[2*i]);
5134 }
5135 PyFrameConstructor constr = {
5136 .fc_globals = globals,
5137 .fc_builtins = builtins,
5138 .fc_name = ((PyCodeObject *)_co)->co_name,
5139 .fc_qualname = ((PyCodeObject *)_co)->co_name,
5140 .fc_code = _co,
5141 .fc_defaults = defaults,
5142 .fc_kwdefaults = kwdefs,
5143 .fc_closure = closure
5144 };
5145 res = _PyEval_Vector(tstate, &constr, locals,
5146 allargs, argcount,
5147 kwnames);
5148 fail:
5149 Py_XDECREF(kwnames);
5150 PyMem_Free(newargs);
5151 Py_DECREF(defaults);
5152 return res;
5153 }
5154
5155
5156 static PyObject *
special_lookup(PyThreadState * tstate,PyObject * o,_Py_Identifier * id)5157 special_lookup(PyThreadState *tstate, PyObject *o, _Py_Identifier *id)
5158 {
5159 PyObject *res;
5160 res = _PyObject_LookupSpecial(o, id);
5161 if (res == NULL && !_PyErr_Occurred(tstate)) {
5162 _PyErr_SetObject(tstate, PyExc_AttributeError, _PyUnicode_FromId(id));
5163 return NULL;
5164 }
5165 return res;
5166 }
5167
5168
5169 /* Logic for the raise statement (too complicated for inlining).
5170 This *consumes* a reference count to each of its arguments. */
5171 static int
do_raise(PyThreadState * tstate,PyObject * exc,PyObject * cause)5172 do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause)
5173 {
5174 PyObject *type = NULL, *value = NULL;
5175
5176 if (exc == NULL) {
5177 /* Reraise */
5178 _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
5179 PyObject *tb;
5180 type = exc_info->exc_type;
5181 value = exc_info->exc_value;
5182 tb = exc_info->exc_traceback;
5183 if (Py_IsNone(type) || type == NULL) {
5184 _PyErr_SetString(tstate, PyExc_RuntimeError,
5185 "No active exception to reraise");
5186 return 0;
5187 }
5188 Py_XINCREF(type);
5189 Py_XINCREF(value);
5190 Py_XINCREF(tb);
5191 _PyErr_Restore(tstate, type, value, tb);
5192 return 1;
5193 }
5194
5195 /* We support the following forms of raise:
5196 raise
5197 raise <instance>
5198 raise <type> */
5199
5200 if (PyExceptionClass_Check(exc)) {
5201 type = exc;
5202 value = _PyObject_CallNoArg(exc);
5203 if (value == NULL)
5204 goto raise_error;
5205 if (!PyExceptionInstance_Check(value)) {
5206 _PyErr_Format(tstate, PyExc_TypeError,
5207 "calling %R should have returned an instance of "
5208 "BaseException, not %R",
5209 type, Py_TYPE(value));
5210 goto raise_error;
5211 }
5212 }
5213 else if (PyExceptionInstance_Check(exc)) {
5214 value = exc;
5215 type = PyExceptionInstance_Class(exc);
5216 Py_INCREF(type);
5217 }
5218 else {
5219 /* Not something you can raise. You get an exception
5220 anyway, just not what you specified :-) */
5221 Py_DECREF(exc);
5222 _PyErr_SetString(tstate, PyExc_TypeError,
5223 "exceptions must derive from BaseException");
5224 goto raise_error;
5225 }
5226
5227 assert(type != NULL);
5228 assert(value != NULL);
5229
5230 if (cause) {
5231 PyObject *fixed_cause;
5232 if (PyExceptionClass_Check(cause)) {
5233 fixed_cause = _PyObject_CallNoArg(cause);
5234 if (fixed_cause == NULL)
5235 goto raise_error;
5236 Py_DECREF(cause);
5237 }
5238 else if (PyExceptionInstance_Check(cause)) {
5239 fixed_cause = cause;
5240 }
5241 else if (Py_IsNone(cause)) {
5242 Py_DECREF(cause);
5243 fixed_cause = NULL;
5244 }
5245 else {
5246 _PyErr_SetString(tstate, PyExc_TypeError,
5247 "exception causes must derive from "
5248 "BaseException");
5249 goto raise_error;
5250 }
5251 PyException_SetCause(value, fixed_cause);
5252 }
5253
5254 _PyErr_SetObject(tstate, type, value);
5255 /* _PyErr_SetObject incref's its arguments */
5256 Py_DECREF(value);
5257 Py_DECREF(type);
5258 return 0;
5259
5260 raise_error:
5261 Py_XDECREF(value);
5262 Py_XDECREF(type);
5263 Py_XDECREF(cause);
5264 return 0;
5265 }
5266
5267 /* Iterate v argcnt times and store the results on the stack (via decreasing
5268 sp). Return 1 for success, 0 if error.
5269
5270 If argcntafter == -1, do a simple unpack. If it is >= 0, do an unpack
5271 with a variable target.
5272 */
5273
5274 static int
unpack_iterable(PyThreadState * tstate,PyObject * v,int argcnt,int argcntafter,PyObject ** sp)5275 unpack_iterable(PyThreadState *tstate, PyObject *v,
5276 int argcnt, int argcntafter, PyObject **sp)
5277 {
5278 int i = 0, j = 0;
5279 Py_ssize_t ll = 0;
5280 PyObject *it; /* iter(v) */
5281 PyObject *w;
5282 PyObject *l = NULL; /* variable list */
5283
5284 assert(v != NULL);
5285
5286 it = PyObject_GetIter(v);
5287 if (it == NULL) {
5288 if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
5289 Py_TYPE(v)->tp_iter == NULL && !PySequence_Check(v))
5290 {
5291 _PyErr_Format(tstate, PyExc_TypeError,
5292 "cannot unpack non-iterable %.200s object",
5293 Py_TYPE(v)->tp_name);
5294 }
5295 return 0;
5296 }
5297
5298 for (; i < argcnt; i++) {
5299 w = PyIter_Next(it);
5300 if (w == NULL) {
5301 /* Iterator done, via error or exhaustion. */
5302 if (!_PyErr_Occurred(tstate)) {
5303 if (argcntafter == -1) {
5304 _PyErr_Format(tstate, PyExc_ValueError,
5305 "not enough values to unpack "
5306 "(expected %d, got %d)",
5307 argcnt, i);
5308 }
5309 else {
5310 _PyErr_Format(tstate, PyExc_ValueError,
5311 "not enough values to unpack "
5312 "(expected at least %d, got %d)",
5313 argcnt + argcntafter, i);
5314 }
5315 }
5316 goto Error;
5317 }
5318 *--sp = w;
5319 }
5320
5321 if (argcntafter == -1) {
5322 /* We better have exhausted the iterator now. */
5323 w = PyIter_Next(it);
5324 if (w == NULL) {
5325 if (_PyErr_Occurred(tstate))
5326 goto Error;
5327 Py_DECREF(it);
5328 return 1;
5329 }
5330 Py_DECREF(w);
5331 _PyErr_Format(tstate, PyExc_ValueError,
5332 "too many values to unpack (expected %d)",
5333 argcnt);
5334 goto Error;
5335 }
5336
5337 l = PySequence_List(it);
5338 if (l == NULL)
5339 goto Error;
5340 *--sp = l;
5341 i++;
5342
5343 ll = PyList_GET_SIZE(l);
5344 if (ll < argcntafter) {
5345 _PyErr_Format(tstate, PyExc_ValueError,
5346 "not enough values to unpack (expected at least %d, got %zd)",
5347 argcnt + argcntafter, argcnt + ll);
5348 goto Error;
5349 }
5350
5351 /* Pop the "after-variable" args off the list. */
5352 for (j = argcntafter; j > 0; j--, i++) {
5353 *--sp = PyList_GET_ITEM(l, ll - j);
5354 }
5355 /* Resize the list. */
5356 Py_SET_SIZE(l, ll - argcntafter);
5357 Py_DECREF(it);
5358 return 1;
5359
5360 Error:
5361 for (; i > 0; i--, sp++)
5362 Py_DECREF(*sp);
5363 Py_XDECREF(it);
5364 return 0;
5365 }
5366
5367 #ifdef LLTRACE
5368 static int
prtrace(PyThreadState * tstate,PyObject * v,const char * str)5369 prtrace(PyThreadState *tstate, PyObject *v, const char *str)
5370 {
5371 printf("%s ", str);
5372 PyObject *type, *value, *traceback;
5373 PyErr_Fetch(&type, &value, &traceback);
5374 if (PyObject_Print(v, stdout, 0) != 0) {
5375 /* Don't know what else to do */
5376 _PyErr_Clear(tstate);
5377 }
5378 printf("\n");
5379 PyErr_Restore(type, value, traceback);
5380 return 1;
5381 }
5382 #endif
5383
5384 static void
call_exc_trace(Py_tracefunc func,PyObject * self,PyThreadState * tstate,PyFrameObject * f,PyTraceInfo * trace_info)5385 call_exc_trace(Py_tracefunc func, PyObject *self,
5386 PyThreadState *tstate,
5387 PyFrameObject *f,
5388 PyTraceInfo *trace_info)
5389 {
5390 PyObject *type, *value, *traceback, *orig_traceback, *arg;
5391 int err;
5392 _PyErr_Fetch(tstate, &type, &value, &orig_traceback);
5393 if (value == NULL) {
5394 value = Py_None;
5395 Py_INCREF(value);
5396 }
5397 _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback);
5398 traceback = (orig_traceback != NULL) ? orig_traceback : Py_None;
5399 arg = PyTuple_Pack(3, type, value, traceback);
5400 if (arg == NULL) {
5401 _PyErr_Restore(tstate, type, value, orig_traceback);
5402 return;
5403 }
5404 err = call_trace(func, self, tstate, f, trace_info, PyTrace_EXCEPTION, arg);
5405 Py_DECREF(arg);
5406 if (err == 0) {
5407 _PyErr_Restore(tstate, type, value, orig_traceback);
5408 }
5409 else {
5410 Py_XDECREF(type);
5411 Py_XDECREF(value);
5412 Py_XDECREF(orig_traceback);
5413 }
5414 }
5415
5416 static int
call_trace_protected(Py_tracefunc func,PyObject * obj,PyThreadState * tstate,PyFrameObject * frame,PyTraceInfo * trace_info,int what,PyObject * arg)5417 call_trace_protected(Py_tracefunc func, PyObject *obj,
5418 PyThreadState *tstate, PyFrameObject *frame,
5419 PyTraceInfo *trace_info,
5420 int what, PyObject *arg)
5421 {
5422 PyObject *type, *value, *traceback;
5423 int err;
5424 _PyErr_Fetch(tstate, &type, &value, &traceback);
5425 err = call_trace(func, obj, tstate, frame, trace_info, what, arg);
5426 if (err == 0)
5427 {
5428 _PyErr_Restore(tstate, type, value, traceback);
5429 return 0;
5430 }
5431 else {
5432 Py_XDECREF(type);
5433 Py_XDECREF(value);
5434 Py_XDECREF(traceback);
5435 return -1;
5436 }
5437 }
5438
5439 static void
initialize_trace_info(PyTraceInfo * trace_info,PyFrameObject * frame)5440 initialize_trace_info(PyTraceInfo *trace_info, PyFrameObject *frame)
5441 {
5442 if (trace_info->code != frame->f_code) {
5443 trace_info->code = frame->f_code;
5444 _PyCode_InitAddressRange(frame->f_code, &trace_info->bounds);
5445 }
5446 }
5447
5448 static int
call_trace(Py_tracefunc func,PyObject * obj,PyThreadState * tstate,PyFrameObject * frame,PyTraceInfo * trace_info,int what,PyObject * arg)5449 call_trace(Py_tracefunc func, PyObject *obj,
5450 PyThreadState *tstate, PyFrameObject *frame,
5451 PyTraceInfo *trace_info,
5452 int what, PyObject *arg)
5453 {
5454 int result;
5455 if (tstate->tracing)
5456 return 0;
5457 tstate->tracing++;
5458 tstate->cframe->use_tracing = 0;
5459 if (frame->f_lasti < 0) {
5460 frame->f_lineno = frame->f_code->co_firstlineno;
5461 }
5462 else {
5463 initialize_trace_info(trace_info, frame);
5464 frame->f_lineno = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &trace_info->bounds);
5465 }
5466 result = func(obj, frame, what, arg);
5467 frame->f_lineno = 0;
5468 tstate->cframe->use_tracing = ((tstate->c_tracefunc != NULL)
5469 || (tstate->c_profilefunc != NULL));
5470 tstate->tracing--;
5471 return result;
5472 }
5473
5474 PyObject *
_PyEval_CallTracing(PyObject * func,PyObject * args)5475 _PyEval_CallTracing(PyObject *func, PyObject *args)
5476 {
5477 PyThreadState *tstate = _PyThreadState_GET();
5478 int save_tracing = tstate->tracing;
5479 int save_use_tracing = tstate->cframe->use_tracing;
5480 PyObject *result;
5481
5482 tstate->tracing = 0;
5483 tstate->cframe->use_tracing = ((tstate->c_tracefunc != NULL)
5484 || (tstate->c_profilefunc != NULL));
5485 result = PyObject_Call(func, args, NULL);
5486 tstate->tracing = save_tracing;
5487 tstate->cframe->use_tracing = save_use_tracing;
5488 return result;
5489 }
5490
5491 /* See Objects/lnotab_notes.txt for a description of how tracing works. */
5492 static int
maybe_call_line_trace(Py_tracefunc func,PyObject * obj,PyThreadState * tstate,PyFrameObject * frame,PyTraceInfo * trace_info,int instr_prev)5493 maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
5494 PyThreadState *tstate, PyFrameObject *frame,
5495 PyTraceInfo *trace_info, int instr_prev)
5496 {
5497 int result = 0;
5498
5499 /* If the last instruction falls at the start of a line or if it
5500 represents a jump backwards, update the frame's line number and
5501 then call the trace function if we're tracing source lines.
5502 */
5503 initialize_trace_info(trace_info, frame);
5504 int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &trace_info->bounds);
5505 int line = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &trace_info->bounds);
5506 if (line != -1 && frame->f_trace_lines) {
5507 /* Trace backward edges or if line number has changed */
5508 if (frame->f_lasti < instr_prev || line != lastline) {
5509 result = call_trace(func, obj, tstate, frame, trace_info, PyTrace_LINE, Py_None);
5510 }
5511 }
5512 /* Always emit an opcode event if we're tracing all opcodes. */
5513 if (frame->f_trace_opcodes) {
5514 result = call_trace(func, obj, tstate, frame, trace_info, PyTrace_OPCODE, Py_None);
5515 }
5516 return result;
5517 }
5518
5519 int
_PyEval_SetProfile(PyThreadState * tstate,Py_tracefunc func,PyObject * arg)5520 _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
5521 {
5522 assert(is_tstate_valid(tstate));
5523 /* The caller must hold the GIL */
5524 assert(PyGILState_Check());
5525
5526 /* Call _PySys_Audit() in the context of the current thread state,
5527 even if tstate is not the current thread state. */
5528 PyThreadState *current_tstate = _PyThreadState_GET();
5529 if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) {
5530 return -1;
5531 }
5532
5533 PyObject *profileobj = tstate->c_profileobj;
5534
5535 tstate->c_profilefunc = NULL;
5536 tstate->c_profileobj = NULL;
5537 /* Must make sure that tracing is not ignored if 'profileobj' is freed */
5538 tstate->cframe->use_tracing = tstate->c_tracefunc != NULL;
5539 Py_XDECREF(profileobj);
5540
5541 Py_XINCREF(arg);
5542 tstate->c_profileobj = arg;
5543 tstate->c_profilefunc = func;
5544
5545 /* Flag that tracing or profiling is turned on */
5546 tstate->cframe->use_tracing = (func != NULL) || (tstate->c_tracefunc != NULL);
5547 return 0;
5548 }
5549
5550 void
PyEval_SetProfile(Py_tracefunc func,PyObject * arg)5551 PyEval_SetProfile(Py_tracefunc func, PyObject *arg)
5552 {
5553 PyThreadState *tstate = _PyThreadState_GET();
5554 if (_PyEval_SetProfile(tstate, func, arg) < 0) {
5555 /* Log _PySys_Audit() error */
5556 _PyErr_WriteUnraisableMsg("in PyEval_SetProfile", NULL);
5557 }
5558 }
5559
5560 int
_PyEval_SetTrace(PyThreadState * tstate,Py_tracefunc func,PyObject * arg)5561 _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
5562 {
5563 assert(is_tstate_valid(tstate));
5564 /* The caller must hold the GIL */
5565 assert(PyGILState_Check());
5566
5567 /* Call _PySys_Audit() in the context of the current thread state,
5568 even if tstate is not the current thread state. */
5569 PyThreadState *current_tstate = _PyThreadState_GET();
5570 if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) {
5571 return -1;
5572 }
5573
5574 PyObject *traceobj = tstate->c_traceobj;
5575
5576 tstate->c_tracefunc = NULL;
5577 tstate->c_traceobj = NULL;
5578 /* Must make sure that profiling is not ignored if 'traceobj' is freed */
5579 tstate->cframe->use_tracing = (tstate->c_profilefunc != NULL);
5580 Py_XDECREF(traceobj);
5581
5582 Py_XINCREF(arg);
5583 tstate->c_traceobj = arg;
5584 tstate->c_tracefunc = func;
5585
5586 /* Flag that tracing or profiling is turned on */
5587 tstate->cframe->use_tracing = ((func != NULL)
5588 || (tstate->c_profilefunc != NULL));
5589
5590 return 0;
5591 }
5592
5593 void
PyEval_SetTrace(Py_tracefunc func,PyObject * arg)5594 PyEval_SetTrace(Py_tracefunc func, PyObject *arg)
5595 {
5596 PyThreadState *tstate = _PyThreadState_GET();
5597 if (_PyEval_SetTrace(tstate, func, arg) < 0) {
5598 /* Log _PySys_Audit() error */
5599 _PyErr_WriteUnraisableMsg("in PyEval_SetTrace", NULL);
5600 }
5601 }
5602
5603
5604 void
_PyEval_SetCoroutineOriginTrackingDepth(PyThreadState * tstate,int new_depth)5605 _PyEval_SetCoroutineOriginTrackingDepth(PyThreadState *tstate, int new_depth)
5606 {
5607 assert(new_depth >= 0);
5608 tstate->coroutine_origin_tracking_depth = new_depth;
5609 }
5610
5611 int
_PyEval_GetCoroutineOriginTrackingDepth(void)5612 _PyEval_GetCoroutineOriginTrackingDepth(void)
5613 {
5614 PyThreadState *tstate = _PyThreadState_GET();
5615 return tstate->coroutine_origin_tracking_depth;
5616 }
5617
5618 int
_PyEval_SetAsyncGenFirstiter(PyObject * firstiter)5619 _PyEval_SetAsyncGenFirstiter(PyObject *firstiter)
5620 {
5621 PyThreadState *tstate = _PyThreadState_GET();
5622
5623 if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_firstiter", NULL) < 0) {
5624 return -1;
5625 }
5626
5627 Py_XINCREF(firstiter);
5628 Py_XSETREF(tstate->async_gen_firstiter, firstiter);
5629 return 0;
5630 }
5631
5632 PyObject *
_PyEval_GetAsyncGenFirstiter(void)5633 _PyEval_GetAsyncGenFirstiter(void)
5634 {
5635 PyThreadState *tstate = _PyThreadState_GET();
5636 return tstate->async_gen_firstiter;
5637 }
5638
5639 int
_PyEval_SetAsyncGenFinalizer(PyObject * finalizer)5640 _PyEval_SetAsyncGenFinalizer(PyObject *finalizer)
5641 {
5642 PyThreadState *tstate = _PyThreadState_GET();
5643
5644 if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_finalizer", NULL) < 0) {
5645 return -1;
5646 }
5647
5648 Py_XINCREF(finalizer);
5649 Py_XSETREF(tstate->async_gen_finalizer, finalizer);
5650 return 0;
5651 }
5652
5653 PyObject *
_PyEval_GetAsyncGenFinalizer(void)5654 _PyEval_GetAsyncGenFinalizer(void)
5655 {
5656 PyThreadState *tstate = _PyThreadState_GET();
5657 return tstate->async_gen_finalizer;
5658 }
5659
5660 PyFrameObject *
PyEval_GetFrame(void)5661 PyEval_GetFrame(void)
5662 {
5663 PyThreadState *tstate = _PyThreadState_GET();
5664 return tstate->frame;
5665 }
5666
5667 PyObject *
_PyEval_GetBuiltins(PyThreadState * tstate)5668 _PyEval_GetBuiltins(PyThreadState *tstate)
5669 {
5670 PyFrameObject *frame = tstate->frame;
5671 if (frame != NULL) {
5672 return frame->f_builtins;
5673 }
5674 return tstate->interp->builtins;
5675 }
5676
5677 PyObject *
PyEval_GetBuiltins(void)5678 PyEval_GetBuiltins(void)
5679 {
5680 PyThreadState *tstate = _PyThreadState_GET();
5681 return _PyEval_GetBuiltins(tstate);
5682 }
5683
5684 /* Convenience function to get a builtin from its name */
5685 PyObject *
_PyEval_GetBuiltinId(_Py_Identifier * name)5686 _PyEval_GetBuiltinId(_Py_Identifier *name)
5687 {
5688 PyThreadState *tstate = _PyThreadState_GET();
5689 PyObject *attr = _PyDict_GetItemIdWithError(PyEval_GetBuiltins(), name);
5690 if (attr) {
5691 Py_INCREF(attr);
5692 }
5693 else if (!_PyErr_Occurred(tstate)) {
5694 _PyErr_SetObject(tstate, PyExc_AttributeError, _PyUnicode_FromId(name));
5695 }
5696 return attr;
5697 }
5698
5699 PyObject *
PyEval_GetLocals(void)5700 PyEval_GetLocals(void)
5701 {
5702 PyThreadState *tstate = _PyThreadState_GET();
5703 PyFrameObject *current_frame = tstate->frame;
5704 if (current_frame == NULL) {
5705 _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist");
5706 return NULL;
5707 }
5708
5709 if (PyFrame_FastToLocalsWithError(current_frame) < 0) {
5710 return NULL;
5711 }
5712
5713 assert(current_frame->f_locals != NULL);
5714 return current_frame->f_locals;
5715 }
5716
5717 PyObject *
PyEval_GetGlobals(void)5718 PyEval_GetGlobals(void)
5719 {
5720 PyThreadState *tstate = _PyThreadState_GET();
5721 PyFrameObject *current_frame = tstate->frame;
5722 if (current_frame == NULL) {
5723 return NULL;
5724 }
5725
5726 assert(current_frame->f_globals != NULL);
5727 return current_frame->f_globals;
5728 }
5729
5730 int
PyEval_MergeCompilerFlags(PyCompilerFlags * cf)5731 PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
5732 {
5733 PyThreadState *tstate = _PyThreadState_GET();
5734 PyFrameObject *current_frame = tstate->frame;
5735 int result = cf->cf_flags != 0;
5736
5737 if (current_frame != NULL) {
5738 const int codeflags = current_frame->f_code->co_flags;
5739 const int compilerflags = codeflags & PyCF_MASK;
5740 if (compilerflags) {
5741 result = 1;
5742 cf->cf_flags |= compilerflags;
5743 }
5744 #if 0 /* future keyword */
5745 if (codeflags & CO_GENERATOR_ALLOWED) {
5746 result = 1;
5747 cf->cf_flags |= CO_GENERATOR_ALLOWED;
5748 }
5749 #endif
5750 }
5751 return result;
5752 }
5753
5754
5755 const char *
PyEval_GetFuncName(PyObject * func)5756 PyEval_GetFuncName(PyObject *func)
5757 {
5758 if (PyMethod_Check(func))
5759 return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func));
5760 else if (PyFunction_Check(func))
5761 return PyUnicode_AsUTF8(((PyFunctionObject*)func)->func_name);
5762 else if (PyCFunction_Check(func))
5763 return ((PyCFunctionObject*)func)->m_ml->ml_name;
5764 else
5765 return Py_TYPE(func)->tp_name;
5766 }
5767
5768 const char *
PyEval_GetFuncDesc(PyObject * func)5769 PyEval_GetFuncDesc(PyObject *func)
5770 {
5771 if (PyMethod_Check(func))
5772 return "()";
5773 else if (PyFunction_Check(func))
5774 return "()";
5775 else if (PyCFunction_Check(func))
5776 return "()";
5777 else
5778 return " object";
5779 }
5780
5781 #define C_TRACE(x, call) \
5782 if (trace_info->cframe.use_tracing && tstate->c_profilefunc) { \
5783 if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, \
5784 tstate, tstate->frame, trace_info, \
5785 PyTrace_C_CALL, func)) { \
5786 x = NULL; \
5787 } \
5788 else { \
5789 x = call; \
5790 if (tstate->c_profilefunc != NULL) { \
5791 if (x == NULL) { \
5792 call_trace_protected(tstate->c_profilefunc, \
5793 tstate->c_profileobj, \
5794 tstate, tstate->frame, trace_info, \
5795 PyTrace_C_EXCEPTION, func); \
5796 /* XXX should pass (type, value, tb) */ \
5797 } else { \
5798 if (call_trace(tstate->c_profilefunc, \
5799 tstate->c_profileobj, \
5800 tstate, tstate->frame, trace_info, \
5801 PyTrace_C_RETURN, func)) { \
5802 Py_DECREF(x); \
5803 x = NULL; \
5804 } \
5805 } \
5806 } \
5807 } \
5808 } else { \
5809 x = call; \
5810 }
5811
5812
5813 static PyObject *
trace_call_function(PyThreadState * tstate,PyTraceInfo * trace_info,PyObject * func,PyObject ** args,Py_ssize_t nargs,PyObject * kwnames)5814 trace_call_function(PyThreadState *tstate,
5815 PyTraceInfo *trace_info,
5816 PyObject *func,
5817 PyObject **args, Py_ssize_t nargs,
5818 PyObject *kwnames)
5819 {
5820 PyObject *x;
5821 if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) {
5822 C_TRACE(x, PyObject_Vectorcall(func, args, nargs, kwnames));
5823 return x;
5824 }
5825 else if (Py_IS_TYPE(func, &PyMethodDescr_Type) && nargs > 0) {
5826 /* We need to create a temporary bound method as argument
5827 for profiling.
5828
5829 If nargs == 0, then this cannot work because we have no
5830 "self". In any case, the call itself would raise
5831 TypeError (foo needs an argument), so we just skip
5832 profiling. */
5833 PyObject *self = args[0];
5834 func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self));
5835 if (func == NULL) {
5836 return NULL;
5837 }
5838 C_TRACE(x, PyObject_Vectorcall(func,
5839 args+1, nargs-1,
5840 kwnames));
5841 Py_DECREF(func);
5842 return x;
5843 }
5844 return PyObject_Vectorcall(func, args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
5845 }
5846
5847 /* Issue #29227: Inline call_function() into _PyEval_EvalFrameDefault()
5848 to reduce the stack consumption. */
Py_LOCAL_INLINE(PyObject *)5849 Py_LOCAL_INLINE(PyObject *) _Py_HOT_FUNCTION
5850 call_function(PyThreadState *tstate,
5851 PyTraceInfo *trace_info,
5852 PyObject ***pp_stack,
5853 Py_ssize_t oparg,
5854 PyObject *kwnames)
5855 {
5856 PyObject **pfunc = (*pp_stack) - oparg - 1;
5857 PyObject *func = *pfunc;
5858 PyObject *x, *w;
5859 Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
5860 Py_ssize_t nargs = oparg - nkwargs;
5861 PyObject **stack = (*pp_stack) - nargs - nkwargs;
5862
5863 if (trace_info->cframe.use_tracing) {
5864 x = trace_call_function(tstate, trace_info, func, stack, nargs, kwnames);
5865 }
5866 else {
5867 x = PyObject_Vectorcall(func, stack, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
5868 }
5869
5870 assert((x != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
5871
5872 /* Clear the stack of the function object. */
5873 while ((*pp_stack) > pfunc) {
5874 w = EXT_POP(*pp_stack);
5875 Py_DECREF(w);
5876 }
5877
5878 return x;
5879 }
5880
5881 static PyObject *
do_call_core(PyThreadState * tstate,PyTraceInfo * trace_info,PyObject * func,PyObject * callargs,PyObject * kwdict)5882 do_call_core(PyThreadState *tstate,
5883 PyTraceInfo *trace_info,
5884 PyObject *func,
5885 PyObject *callargs,
5886 PyObject *kwdict)
5887 {
5888 PyObject *result;
5889
5890 if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) {
5891 C_TRACE(result, PyObject_Call(func, callargs, kwdict));
5892 return result;
5893 }
5894 else if (Py_IS_TYPE(func, &PyMethodDescr_Type)) {
5895 Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
5896 if (nargs > 0 && trace_info->cframe.use_tracing) {
5897 /* We need to create a temporary bound method as argument
5898 for profiling.
5899
5900 If nargs == 0, then this cannot work because we have no
5901 "self". In any case, the call itself would raise
5902 TypeError (foo needs an argument), so we just skip
5903 profiling. */
5904 PyObject *self = PyTuple_GET_ITEM(callargs, 0);
5905 func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self));
5906 if (func == NULL) {
5907 return NULL;
5908 }
5909
5910 C_TRACE(result, _PyObject_FastCallDictTstate(
5911 tstate, func,
5912 &_PyTuple_ITEMS(callargs)[1],
5913 nargs - 1,
5914 kwdict));
5915 Py_DECREF(func);
5916 return result;
5917 }
5918 }
5919 return PyObject_Call(func, callargs, kwdict);
5920 }
5921
5922 /* Extract a slice index from a PyLong or an object with the
5923 nb_index slot defined, and store in *pi.
5924 Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX,
5925 and silently boost values less than PY_SSIZE_T_MIN to PY_SSIZE_T_MIN.
5926 Return 0 on error, 1 on success.
5927 */
5928 int
_PyEval_SliceIndex(PyObject * v,Py_ssize_t * pi)5929 _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi)
5930 {
5931 PyThreadState *tstate = _PyThreadState_GET();
5932 if (!Py_IsNone(v)) {
5933 Py_ssize_t x;
5934 if (_PyIndex_Check(v)) {
5935 x = PyNumber_AsSsize_t(v, NULL);
5936 if (x == -1 && _PyErr_Occurred(tstate))
5937 return 0;
5938 }
5939 else {
5940 _PyErr_SetString(tstate, PyExc_TypeError,
5941 "slice indices must be integers or "
5942 "None or have an __index__ method");
5943 return 0;
5944 }
5945 *pi = x;
5946 }
5947 return 1;
5948 }
5949
5950 int
_PyEval_SliceIndexNotNone(PyObject * v,Py_ssize_t * pi)5951 _PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi)
5952 {
5953 PyThreadState *tstate = _PyThreadState_GET();
5954 Py_ssize_t x;
5955 if (_PyIndex_Check(v)) {
5956 x = PyNumber_AsSsize_t(v, NULL);
5957 if (x == -1 && _PyErr_Occurred(tstate))
5958 return 0;
5959 }
5960 else {
5961 _PyErr_SetString(tstate, PyExc_TypeError,
5962 "slice indices must be integers or "
5963 "have an __index__ method");
5964 return 0;
5965 }
5966 *pi = x;
5967 return 1;
5968 }
5969
5970 static PyObject *
import_name(PyThreadState * tstate,PyFrameObject * f,PyObject * name,PyObject * fromlist,PyObject * level)5971 import_name(PyThreadState *tstate, PyFrameObject *f,
5972 PyObject *name, PyObject *fromlist, PyObject *level)
5973 {
5974 _Py_IDENTIFIER(__import__);
5975 PyObject *import_func, *res;
5976 PyObject* stack[5];
5977
5978 import_func = _PyDict_GetItemIdWithError(f->f_builtins, &PyId___import__);
5979 if (import_func == NULL) {
5980 if (!_PyErr_Occurred(tstate)) {
5981 _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found");
5982 }
5983 return NULL;
5984 }
5985
5986 /* Fast path for not overloaded __import__. */
5987 if (import_func == tstate->interp->import_func) {
5988 int ilevel = _PyLong_AsInt(level);
5989 if (ilevel == -1 && _PyErr_Occurred(tstate)) {
5990 return NULL;
5991 }
5992 res = PyImport_ImportModuleLevelObject(
5993 name,
5994 f->f_globals,
5995 f->f_locals == NULL ? Py_None : f->f_locals,
5996 fromlist,
5997 ilevel);
5998 return res;
5999 }
6000
6001 Py_INCREF(import_func);
6002
6003 stack[0] = name;
6004 stack[1] = f->f_globals;
6005 stack[2] = f->f_locals == NULL ? Py_None : f->f_locals;
6006 stack[3] = fromlist;
6007 stack[4] = level;
6008 res = _PyObject_FastCall(import_func, stack, 5);
6009 Py_DECREF(import_func);
6010 return res;
6011 }
6012
6013 static PyObject *
import_from(PyThreadState * tstate,PyObject * v,PyObject * name)6014 import_from(PyThreadState *tstate, PyObject *v, PyObject *name)
6015 {
6016 PyObject *x;
6017 PyObject *fullmodname, *pkgname, *pkgpath, *pkgname_or_unknown, *errmsg;
6018
6019 if (_PyObject_LookupAttr(v, name, &x) != 0) {
6020 return x;
6021 }
6022 /* Issue #17636: in case this failed because of a circular relative
6023 import, try to fallback on reading the module directly from
6024 sys.modules. */
6025 pkgname = _PyObject_GetAttrId(v, &PyId___name__);
6026 if (pkgname == NULL) {
6027 goto error;
6028 }
6029 if (!PyUnicode_Check(pkgname)) {
6030 Py_CLEAR(pkgname);
6031 goto error;
6032 }
6033 fullmodname = PyUnicode_FromFormat("%U.%U", pkgname, name);
6034 if (fullmodname == NULL) {
6035 Py_DECREF(pkgname);
6036 return NULL;
6037 }
6038 x = PyImport_GetModule(fullmodname);
6039 Py_DECREF(fullmodname);
6040 if (x == NULL && !_PyErr_Occurred(tstate)) {
6041 goto error;
6042 }
6043 Py_DECREF(pkgname);
6044 return x;
6045 error:
6046 pkgpath = PyModule_GetFilenameObject(v);
6047 if (pkgname == NULL) {
6048 pkgname_or_unknown = PyUnicode_FromString("<unknown module name>");
6049 if (pkgname_or_unknown == NULL) {
6050 Py_XDECREF(pkgpath);
6051 return NULL;
6052 }
6053 } else {
6054 pkgname_or_unknown = pkgname;
6055 }
6056
6057 if (pkgpath == NULL || !PyUnicode_Check(pkgpath)) {
6058 _PyErr_Clear(tstate);
6059 errmsg = PyUnicode_FromFormat(
6060 "cannot import name %R from %R (unknown location)",
6061 name, pkgname_or_unknown
6062 );
6063 /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */
6064 PyErr_SetImportError(errmsg, pkgname, NULL);
6065 }
6066 else {
6067 _Py_IDENTIFIER(__spec__);
6068 PyObject *spec = _PyObject_GetAttrId(v, &PyId___spec__);
6069 const char *fmt =
6070 _PyModuleSpec_IsInitializing(spec) ?
6071 "cannot import name %R from partially initialized module %R "
6072 "(most likely due to a circular import) (%S)" :
6073 "cannot import name %R from %R (%S)";
6074 Py_XDECREF(spec);
6075
6076 errmsg = PyUnicode_FromFormat(fmt, name, pkgname_or_unknown, pkgpath);
6077 /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */
6078 PyErr_SetImportError(errmsg, pkgname, pkgpath);
6079 }
6080
6081 Py_XDECREF(errmsg);
6082 Py_XDECREF(pkgname_or_unknown);
6083 Py_XDECREF(pkgpath);
6084 return NULL;
6085 }
6086
6087 static int
import_all_from(PyThreadState * tstate,PyObject * locals,PyObject * v)6088 import_all_from(PyThreadState *tstate, PyObject *locals, PyObject *v)
6089 {
6090 _Py_IDENTIFIER(__all__);
6091 _Py_IDENTIFIER(__dict__);
6092 PyObject *all, *dict, *name, *value;
6093 int skip_leading_underscores = 0;
6094 int pos, err;
6095
6096 if (_PyObject_LookupAttrId(v, &PyId___all__, &all) < 0) {
6097 return -1; /* Unexpected error */
6098 }
6099 if (all == NULL) {
6100 if (_PyObject_LookupAttrId(v, &PyId___dict__, &dict) < 0) {
6101 return -1;
6102 }
6103 if (dict == NULL) {
6104 _PyErr_SetString(tstate, PyExc_ImportError,
6105 "from-import-* object has no __dict__ and no __all__");
6106 return -1;
6107 }
6108 all = PyMapping_Keys(dict);
6109 Py_DECREF(dict);
6110 if (all == NULL)
6111 return -1;
6112 skip_leading_underscores = 1;
6113 }
6114
6115 for (pos = 0, err = 0; ; pos++) {
6116 name = PySequence_GetItem(all, pos);
6117 if (name == NULL) {
6118 if (!_PyErr_ExceptionMatches(tstate, PyExc_IndexError)) {
6119 err = -1;
6120 }
6121 else {
6122 _PyErr_Clear(tstate);
6123 }
6124 break;
6125 }
6126 if (!PyUnicode_Check(name)) {
6127 PyObject *modname = _PyObject_GetAttrId(v, &PyId___name__);
6128 if (modname == NULL) {
6129 Py_DECREF(name);
6130 err = -1;
6131 break;
6132 }
6133 if (!PyUnicode_Check(modname)) {
6134 _PyErr_Format(tstate, PyExc_TypeError,
6135 "module __name__ must be a string, not %.100s",
6136 Py_TYPE(modname)->tp_name);
6137 }
6138 else {
6139 _PyErr_Format(tstate, PyExc_TypeError,
6140 "%s in %U.%s must be str, not %.100s",
6141 skip_leading_underscores ? "Key" : "Item",
6142 modname,
6143 skip_leading_underscores ? "__dict__" : "__all__",
6144 Py_TYPE(name)->tp_name);
6145 }
6146 Py_DECREF(modname);
6147 Py_DECREF(name);
6148 err = -1;
6149 break;
6150 }
6151 if (skip_leading_underscores) {
6152 if (PyUnicode_READY(name) == -1) {
6153 Py_DECREF(name);
6154 err = -1;
6155 break;
6156 }
6157 if (PyUnicode_READ_CHAR(name, 0) == '_') {
6158 Py_DECREF(name);
6159 continue;
6160 }
6161 }
6162 value = PyObject_GetAttr(v, name);
6163 if (value == NULL)
6164 err = -1;
6165 else if (PyDict_CheckExact(locals))
6166 err = PyDict_SetItem(locals, name, value);
6167 else
6168 err = PyObject_SetItem(locals, name, value);
6169 Py_DECREF(name);
6170 Py_XDECREF(value);
6171 if (err != 0)
6172 break;
6173 }
6174 Py_DECREF(all);
6175 return err;
6176 }
6177
6178 static int
check_args_iterable(PyThreadState * tstate,PyObject * func,PyObject * args)6179 check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args)
6180 {
6181 if (Py_TYPE(args)->tp_iter == NULL && !PySequence_Check(args)) {
6182 /* check_args_iterable() may be called with a live exception:
6183 * clear it to prevent calling _PyObject_FunctionStr() with an
6184 * exception set. */
6185 _PyErr_Clear(tstate);
6186 PyObject *funcstr = _PyObject_FunctionStr(func);
6187 if (funcstr != NULL) {
6188 _PyErr_Format(tstate, PyExc_TypeError,
6189 "%U argument after * must be an iterable, not %.200s",
6190 funcstr, Py_TYPE(args)->tp_name);
6191 Py_DECREF(funcstr);
6192 }
6193 return -1;
6194 }
6195 return 0;
6196 }
6197
6198 static void
format_kwargs_error(PyThreadState * tstate,PyObject * func,PyObject * kwargs)6199 format_kwargs_error(PyThreadState *tstate, PyObject *func, PyObject *kwargs)
6200 {
6201 /* _PyDict_MergeEx raises attribute
6202 * error (percolated from an attempt
6203 * to get 'keys' attribute) instead of
6204 * a type error if its second argument
6205 * is not a mapping.
6206 */
6207 if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
6208 _PyErr_Clear(tstate);
6209 PyObject *funcstr = _PyObject_FunctionStr(func);
6210 if (funcstr != NULL) {
6211 _PyErr_Format(
6212 tstate, PyExc_TypeError,
6213 "%U argument after ** must be a mapping, not %.200s",
6214 funcstr, Py_TYPE(kwargs)->tp_name);
6215 Py_DECREF(funcstr);
6216 }
6217 }
6218 else if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
6219 PyObject *exc, *val, *tb;
6220 _PyErr_Fetch(tstate, &exc, &val, &tb);
6221 if (val && PyTuple_Check(val) && PyTuple_GET_SIZE(val) == 1) {
6222 _PyErr_Clear(tstate);
6223 PyObject *funcstr = _PyObject_FunctionStr(func);
6224 if (funcstr != NULL) {
6225 PyObject *key = PyTuple_GET_ITEM(val, 0);
6226 _PyErr_Format(
6227 tstate, PyExc_TypeError,
6228 "%U got multiple values for keyword argument '%S'",
6229 funcstr, key);
6230 Py_DECREF(funcstr);
6231 }
6232 Py_XDECREF(exc);
6233 Py_XDECREF(val);
6234 Py_XDECREF(tb);
6235 }
6236 else {
6237 _PyErr_Restore(tstate, exc, val, tb);
6238 }
6239 }
6240 }
6241
6242 static void
format_exc_check_arg(PyThreadState * tstate,PyObject * exc,const char * format_str,PyObject * obj)6243 format_exc_check_arg(PyThreadState *tstate, PyObject *exc,
6244 const char *format_str, PyObject *obj)
6245 {
6246 const char *obj_str;
6247
6248 if (!obj)
6249 return;
6250
6251 obj_str = PyUnicode_AsUTF8(obj);
6252 if (!obj_str)
6253 return;
6254
6255 _PyErr_Format(tstate, exc, format_str, obj_str);
6256
6257 if (exc == PyExc_NameError) {
6258 // Include the name in the NameError exceptions to offer suggestions later.
6259 _Py_IDENTIFIER(name);
6260 PyObject *type, *value, *traceback;
6261 PyErr_Fetch(&type, &value, &traceback);
6262 PyErr_NormalizeException(&type, &value, &traceback);
6263 if (PyErr_GivenExceptionMatches(value, PyExc_NameError)) {
6264 PyNameErrorObject* exc = (PyNameErrorObject*) value;
6265 if (exc->name == NULL) {
6266 // We do not care if this fails because we are going to restore the
6267 // NameError anyway.
6268 (void)_PyObject_SetAttrId(value, &PyId_name, obj);
6269 }
6270 }
6271 PyErr_Restore(type, value, traceback);
6272 }
6273 }
6274
6275 static void
format_exc_unbound(PyThreadState * tstate,PyCodeObject * co,int oparg)6276 format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
6277 {
6278 PyObject *name;
6279 /* Don't stomp existing exception */
6280 if (_PyErr_Occurred(tstate))
6281 return;
6282 if (oparg < PyTuple_GET_SIZE(co->co_cellvars)) {
6283 name = PyTuple_GET_ITEM(co->co_cellvars,
6284 oparg);
6285 format_exc_check_arg(tstate,
6286 PyExc_UnboundLocalError,
6287 UNBOUNDLOCAL_ERROR_MSG,
6288 name);
6289 } else {
6290 name = PyTuple_GET_ITEM(co->co_freevars, oparg -
6291 PyTuple_GET_SIZE(co->co_cellvars));
6292 format_exc_check_arg(tstate, PyExc_NameError,
6293 UNBOUNDFREE_ERROR_MSG, name);
6294 }
6295 }
6296
6297 static void
format_awaitable_error(PyThreadState * tstate,PyTypeObject * type,int prevprevopcode,int prevopcode)6298 format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevopcode, int prevopcode)
6299 {
6300 if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) {
6301 if (prevopcode == BEFORE_ASYNC_WITH) {
6302 _PyErr_Format(tstate, PyExc_TypeError,
6303 "'async with' received an object from __aenter__ "
6304 "that does not implement __await__: %.100s",
6305 type->tp_name);
6306 }
6307 else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL_FUNCTION && prevprevopcode == DUP_TOP)) {
6308 _PyErr_Format(tstate, PyExc_TypeError,
6309 "'async with' received an object from __aexit__ "
6310 "that does not implement __await__: %.100s",
6311 type->tp_name);
6312 }
6313 }
6314 }
6315
6316 static PyObject *
unicode_concatenate(PyThreadState * tstate,PyObject * v,PyObject * w,PyFrameObject * f,const _Py_CODEUNIT * next_instr)6317 unicode_concatenate(PyThreadState *tstate, PyObject *v, PyObject *w,
6318 PyFrameObject *f, const _Py_CODEUNIT *next_instr)
6319 {
6320 PyObject *res;
6321 if (Py_REFCNT(v) == 2) {
6322 /* In the common case, there are 2 references to the value
6323 * stored in 'variable' when the += is performed: one on the
6324 * value stack (in 'v') and one still stored in the
6325 * 'variable'. We try to delete the variable now to reduce
6326 * the refcnt to 1.
6327 */
6328 int opcode, oparg;
6329 NEXTOPARG();
6330 switch (opcode) {
6331 case STORE_FAST:
6332 {
6333 PyObject **fastlocals = f->f_localsplus;
6334 if (GETLOCAL(oparg) == v)
6335 SETLOCAL(oparg, NULL);
6336 break;
6337 }
6338 case STORE_DEREF:
6339 {
6340 PyObject **freevars = (f->f_localsplus +
6341 f->f_code->co_nlocals);
6342 PyObject *c = freevars[oparg];
6343 if (PyCell_GET(c) == v) {
6344 PyCell_SET(c, NULL);
6345 Py_DECREF(v);
6346 }
6347 break;
6348 }
6349 case STORE_NAME:
6350 {
6351 PyObject *names = f->f_code->co_names;
6352 PyObject *name = GETITEM(names, oparg);
6353 PyObject *locals = f->f_locals;
6354 if (locals && PyDict_CheckExact(locals)) {
6355 PyObject *w = PyDict_GetItemWithError(locals, name);
6356 if ((w == v && PyDict_DelItem(locals, name) != 0) ||
6357 (w == NULL && _PyErr_Occurred(tstate)))
6358 {
6359 Py_DECREF(v);
6360 return NULL;
6361 }
6362 }
6363 break;
6364 }
6365 }
6366 }
6367 res = v;
6368 PyUnicode_Append(&res, w);
6369 return res;
6370 }
6371
6372 #ifdef DYNAMIC_EXECUTION_PROFILE
6373
6374 static PyObject *
getarray(long a[256])6375 getarray(long a[256])
6376 {
6377 int i;
6378 PyObject *l = PyList_New(256);
6379 if (l == NULL) return NULL;
6380 for (i = 0; i < 256; i++) {
6381 PyObject *x = PyLong_FromLong(a[i]);
6382 if (x == NULL) {
6383 Py_DECREF(l);
6384 return NULL;
6385 }
6386 PyList_SET_ITEM(l, i, x);
6387 }
6388 for (i = 0; i < 256; i++)
6389 a[i] = 0;
6390 return l;
6391 }
6392
6393 PyObject *
_Py_GetDXProfile(PyObject * self,PyObject * args)6394 _Py_GetDXProfile(PyObject *self, PyObject *args)
6395 {
6396 #ifndef DXPAIRS
6397 return getarray(dxp);
6398 #else
6399 int i;
6400 PyObject *l = PyList_New(257);
6401 if (l == NULL) return NULL;
6402 for (i = 0; i < 257; i++) {
6403 PyObject *x = getarray(dxpairs[i]);
6404 if (x == NULL) {
6405 Py_DECREF(l);
6406 return NULL;
6407 }
6408 PyList_SET_ITEM(l, i, x);
6409 }
6410 return l;
6411 #endif
6412 }
6413
6414 #endif
6415
6416 Py_ssize_t
_PyEval_RequestCodeExtraIndex(freefunc free)6417 _PyEval_RequestCodeExtraIndex(freefunc free)
6418 {
6419 PyInterpreterState *interp = _PyInterpreterState_GET();
6420 Py_ssize_t new_index;
6421
6422 if (interp->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) {
6423 return -1;
6424 }
6425 new_index = interp->co_extra_user_count++;
6426 interp->co_extra_freefuncs[new_index] = free;
6427 return new_index;
6428 }
6429
6430 static void
dtrace_function_entry(PyFrameObject * f)6431 dtrace_function_entry(PyFrameObject *f)
6432 {
6433 const char *filename;
6434 const char *funcname;
6435 int lineno;
6436
6437 PyCodeObject *code = f->f_code;
6438 filename = PyUnicode_AsUTF8(code->co_filename);
6439 funcname = PyUnicode_AsUTF8(code->co_name);
6440 lineno = PyFrame_GetLineNumber(f);
6441
6442 PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno);
6443 }
6444
6445 static void
dtrace_function_return(PyFrameObject * f)6446 dtrace_function_return(PyFrameObject *f)
6447 {
6448 const char *filename;
6449 const char *funcname;
6450 int lineno;
6451
6452 PyCodeObject *code = f->f_code;
6453 filename = PyUnicode_AsUTF8(code->co_filename);
6454 funcname = PyUnicode_AsUTF8(code->co_name);
6455 lineno = PyFrame_GetLineNumber(f);
6456
6457 PyDTrace_FUNCTION_RETURN(filename, funcname, lineno);
6458 }
6459
6460 /* DTrace equivalent of maybe_call_line_trace. */
6461 static void
maybe_dtrace_line(PyFrameObject * frame,PyTraceInfo * trace_info,int instr_prev)6462 maybe_dtrace_line(PyFrameObject *frame,
6463 PyTraceInfo *trace_info, int instr_prev)
6464 {
6465 const char *co_filename, *co_name;
6466
6467 /* If the last instruction executed isn't in the current
6468 instruction window, reset the window.
6469 */
6470 initialize_trace_info(trace_info, frame);
6471 int line = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &trace_info->bounds);
6472 /* If the last instruction falls at the start of a line or if
6473 it represents a jump backwards, update the frame's line
6474 number and call the trace function. */
6475 if (line != frame->f_lineno || frame->f_lasti < instr_prev) {
6476 if (line != -1) {
6477 frame->f_lineno = line;
6478 co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename);
6479 if (!co_filename)
6480 co_filename = "?";
6481 co_name = PyUnicode_AsUTF8(frame->f_code->co_name);
6482 if (!co_name)
6483 co_name = "?";
6484 PyDTrace_LINE(co_filename, co_name, line);
6485 }
6486 }
6487 }
6488
6489
6490 /* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions
6491 for the limited API. */
6492
6493 #undef Py_EnterRecursiveCall
6494
Py_EnterRecursiveCall(const char * where)6495 int Py_EnterRecursiveCall(const char *where)
6496 {
6497 return _Py_EnterRecursiveCall_inline(where);
6498 }
6499
6500 #undef Py_LeaveRecursiveCall
6501
Py_LeaveRecursiveCall(void)6502 void Py_LeaveRecursiveCall(void)
6503 {
6504 _Py_LeaveRecursiveCall_inline();
6505 }
6506