1 // This file is generated by Tools/cases_generator/tier1_generator.py 2 // from: 3 // Python/bytecodes.c 4 // Do not edit! 5 6 #ifdef TIER_TWO 7 #error "This file is for Tier 1 only" 8 #endif 9 #define TIER_ONE 1 10 11 TARGET(BEFORE_ASYNC_WITH)12 TARGET(BEFORE_ASYNC_WITH) { 13 frame->instr_ptr = next_instr; 14 next_instr += 1; 15 INSTRUCTION_STATS(BEFORE_ASYNC_WITH); 16 PyObject *mgr; 17 PyObject *exit; 18 PyObject *res; 19 mgr = stack_pointer[-1]; 20 PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); 21 if (enter == NULL) { 22 if (!_PyErr_Occurred(tstate)) { 23 _PyErr_Format(tstate, PyExc_TypeError, 24 "'%.200s' object does not support the " 25 "asynchronous context manager protocol", 26 Py_TYPE(mgr)->tp_name); 27 } 28 goto error; 29 } 30 exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); 31 if (exit == NULL) { 32 if (!_PyErr_Occurred(tstate)) { 33 _PyErr_Format(tstate, PyExc_TypeError, 34 "'%.200s' object does not support the " 35 "asynchronous context manager protocol " 36 "(missed __aexit__ method)", 37 Py_TYPE(mgr)->tp_name); 38 } 39 Py_DECREF(enter); 40 goto error; 41 } 42 Py_DECREF(mgr); 43 res = PyObject_CallNoArgs(enter); 44 Py_DECREF(enter); 45 if (res == NULL) { 46 Py_DECREF(exit); 47 if (true) goto pop_1_error; 48 } 49 stack_pointer[-1] = exit; 50 stack_pointer[0] = res; 51 stack_pointer += 1; 52 DISPATCH(); 53 } 54 TARGET(BEFORE_WITH)55 TARGET(BEFORE_WITH) { 56 frame->instr_ptr = next_instr; 57 next_instr += 1; 58 INSTRUCTION_STATS(BEFORE_WITH); 59 PyObject *mgr; 60 PyObject *exit; 61 PyObject *res; 62 mgr = stack_pointer[-1]; 63 /* pop the context manager, push its __exit__ and the 64 * value returned from calling its __enter__ 65 */ 66 PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); 67 if (enter == NULL) { 68 if (!_PyErr_Occurred(tstate)) { 69 _PyErr_Format(tstate, PyExc_TypeError, 70 "'%.200s' object does not support the " 71 "context manager protocol", 72 Py_TYPE(mgr)->tp_name); 73 } 74 goto error; 75 } 76 exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); 77 if (exit == NULL) { 78 if (!_PyErr_Occurred(tstate)) { 79 _PyErr_Format(tstate, PyExc_TypeError, 80 "'%.200s' object does not support the " 81 "context manager protocol " 82 "(missed __exit__ method)", 83 Py_TYPE(mgr)->tp_name); 84 } 85 Py_DECREF(enter); 86 goto error; 87 } 88 Py_DECREF(mgr); 89 res = PyObject_CallNoArgs(enter); 90 Py_DECREF(enter); 91 if (res == NULL) { 92 Py_DECREF(exit); 93 if (true) goto pop_1_error; 94 } 95 stack_pointer[-1] = exit; 96 stack_pointer[0] = res; 97 stack_pointer += 1; 98 DISPATCH(); 99 } 100 TARGET(BINARY_OP)101 TARGET(BINARY_OP) { 102 frame->instr_ptr = next_instr; 103 next_instr += 2; 104 INSTRUCTION_STATS(BINARY_OP); 105 PREDICTED(BINARY_OP); 106 _Py_CODEUNIT *this_instr = next_instr - 2; 107 (void)this_instr; 108 PyObject *rhs; 109 PyObject *lhs; 110 PyObject *res; 111 // _SPECIALIZE_BINARY_OP 112 rhs = stack_pointer[-1]; 113 lhs = stack_pointer[-2]; 114 { 115 uint16_t counter = read_u16(&this_instr[1].cache); 116 (void)counter; 117 #if ENABLE_SPECIALIZATION 118 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 119 next_instr = this_instr; 120 _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); 121 DISPATCH_SAME_OPARG(); 122 } 123 STAT_INC(BINARY_OP, deferred); 124 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 125 #endif /* ENABLE_SPECIALIZATION */ 126 assert(NB_ADD <= oparg); 127 assert(oparg <= NB_INPLACE_XOR); 128 } 129 // _BINARY_OP 130 { 131 assert(_PyEval_BinaryOps[oparg]); 132 res = _PyEval_BinaryOps[oparg](lhs, rhs); 133 Py_DECREF(lhs); 134 Py_DECREF(rhs); 135 if (res == NULL) goto pop_2_error; 136 } 137 stack_pointer[-2] = res; 138 stack_pointer += -1; 139 DISPATCH(); 140 } 141 TARGET(BINARY_OP_ADD_FLOAT)142 TARGET(BINARY_OP_ADD_FLOAT) { 143 frame->instr_ptr = next_instr; 144 next_instr += 2; 145 INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); 146 static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); 147 PyObject *right; 148 PyObject *left; 149 PyObject *res; 150 // _GUARD_BOTH_FLOAT 151 right = stack_pointer[-1]; 152 left = stack_pointer[-2]; 153 { 154 DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); 155 DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); 156 } 157 /* Skip 1 cache entry */ 158 // _BINARY_OP_ADD_FLOAT 159 { 160 STAT_INC(BINARY_OP, hit); 161 double dres = 162 ((PyFloatObject *)left)->ob_fval + 163 ((PyFloatObject *)right)->ob_fval; 164 DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); 165 } 166 stack_pointer[-2] = res; 167 stack_pointer += -1; 168 DISPATCH(); 169 } 170 TARGET(BINARY_OP_ADD_INT)171 TARGET(BINARY_OP_ADD_INT) { 172 frame->instr_ptr = next_instr; 173 next_instr += 2; 174 INSTRUCTION_STATS(BINARY_OP_ADD_INT); 175 static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); 176 PyObject *right; 177 PyObject *left; 178 PyObject *res; 179 // _GUARD_BOTH_INT 180 right = stack_pointer[-1]; 181 left = stack_pointer[-2]; 182 { 183 DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); 184 DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); 185 } 186 /* Skip 1 cache entry */ 187 // _BINARY_OP_ADD_INT 188 { 189 STAT_INC(BINARY_OP, hit); 190 res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); 191 _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); 192 _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); 193 if (res == NULL) goto pop_2_error; 194 } 195 stack_pointer[-2] = res; 196 stack_pointer += -1; 197 DISPATCH(); 198 } 199 TARGET(BINARY_OP_ADD_UNICODE)200 TARGET(BINARY_OP_ADD_UNICODE) { 201 frame->instr_ptr = next_instr; 202 next_instr += 2; 203 INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); 204 static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); 205 PyObject *right; 206 PyObject *left; 207 PyObject *res; 208 // _GUARD_BOTH_UNICODE 209 right = stack_pointer[-1]; 210 left = stack_pointer[-2]; 211 { 212 DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); 213 DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); 214 } 215 /* Skip 1 cache entry */ 216 // _BINARY_OP_ADD_UNICODE 217 { 218 STAT_INC(BINARY_OP, hit); 219 res = PyUnicode_Concat(left, right); 220 _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); 221 _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); 222 if (res == NULL) goto pop_2_error; 223 } 224 stack_pointer[-2] = res; 225 stack_pointer += -1; 226 DISPATCH(); 227 } 228 TARGET(BINARY_OP_INPLACE_ADD_UNICODE)229 TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { 230 frame->instr_ptr = next_instr; 231 next_instr += 2; 232 INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); 233 static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); 234 PyObject *right; 235 PyObject *left; 236 // _GUARD_BOTH_UNICODE 237 right = stack_pointer[-1]; 238 left = stack_pointer[-2]; 239 { 240 DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); 241 DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); 242 } 243 /* Skip 1 cache entry */ 244 // _BINARY_OP_INPLACE_ADD_UNICODE 245 { 246 assert(next_instr->op.code == STORE_FAST); 247 PyObject **target_local = &GETLOCAL(next_instr->op.arg); 248 DEOPT_IF(*target_local != left, BINARY_OP); 249 STAT_INC(BINARY_OP, hit); 250 /* Handle `left = left + right` or `left += right` for str. 251 * 252 * When possible, extend `left` in place rather than 253 * allocating a new PyUnicodeObject. This attempts to avoid 254 * quadratic behavior when one neglects to use str.join(). 255 * 256 * If `left` has only two references remaining (one from 257 * the stack, one in the locals), DECREFing `left` leaves 258 * only the locals reference, so PyUnicode_Append knows 259 * that the string is safe to mutate. 260 */ 261 assert(Py_REFCNT(left) >= 2); 262 _Py_DECREF_NO_DEALLOC(left); 263 PyUnicode_Append(target_local, right); 264 _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); 265 if (*target_local == NULL) goto pop_2_error; 266 // The STORE_FAST is already done. 267 assert(next_instr->op.code == STORE_FAST); 268 SKIP_OVER(1); 269 } 270 stack_pointer += -2; 271 DISPATCH(); 272 } 273 TARGET(BINARY_OP_MULTIPLY_FLOAT)274 TARGET(BINARY_OP_MULTIPLY_FLOAT) { 275 frame->instr_ptr = next_instr; 276 next_instr += 2; 277 INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); 278 static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); 279 PyObject *right; 280 PyObject *left; 281 PyObject *res; 282 // _GUARD_BOTH_FLOAT 283 right = stack_pointer[-1]; 284 left = stack_pointer[-2]; 285 { 286 DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); 287 DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); 288 } 289 /* Skip 1 cache entry */ 290 // _BINARY_OP_MULTIPLY_FLOAT 291 { 292 STAT_INC(BINARY_OP, hit); 293 double dres = 294 ((PyFloatObject *)left)->ob_fval * 295 ((PyFloatObject *)right)->ob_fval; 296 DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); 297 } 298 stack_pointer[-2] = res; 299 stack_pointer += -1; 300 DISPATCH(); 301 } 302 TARGET(BINARY_OP_MULTIPLY_INT)303 TARGET(BINARY_OP_MULTIPLY_INT) { 304 frame->instr_ptr = next_instr; 305 next_instr += 2; 306 INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); 307 static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); 308 PyObject *right; 309 PyObject *left; 310 PyObject *res; 311 // _GUARD_BOTH_INT 312 right = stack_pointer[-1]; 313 left = stack_pointer[-2]; 314 { 315 DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); 316 DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); 317 } 318 /* Skip 1 cache entry */ 319 // _BINARY_OP_MULTIPLY_INT 320 { 321 STAT_INC(BINARY_OP, hit); 322 res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); 323 _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); 324 _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); 325 if (res == NULL) goto pop_2_error; 326 } 327 stack_pointer[-2] = res; 328 stack_pointer += -1; 329 DISPATCH(); 330 } 331 TARGET(BINARY_OP_SUBTRACT_FLOAT)332 TARGET(BINARY_OP_SUBTRACT_FLOAT) { 333 frame->instr_ptr = next_instr; 334 next_instr += 2; 335 INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); 336 static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); 337 PyObject *right; 338 PyObject *left; 339 PyObject *res; 340 // _GUARD_BOTH_FLOAT 341 right = stack_pointer[-1]; 342 left = stack_pointer[-2]; 343 { 344 DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); 345 DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); 346 } 347 /* Skip 1 cache entry */ 348 // _BINARY_OP_SUBTRACT_FLOAT 349 { 350 STAT_INC(BINARY_OP, hit); 351 double dres = 352 ((PyFloatObject *)left)->ob_fval - 353 ((PyFloatObject *)right)->ob_fval; 354 DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); 355 } 356 stack_pointer[-2] = res; 357 stack_pointer += -1; 358 DISPATCH(); 359 } 360 TARGET(BINARY_OP_SUBTRACT_INT)361 TARGET(BINARY_OP_SUBTRACT_INT) { 362 frame->instr_ptr = next_instr; 363 next_instr += 2; 364 INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); 365 static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); 366 PyObject *right; 367 PyObject *left; 368 PyObject *res; 369 // _GUARD_BOTH_INT 370 right = stack_pointer[-1]; 371 left = stack_pointer[-2]; 372 { 373 DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); 374 DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); 375 } 376 /* Skip 1 cache entry */ 377 // _BINARY_OP_SUBTRACT_INT 378 { 379 STAT_INC(BINARY_OP, hit); 380 res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); 381 _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); 382 _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); 383 if (res == NULL) goto pop_2_error; 384 } 385 stack_pointer[-2] = res; 386 stack_pointer += -1; 387 DISPATCH(); 388 } 389 TARGET(BINARY_SLICE)390 TARGET(BINARY_SLICE) { 391 frame->instr_ptr = next_instr; 392 next_instr += 1; 393 INSTRUCTION_STATS(BINARY_SLICE); 394 PyObject *stop; 395 PyObject *start; 396 PyObject *container; 397 PyObject *res; 398 stop = stack_pointer[-1]; 399 start = stack_pointer[-2]; 400 container = stack_pointer[-3]; 401 PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); 402 // Can't use ERROR_IF() here, because we haven't 403 // DECREF'ed container yet, and we still own slice. 404 if (slice == NULL) { 405 res = NULL; 406 } 407 else { 408 res = PyObject_GetItem(container, slice); 409 Py_DECREF(slice); 410 } 411 Py_DECREF(container); 412 if (res == NULL) goto pop_3_error; 413 stack_pointer[-3] = res; 414 stack_pointer += -2; 415 DISPATCH(); 416 } 417 TARGET(BINARY_SUBSCR)418 TARGET(BINARY_SUBSCR) { 419 frame->instr_ptr = next_instr; 420 next_instr += 2; 421 INSTRUCTION_STATS(BINARY_SUBSCR); 422 PREDICTED(BINARY_SUBSCR); 423 _Py_CODEUNIT *this_instr = next_instr - 2; 424 (void)this_instr; 425 PyObject *sub; 426 PyObject *container; 427 PyObject *res; 428 // _SPECIALIZE_BINARY_SUBSCR 429 sub = stack_pointer[-1]; 430 container = stack_pointer[-2]; 431 { 432 uint16_t counter = read_u16(&this_instr[1].cache); 433 (void)counter; 434 #if ENABLE_SPECIALIZATION 435 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 436 next_instr = this_instr; 437 _Py_Specialize_BinarySubscr(container, sub, next_instr); 438 DISPATCH_SAME_OPARG(); 439 } 440 STAT_INC(BINARY_SUBSCR, deferred); 441 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 442 #endif /* ENABLE_SPECIALIZATION */ 443 } 444 // _BINARY_SUBSCR 445 { 446 res = PyObject_GetItem(container, sub); 447 Py_DECREF(container); 448 Py_DECREF(sub); 449 if (res == NULL) goto pop_2_error; 450 } 451 stack_pointer[-2] = res; 452 stack_pointer += -1; 453 DISPATCH(); 454 } 455 TARGET(BINARY_SUBSCR_DICT)456 TARGET(BINARY_SUBSCR_DICT) { 457 frame->instr_ptr = next_instr; 458 next_instr += 2; 459 INSTRUCTION_STATS(BINARY_SUBSCR_DICT); 460 static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); 461 PyObject *sub; 462 PyObject *dict; 463 PyObject *res; 464 /* Skip 1 cache entry */ 465 sub = stack_pointer[-1]; 466 dict = stack_pointer[-2]; 467 DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); 468 STAT_INC(BINARY_SUBSCR, hit); 469 int rc = PyDict_GetItemRef(dict, sub, &res); 470 if (rc == 0) { 471 _PyErr_SetKeyError(sub); 472 } 473 Py_DECREF(dict); 474 Py_DECREF(sub); 475 if (rc <= 0) goto pop_2_error; 476 // not found or error 477 stack_pointer[-2] = res; 478 stack_pointer += -1; 479 DISPATCH(); 480 } 481 TARGET(BINARY_SUBSCR_GETITEM)482 TARGET(BINARY_SUBSCR_GETITEM) { 483 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 484 next_instr += 2; 485 INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); 486 static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); 487 PyObject *sub; 488 PyObject *container; 489 /* Skip 1 cache entry */ 490 sub = stack_pointer[-1]; 491 container = stack_pointer[-2]; 492 DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); 493 PyTypeObject *tp = Py_TYPE(container); 494 DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); 495 PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; 496 PyObject *cached = ht->_spec_cache.getitem; 497 DEOPT_IF(cached == NULL, BINARY_SUBSCR); 498 assert(PyFunction_Check(cached)); 499 PyFunctionObject *getitem = (PyFunctionObject *)cached; 500 uint32_t cached_version = ht->_spec_cache.getitem_version; 501 DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); 502 PyCodeObject *code = (PyCodeObject *)getitem->func_code; 503 assert(code->co_argcount == 2); 504 DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); 505 STAT_INC(BINARY_SUBSCR, hit); 506 Py_INCREF(getitem); 507 _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); 508 STACK_SHRINK(2); 509 new_frame->localsplus[0] = container; 510 new_frame->localsplus[1] = sub; 511 frame->return_offset = (uint16_t)(next_instr - this_instr); 512 DISPATCH_INLINED(new_frame); 513 } 514 TARGET(BINARY_SUBSCR_LIST_INT)515 TARGET(BINARY_SUBSCR_LIST_INT) { 516 frame->instr_ptr = next_instr; 517 next_instr += 2; 518 INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); 519 static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); 520 PyObject *sub; 521 PyObject *list; 522 PyObject *res; 523 /* Skip 1 cache entry */ 524 sub = stack_pointer[-1]; 525 list = stack_pointer[-2]; 526 DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); 527 DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); 528 // Deopt unless 0 <= sub < PyList_Size(list) 529 DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); 530 Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; 531 DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); 532 STAT_INC(BINARY_SUBSCR, hit); 533 res = PyList_GET_ITEM(list, index); 534 assert(res != NULL); 535 Py_INCREF(res); 536 _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); 537 Py_DECREF(list); 538 stack_pointer[-2] = res; 539 stack_pointer += -1; 540 DISPATCH(); 541 } 542 TARGET(BINARY_SUBSCR_STR_INT)543 TARGET(BINARY_SUBSCR_STR_INT) { 544 frame->instr_ptr = next_instr; 545 next_instr += 2; 546 INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); 547 static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); 548 PyObject *sub; 549 PyObject *str; 550 PyObject *res; 551 /* Skip 1 cache entry */ 552 sub = stack_pointer[-1]; 553 str = stack_pointer[-2]; 554 DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); 555 DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); 556 DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); 557 Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; 558 DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); 559 // Specialize for reading an ASCII character from any string: 560 Py_UCS4 c = PyUnicode_READ_CHAR(str, index); 561 DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); 562 STAT_INC(BINARY_SUBSCR, hit); 563 res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; 564 _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); 565 Py_DECREF(str); 566 stack_pointer[-2] = res; 567 stack_pointer += -1; 568 DISPATCH(); 569 } 570 TARGET(BINARY_SUBSCR_TUPLE_INT)571 TARGET(BINARY_SUBSCR_TUPLE_INT) { 572 frame->instr_ptr = next_instr; 573 next_instr += 2; 574 INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); 575 static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); 576 PyObject *sub; 577 PyObject *tuple; 578 PyObject *res; 579 /* Skip 1 cache entry */ 580 sub = stack_pointer[-1]; 581 tuple = stack_pointer[-2]; 582 DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); 583 DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); 584 // Deopt unless 0 <= sub < PyTuple_Size(list) 585 DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); 586 Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; 587 DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); 588 STAT_INC(BINARY_SUBSCR, hit); 589 res = PyTuple_GET_ITEM(tuple, index); 590 assert(res != NULL); 591 Py_INCREF(res); 592 _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); 593 Py_DECREF(tuple); 594 stack_pointer[-2] = res; 595 stack_pointer += -1; 596 DISPATCH(); 597 } 598 TARGET(BUILD_CONST_KEY_MAP)599 TARGET(BUILD_CONST_KEY_MAP) { 600 frame->instr_ptr = next_instr; 601 next_instr += 1; 602 INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); 603 PyObject *keys; 604 PyObject **values; 605 PyObject *map; 606 keys = stack_pointer[-1]; 607 values = &stack_pointer[-1 - oparg]; 608 assert(PyTuple_CheckExact(keys)); 609 assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); 610 map = _PyDict_FromItems( 611 &PyTuple_GET_ITEM(keys, 0), 1, 612 values, 1, oparg); 613 for (int _i = oparg; --_i >= 0;) { 614 Py_DECREF(values[_i]); 615 } 616 Py_DECREF(keys); 617 if (map == NULL) { stack_pointer += -1 - oparg; goto error; } 618 stack_pointer[-1 - oparg] = map; 619 stack_pointer += -oparg; 620 DISPATCH(); 621 } 622 TARGET(BUILD_LIST)623 TARGET(BUILD_LIST) { 624 frame->instr_ptr = next_instr; 625 next_instr += 1; 626 INSTRUCTION_STATS(BUILD_LIST); 627 PyObject **values; 628 PyObject *list; 629 values = &stack_pointer[-oparg]; 630 list = _PyList_FromArraySteal(values, oparg); 631 if (list == NULL) { stack_pointer += -oparg; goto error; } 632 stack_pointer[-oparg] = list; 633 stack_pointer += 1 - oparg; 634 DISPATCH(); 635 } 636 TARGET(BUILD_MAP)637 TARGET(BUILD_MAP) { 638 frame->instr_ptr = next_instr; 639 next_instr += 1; 640 INSTRUCTION_STATS(BUILD_MAP); 641 PyObject **values; 642 PyObject *map; 643 values = &stack_pointer[-oparg*2]; 644 map = _PyDict_FromItems( 645 values, 2, 646 values+1, 2, 647 oparg); 648 for (int _i = oparg*2; --_i >= 0;) { 649 Py_DECREF(values[_i]); 650 } 651 if (map == NULL) { stack_pointer += -oparg*2; goto error; } 652 stack_pointer[-oparg*2] = map; 653 stack_pointer += 1 - oparg*2; 654 DISPATCH(); 655 } 656 TARGET(BUILD_SET)657 TARGET(BUILD_SET) { 658 frame->instr_ptr = next_instr; 659 next_instr += 1; 660 INSTRUCTION_STATS(BUILD_SET); 661 PyObject **values; 662 PyObject *set; 663 values = &stack_pointer[-oparg]; 664 set = PySet_New(NULL); 665 if (set == NULL) 666 goto error; 667 int err = 0; 668 for (int i = 0; i < oparg; i++) { 669 PyObject *item = values[i]; 670 if (err == 0) 671 err = PySet_Add(set, item); 672 Py_DECREF(item); 673 } 674 if (err != 0) { 675 Py_DECREF(set); 676 if (true) { stack_pointer += -oparg; goto error; } 677 } 678 stack_pointer[-oparg] = set; 679 stack_pointer += 1 - oparg; 680 DISPATCH(); 681 } 682 TARGET(BUILD_SLICE)683 TARGET(BUILD_SLICE) { 684 frame->instr_ptr = next_instr; 685 next_instr += 1; 686 INSTRUCTION_STATS(BUILD_SLICE); 687 PyObject *step = NULL; 688 PyObject *stop; 689 PyObject *start; 690 PyObject *slice; 691 if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } 692 stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; 693 start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; 694 slice = PySlice_New(start, stop, step); 695 Py_DECREF(start); 696 Py_DECREF(stop); 697 Py_XDECREF(step); 698 if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; } 699 stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; 700 stack_pointer += -1 - ((oparg == 3) ? 1 : 0); 701 DISPATCH(); 702 } 703 TARGET(BUILD_STRING)704 TARGET(BUILD_STRING) { 705 frame->instr_ptr = next_instr; 706 next_instr += 1; 707 INSTRUCTION_STATS(BUILD_STRING); 708 PyObject **pieces; 709 PyObject *str; 710 pieces = &stack_pointer[-oparg]; 711 str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); 712 for (int _i = oparg; --_i >= 0;) { 713 Py_DECREF(pieces[_i]); 714 } 715 if (str == NULL) { stack_pointer += -oparg; goto error; } 716 stack_pointer[-oparg] = str; 717 stack_pointer += 1 - oparg; 718 DISPATCH(); 719 } 720 TARGET(BUILD_TUPLE)721 TARGET(BUILD_TUPLE) { 722 frame->instr_ptr = next_instr; 723 next_instr += 1; 724 INSTRUCTION_STATS(BUILD_TUPLE); 725 PyObject **values; 726 PyObject *tup; 727 values = &stack_pointer[-oparg]; 728 tup = _PyTuple_FromArraySteal(values, oparg); 729 if (tup == NULL) { stack_pointer += -oparg; goto error; } 730 stack_pointer[-oparg] = tup; 731 stack_pointer += 1 - oparg; 732 DISPATCH(); 733 } 734 TARGET(CACHE)735 TARGET(CACHE) { 736 frame->instr_ptr = next_instr; 737 next_instr += 1; 738 INSTRUCTION_STATS(CACHE); 739 assert(0 && "Executing a cache."); 740 Py_FatalError("Executing a cache."); 741 DISPATCH(); 742 } 743 TARGET(CALL)744 TARGET(CALL) { 745 frame->instr_ptr = next_instr; 746 next_instr += 4; 747 INSTRUCTION_STATS(CALL); 748 PREDICTED(CALL); 749 _Py_CODEUNIT *this_instr = next_instr - 4; 750 (void)this_instr; 751 PyObject **args; 752 PyObject *self_or_null; 753 PyObject *callable; 754 PyObject *res; 755 // _SPECIALIZE_CALL 756 args = &stack_pointer[-oparg]; 757 self_or_null = stack_pointer[-1 - oparg]; 758 callable = stack_pointer[-2 - oparg]; 759 { 760 uint16_t counter = read_u16(&this_instr[1].cache); 761 (void)counter; 762 #if ENABLE_SPECIALIZATION 763 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 764 next_instr = this_instr; 765 _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); 766 DISPATCH_SAME_OPARG(); 767 } 768 STAT_INC(CALL, deferred); 769 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 770 #endif /* ENABLE_SPECIALIZATION */ 771 } 772 /* Skip 2 cache entries */ 773 // _CALL 774 { 775 // oparg counts all of the args, but *not* self: 776 int total_args = oparg; 777 if (self_or_null != NULL) { 778 args--; 779 total_args++; 780 } 781 else if (Py_TYPE(callable) == &PyMethod_Type) { 782 args--; 783 total_args++; 784 PyObject *self = ((PyMethodObject *)callable)->im_self; 785 args[0] = Py_NewRef(self); 786 PyObject *method = ((PyMethodObject *)callable)->im_func; 787 args[-1] = Py_NewRef(method); 788 Py_DECREF(callable); 789 callable = method; 790 } 791 // Check if the call can be inlined or not 792 if (Py_TYPE(callable) == &PyFunction_Type && 793 tstate->interp->eval_frame == NULL && 794 ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) 795 { 796 int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; 797 PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); 798 _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( 799 tstate, (PyFunctionObject *)callable, locals, 800 args, total_args, NULL 801 ); 802 // Manipulate stack directly since we leave using DISPATCH_INLINED(). 803 STACK_SHRINK(oparg + 2); 804 // The frame has stolen all the arguments from the stack, 805 // so there is no need to clean them up. 806 if (new_frame == NULL) { 807 goto error; 808 } 809 frame->return_offset = (uint16_t)(next_instr - this_instr); 810 DISPATCH_INLINED(new_frame); 811 } 812 /* Callable is not a normal Python function */ 813 res = PyObject_Vectorcall( 814 callable, args, 815 total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, 816 NULL); 817 if (opcode == INSTRUMENTED_CALL) { 818 PyObject *arg = total_args == 0 ? 819 &_PyInstrumentation_MISSING : args[0]; 820 if (res == NULL) { 821 _Py_call_instrumentation_exc2( 822 tstate, PY_MONITORING_EVENT_C_RAISE, 823 frame, this_instr, callable, arg); 824 } 825 else { 826 int err = _Py_call_instrumentation_2args( 827 tstate, PY_MONITORING_EVENT_C_RETURN, 828 frame, this_instr, callable, arg); 829 if (err < 0) { 830 Py_CLEAR(res); 831 } 832 } 833 } 834 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 835 Py_DECREF(callable); 836 for (int i = 0; i < total_args; i++) { 837 Py_DECREF(args[i]); 838 } 839 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 840 } 841 // _CHECK_PERIODIC 842 { 843 } 844 stack_pointer[-2 - oparg] = res; 845 stack_pointer += -1 - oparg; 846 CHECK_EVAL_BREAKER(); 847 DISPATCH(); 848 } 849 TARGET(CALL_ALLOC_AND_ENTER_INIT)850 TARGET(CALL_ALLOC_AND_ENTER_INIT) { 851 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 852 next_instr += 4; 853 INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); 854 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 855 PyObject **args; 856 PyObject *null; 857 PyObject *callable; 858 /* Skip 1 cache entry */ 859 /* Skip 2 cache entries */ 860 args = &stack_pointer[-oparg]; 861 null = stack_pointer[-1 - oparg]; 862 callable = stack_pointer[-2 - oparg]; 863 /* This instruction does the following: 864 * 1. Creates the object (by calling ``object.__new__``) 865 * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) 866 * 3. Pushes the frame for ``__init__`` to the frame stack 867 * */ 868 _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; 869 DEOPT_IF(null != NULL, CALL); 870 DEOPT_IF(!PyType_Check(callable), CALL); 871 PyTypeObject *tp = (PyTypeObject *)callable; 872 DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); 873 assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); 874 PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; 875 PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; 876 PyCodeObject *code = (PyCodeObject *)init->func_code; 877 DEOPT_IF(code->co_argcount != oparg+1, CALL); 878 DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, CALL); 879 DEOPT_IF(code->co_kwonlyargcount, CALL); 880 DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); 881 STAT_INC(CALL, hit); 882 PyObject *self = _PyType_NewManagedObject(tp); 883 if (self == NULL) { 884 goto error; 885 } 886 Py_DECREF(tp); 887 _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( 888 tstate, (PyCodeObject *)&_Py_InitCleanup, 1); 889 assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); 890 /* Push self onto stack of shim */ 891 Py_INCREF(self); 892 shim->localsplus[0] = self; 893 Py_INCREF(init); 894 _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); 895 /* Copy self followed by args to __init__ frame */ 896 init_frame->localsplus[0] = self; 897 for (int i = 0; i < oparg; i++) { 898 init_frame->localsplus[i+1] = args[i]; 899 } 900 frame->return_offset = (uint16_t)(next_instr - this_instr); 901 STACK_SHRINK(oparg+2); 902 _PyFrame_SetStackPointer(frame, stack_pointer); 903 /* Link frames */ 904 init_frame->previous = shim; 905 shim->previous = frame; 906 frame = tstate->current_frame = init_frame; 907 CALL_STAT_INC(inlined_py_calls); 908 /* Account for pushing the extra frame. 909 * We don't check recursion depth here, 910 * as it will be checked after start_frame */ 911 tstate->py_recursion_remaining--; 912 goto start_frame; 913 } 914 TARGET(CALL_BOUND_METHOD_EXACT_ARGS)915 TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { 916 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 917 next_instr += 4; 918 INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); 919 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 920 PyObject *null; 921 PyObject *callable; 922 PyObject *func; 923 PyObject *self; 924 PyObject *self_or_null; 925 PyObject **args; 926 _PyInterpreterFrame *new_frame; 927 /* Skip 1 cache entry */ 928 // _CHECK_PEP_523 929 { 930 DEOPT_IF(tstate->interp->eval_frame, CALL); 931 } 932 // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 933 null = stack_pointer[-1 - oparg]; 934 callable = stack_pointer[-2 - oparg]; 935 { 936 DEOPT_IF(null != NULL, CALL); 937 DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); 938 } 939 // _INIT_CALL_BOUND_METHOD_EXACT_ARGS 940 { 941 STAT_INC(CALL, hit); 942 self = Py_NewRef(((PyMethodObject *)callable)->im_self); 943 stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS 944 func = Py_NewRef(((PyMethodObject *)callable)->im_func); 945 stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization 946 Py_DECREF(callable); 947 } 948 // _CHECK_FUNCTION_EXACT_ARGS 949 self_or_null = self; 950 callable = func; 951 { 952 uint32_t func_version = read_u32(&this_instr[2].cache); 953 DEOPT_IF(!PyFunction_Check(callable), CALL); 954 PyFunctionObject *func = (PyFunctionObject *)callable; 955 DEOPT_IF(func->func_version != func_version, CALL); 956 PyCodeObject *code = (PyCodeObject *)func->func_code; 957 DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); 958 } 959 // _CHECK_STACK_SPACE 960 { 961 PyFunctionObject *func = (PyFunctionObject *)callable; 962 PyCodeObject *code = (PyCodeObject *)func->func_code; 963 DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); 964 DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); 965 } 966 // _INIT_CALL_PY_EXACT_ARGS 967 args = &stack_pointer[-oparg]; 968 self_or_null = stack_pointer[-1 - oparg]; 969 { 970 int has_self = (self_or_null != NULL); 971 STAT_INC(CALL, hit); 972 PyFunctionObject *func = (PyFunctionObject *)callable; 973 new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); 974 PyObject **first_non_self_local = new_frame->localsplus + has_self; 975 new_frame->localsplus[0] = self_or_null; 976 for (int i = 0; i < oparg; i++) { 977 first_non_self_local[i] = args[i]; 978 } 979 } 980 // _SAVE_RETURN_OFFSET 981 { 982 #if TIER_ONE 983 frame->return_offset = (uint16_t)(next_instr - this_instr); 984 #endif 985 #if TIER_TWO 986 frame->return_offset = oparg; 987 #endif 988 } 989 // _PUSH_FRAME 990 { 991 // Write it out explicitly because it's subtly different. 992 // Eventually this should be the only occurrence of this code. 993 assert(tstate->interp->eval_frame == NULL); 994 stack_pointer += -2 - oparg; 995 _PyFrame_SetStackPointer(frame, stack_pointer); 996 new_frame->previous = frame; 997 CALL_STAT_INC(inlined_py_calls); 998 frame = tstate->current_frame = new_frame; 999 tstate->py_recursion_remaining--; 1000 LOAD_SP(); 1001 LOAD_IP(0); 1002 LLTRACE_RESUME_FRAME(); 1003 } 1004 DISPATCH(); 1005 } 1006 TARGET(CALL_BOUND_METHOD_GENERAL)1007 TARGET(CALL_BOUND_METHOD_GENERAL) { 1008 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 1009 next_instr += 4; 1010 INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); 1011 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1012 PyObject *null; 1013 PyObject *callable; 1014 PyObject *method; 1015 PyObject *self; 1016 PyObject **args; 1017 PyObject *self_or_null; 1018 _PyInterpreterFrame *new_frame; 1019 /* Skip 1 cache entry */ 1020 // _CHECK_PEP_523 1021 { 1022 DEOPT_IF(tstate->interp->eval_frame, CALL); 1023 } 1024 // _CHECK_METHOD_VERSION 1025 null = stack_pointer[-1 - oparg]; 1026 callable = stack_pointer[-2 - oparg]; 1027 { 1028 uint32_t func_version = read_u32(&this_instr[2].cache); 1029 DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); 1030 PyObject *func = ((PyMethodObject *)callable)->im_func; 1031 DEOPT_IF(!PyFunction_Check(func), CALL); 1032 DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); 1033 DEOPT_IF(null != NULL, CALL); 1034 } 1035 // _EXPAND_METHOD 1036 { 1037 assert(null == NULL); 1038 assert(Py_TYPE(callable) == &PyMethod_Type); 1039 self = ((PyMethodObject *)callable)->im_self; 1040 Py_INCREF(self); 1041 stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL 1042 method = ((PyMethodObject *)callable)->im_func; 1043 assert(PyFunction_Check(method)); 1044 Py_INCREF(method); 1045 Py_DECREF(callable); 1046 } 1047 // _PY_FRAME_GENERAL 1048 args = &stack_pointer[-oparg]; 1049 self_or_null = self; 1050 callable = method; 1051 { 1052 // oparg counts all of the args, but *not* self: 1053 int total_args = oparg; 1054 if (self_or_null != NULL) { 1055 args--; 1056 total_args++; 1057 } 1058 assert(Py_TYPE(callable) == &PyFunction_Type); 1059 int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; 1060 PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); 1061 new_frame = _PyEvalFramePushAndInit( 1062 tstate, (PyFunctionObject *)callable, locals, 1063 args, total_args, NULL 1064 ); 1065 // The frame has stolen all the arguments from the stack, 1066 // so there is no need to clean them up. 1067 stack_pointer += -2 - oparg; 1068 if (new_frame == NULL) { 1069 goto error; 1070 } 1071 } 1072 // _SAVE_RETURN_OFFSET 1073 { 1074 #if TIER_ONE 1075 frame->return_offset = (uint16_t)(next_instr - this_instr); 1076 #endif 1077 #if TIER_TWO 1078 frame->return_offset = oparg; 1079 #endif 1080 } 1081 // _PUSH_FRAME 1082 { 1083 // Write it out explicitly because it's subtly different. 1084 // Eventually this should be the only occurrence of this code. 1085 assert(tstate->interp->eval_frame == NULL); 1086 _PyFrame_SetStackPointer(frame, stack_pointer); 1087 new_frame->previous = frame; 1088 CALL_STAT_INC(inlined_py_calls); 1089 frame = tstate->current_frame = new_frame; 1090 tstate->py_recursion_remaining--; 1091 LOAD_SP(); 1092 LOAD_IP(0); 1093 LLTRACE_RESUME_FRAME(); 1094 } 1095 DISPATCH(); 1096 } 1097 TARGET(CALL_BUILTIN_CLASS)1098 TARGET(CALL_BUILTIN_CLASS) { 1099 frame->instr_ptr = next_instr; 1100 next_instr += 4; 1101 INSTRUCTION_STATS(CALL_BUILTIN_CLASS); 1102 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1103 PyObject **args; 1104 PyObject *self_or_null; 1105 PyObject *callable; 1106 PyObject *res; 1107 /* Skip 1 cache entry */ 1108 /* Skip 2 cache entries */ 1109 // _CALL_BUILTIN_CLASS 1110 args = &stack_pointer[-oparg]; 1111 self_or_null = stack_pointer[-1 - oparg]; 1112 callable = stack_pointer[-2 - oparg]; 1113 { 1114 int total_args = oparg; 1115 if (self_or_null != NULL) { 1116 args--; 1117 total_args++; 1118 } 1119 DEOPT_IF(!PyType_Check(callable), CALL); 1120 PyTypeObject *tp = (PyTypeObject *)callable; 1121 DEOPT_IF(tp->tp_vectorcall == NULL, CALL); 1122 STAT_INC(CALL, hit); 1123 res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); 1124 /* Free the arguments. */ 1125 for (int i = 0; i < total_args; i++) { 1126 Py_DECREF(args[i]); 1127 } 1128 Py_DECREF(tp); 1129 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 1130 } 1131 // _CHECK_PERIODIC 1132 { 1133 } 1134 stack_pointer[-2 - oparg] = res; 1135 stack_pointer += -1 - oparg; 1136 CHECK_EVAL_BREAKER(); 1137 DISPATCH(); 1138 } 1139 TARGET(CALL_BUILTIN_FAST)1140 TARGET(CALL_BUILTIN_FAST) { 1141 frame->instr_ptr = next_instr; 1142 next_instr += 4; 1143 INSTRUCTION_STATS(CALL_BUILTIN_FAST); 1144 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1145 PyObject **args; 1146 PyObject *self_or_null; 1147 PyObject *callable; 1148 PyObject *res; 1149 /* Skip 1 cache entry */ 1150 /* Skip 2 cache entries */ 1151 // _CALL_BUILTIN_FAST 1152 args = &stack_pointer[-oparg]; 1153 self_or_null = stack_pointer[-1 - oparg]; 1154 callable = stack_pointer[-2 - oparg]; 1155 { 1156 /* Builtin METH_FASTCALL functions, without keywords */ 1157 int total_args = oparg; 1158 if (self_or_null != NULL) { 1159 args--; 1160 total_args++; 1161 } 1162 DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); 1163 DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); 1164 STAT_INC(CALL, hit); 1165 PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); 1166 /* res = func(self, args, nargs) */ 1167 res = ((PyCFunctionFast)(void(*)(void))cfunc)( 1168 PyCFunction_GET_SELF(callable), 1169 args, 1170 total_args); 1171 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1172 /* Free the arguments. */ 1173 for (int i = 0; i < total_args; i++) { 1174 Py_DECREF(args[i]); 1175 } 1176 Py_DECREF(callable); 1177 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 1178 } 1179 // _CHECK_PERIODIC 1180 { 1181 } 1182 stack_pointer[-2 - oparg] = res; 1183 stack_pointer += -1 - oparg; 1184 CHECK_EVAL_BREAKER(); 1185 DISPATCH(); 1186 } 1187 TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS)1188 TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { 1189 frame->instr_ptr = next_instr; 1190 next_instr += 4; 1191 INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); 1192 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1193 PyObject **args; 1194 PyObject *self_or_null; 1195 PyObject *callable; 1196 PyObject *res; 1197 /* Skip 1 cache entry */ 1198 /* Skip 2 cache entries */ 1199 // _CALL_BUILTIN_FAST_WITH_KEYWORDS 1200 args = &stack_pointer[-oparg]; 1201 self_or_null = stack_pointer[-1 - oparg]; 1202 callable = stack_pointer[-2 - oparg]; 1203 { 1204 /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ 1205 int total_args = oparg; 1206 if (self_or_null != NULL) { 1207 args--; 1208 total_args++; 1209 } 1210 DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); 1211 DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); 1212 STAT_INC(CALL, hit); 1213 /* res = func(self, args, nargs, kwnames) */ 1214 PyCFunctionFastWithKeywords cfunc = 1215 (PyCFunctionFastWithKeywords)(void(*)(void)) 1216 PyCFunction_GET_FUNCTION(callable); 1217 res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); 1218 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1219 /* Free the arguments. */ 1220 for (int i = 0; i < total_args; i++) { 1221 Py_DECREF(args[i]); 1222 } 1223 Py_DECREF(callable); 1224 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 1225 } 1226 // _CHECK_PERIODIC 1227 { 1228 } 1229 stack_pointer[-2 - oparg] = res; 1230 stack_pointer += -1 - oparg; 1231 CHECK_EVAL_BREAKER(); 1232 DISPATCH(); 1233 } 1234 TARGET(CALL_BUILTIN_O)1235 TARGET(CALL_BUILTIN_O) { 1236 frame->instr_ptr = next_instr; 1237 next_instr += 4; 1238 INSTRUCTION_STATS(CALL_BUILTIN_O); 1239 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1240 PyObject **args; 1241 PyObject *self_or_null; 1242 PyObject *callable; 1243 PyObject *res; 1244 /* Skip 1 cache entry */ 1245 /* Skip 2 cache entries */ 1246 // _CALL_BUILTIN_O 1247 args = &stack_pointer[-oparg]; 1248 self_or_null = stack_pointer[-1 - oparg]; 1249 callable = stack_pointer[-2 - oparg]; 1250 { 1251 /* Builtin METH_O functions */ 1252 int total_args = oparg; 1253 if (self_or_null != NULL) { 1254 args--; 1255 total_args++; 1256 } 1257 DEOPT_IF(total_args != 1, CALL); 1258 DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); 1259 DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); 1260 // CPython promises to check all non-vectorcall function calls. 1261 DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); 1262 STAT_INC(CALL, hit); 1263 PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); 1264 PyObject *arg = args[0]; 1265 _Py_EnterRecursiveCallTstateUnchecked(tstate); 1266 res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); 1267 _Py_LeaveRecursiveCallTstate(tstate); 1268 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1269 Py_DECREF(arg); 1270 Py_DECREF(callable); 1271 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 1272 } 1273 // _CHECK_PERIODIC 1274 { 1275 } 1276 stack_pointer[-2 - oparg] = res; 1277 stack_pointer += -1 - oparg; 1278 CHECK_EVAL_BREAKER(); 1279 DISPATCH(); 1280 } 1281 TARGET(CALL_FUNCTION_EX)1282 TARGET(CALL_FUNCTION_EX) { 1283 frame->instr_ptr = next_instr; 1284 next_instr += 1; 1285 INSTRUCTION_STATS(CALL_FUNCTION_EX); 1286 PREDICTED(CALL_FUNCTION_EX); 1287 _Py_CODEUNIT *this_instr = next_instr - 1; 1288 (void)this_instr; 1289 PyObject *kwargs = NULL; 1290 PyObject *callargs; 1291 PyObject *func; 1292 PyObject *result; 1293 if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1)]; } 1294 callargs = stack_pointer[-1 - (oparg & 1)]; 1295 func = stack_pointer[-3 - (oparg & 1)]; 1296 // DICT_MERGE is called before this opcode if there are kwargs. 1297 // It converts all dict subtypes in kwargs into regular dicts. 1298 assert(kwargs == NULL || PyDict_CheckExact(kwargs)); 1299 if (!PyTuple_CheckExact(callargs)) { 1300 if (check_args_iterable(tstate, func, callargs) < 0) { 1301 goto error; 1302 } 1303 PyObject *tuple = PySequence_Tuple(callargs); 1304 if (tuple == NULL) { 1305 goto error; 1306 } 1307 Py_SETREF(callargs, tuple); 1308 } 1309 assert(PyTuple_CheckExact(callargs)); 1310 EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); 1311 if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { 1312 PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? 1313 PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; 1314 int err = _Py_call_instrumentation_2args( 1315 tstate, PY_MONITORING_EVENT_CALL, 1316 frame, this_instr, func, arg); 1317 if (err) goto error; 1318 result = PyObject_Call(func, callargs, kwargs); 1319 if (!PyFunction_Check(func) && !PyMethod_Check(func)) { 1320 if (result == NULL) { 1321 _Py_call_instrumentation_exc2( 1322 tstate, PY_MONITORING_EVENT_C_RAISE, 1323 frame, this_instr, func, arg); 1324 } 1325 else { 1326 int err = _Py_call_instrumentation_2args( 1327 tstate, PY_MONITORING_EVENT_C_RETURN, 1328 frame, this_instr, func, arg); 1329 if (err < 0) { 1330 Py_CLEAR(result); 1331 } 1332 } 1333 } 1334 } 1335 else { 1336 if (Py_TYPE(func) == &PyFunction_Type && 1337 tstate->interp->eval_frame == NULL && 1338 ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { 1339 assert(PyTuple_CheckExact(callargs)); 1340 Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); 1341 int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; 1342 PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); 1343 _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, 1344 (PyFunctionObject *)func, locals, 1345 nargs, callargs, kwargs); 1346 // Need to manually shrink the stack since we exit with DISPATCH_INLINED. 1347 STACK_SHRINK(oparg + 3); 1348 if (new_frame == NULL) { 1349 goto error; 1350 } 1351 assert(next_instr - this_instr == 1); 1352 frame->return_offset = 1; 1353 DISPATCH_INLINED(new_frame); 1354 } 1355 result = PyObject_Call(func, callargs, kwargs); 1356 } 1357 Py_DECREF(func); 1358 Py_DECREF(callargs); 1359 Py_XDECREF(kwargs); 1360 assert(PEEK(2 + (oparg & 1)) == NULL); 1361 if (result == NULL) { stack_pointer += -3 - (oparg & 1); goto error; } 1362 stack_pointer[-3 - (oparg & 1)] = result; 1363 stack_pointer += -2 - (oparg & 1); 1364 CHECK_EVAL_BREAKER(); 1365 DISPATCH(); 1366 } 1367 TARGET(CALL_INTRINSIC_1)1368 TARGET(CALL_INTRINSIC_1) { 1369 frame->instr_ptr = next_instr; 1370 next_instr += 1; 1371 INSTRUCTION_STATS(CALL_INTRINSIC_1); 1372 PyObject *value; 1373 PyObject *res; 1374 value = stack_pointer[-1]; 1375 assert(oparg <= MAX_INTRINSIC_1); 1376 res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); 1377 Py_DECREF(value); 1378 if (res == NULL) goto pop_1_error; 1379 stack_pointer[-1] = res; 1380 DISPATCH(); 1381 } 1382 TARGET(CALL_INTRINSIC_2)1383 TARGET(CALL_INTRINSIC_2) { 1384 frame->instr_ptr = next_instr; 1385 next_instr += 1; 1386 INSTRUCTION_STATS(CALL_INTRINSIC_2); 1387 PyObject *value1; 1388 PyObject *value2; 1389 PyObject *res; 1390 value1 = stack_pointer[-1]; 1391 value2 = stack_pointer[-2]; 1392 assert(oparg <= MAX_INTRINSIC_2); 1393 res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); 1394 Py_DECREF(value2); 1395 Py_DECREF(value1); 1396 if (res == NULL) goto pop_2_error; 1397 stack_pointer[-2] = res; 1398 stack_pointer += -1; 1399 DISPATCH(); 1400 } 1401 TARGET(CALL_ISINSTANCE)1402 TARGET(CALL_ISINSTANCE) { 1403 frame->instr_ptr = next_instr; 1404 next_instr += 4; 1405 INSTRUCTION_STATS(CALL_ISINSTANCE); 1406 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1407 PyObject **args; 1408 PyObject *self_or_null; 1409 PyObject *callable; 1410 PyObject *res; 1411 /* Skip 1 cache entry */ 1412 /* Skip 2 cache entries */ 1413 args = &stack_pointer[-oparg]; 1414 self_or_null = stack_pointer[-1 - oparg]; 1415 callable = stack_pointer[-2 - oparg]; 1416 /* isinstance(o, o2) */ 1417 int total_args = oparg; 1418 if (self_or_null != NULL) { 1419 args--; 1420 total_args++; 1421 } 1422 DEOPT_IF(total_args != 2, CALL); 1423 PyInterpreterState *interp = tstate->interp; 1424 DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); 1425 STAT_INC(CALL, hit); 1426 PyObject *cls = args[1]; 1427 PyObject *inst = args[0]; 1428 int retval = PyObject_IsInstance(inst, cls); 1429 if (retval < 0) { 1430 goto error; 1431 } 1432 res = PyBool_FromLong(retval); 1433 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1434 if (res == NULL) { 1435 GOTO_ERROR(error); 1436 } 1437 Py_DECREF(inst); 1438 Py_DECREF(cls); 1439 Py_DECREF(callable); 1440 stack_pointer[-2 - oparg] = res; 1441 stack_pointer += -1 - oparg; 1442 DISPATCH(); 1443 } 1444 TARGET(CALL_KW)1445 TARGET(CALL_KW) { 1446 frame->instr_ptr = next_instr; 1447 next_instr += 1; 1448 INSTRUCTION_STATS(CALL_KW); 1449 PREDICTED(CALL_KW); 1450 _Py_CODEUNIT *this_instr = next_instr - 1; 1451 (void)this_instr; 1452 PyObject *kwnames; 1453 PyObject **args; 1454 PyObject *self_or_null; 1455 PyObject *callable; 1456 PyObject *res; 1457 kwnames = stack_pointer[-1]; 1458 args = &stack_pointer[-1 - oparg]; 1459 self_or_null = stack_pointer[-2 - oparg]; 1460 callable = stack_pointer[-3 - oparg]; 1461 // oparg counts all of the args, but *not* self: 1462 int total_args = oparg; 1463 if (self_or_null != NULL) { 1464 args--; 1465 total_args++; 1466 } 1467 if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { 1468 args--; 1469 total_args++; 1470 PyObject *self = ((PyMethodObject *)callable)->im_self; 1471 args[0] = Py_NewRef(self); 1472 PyObject *method = ((PyMethodObject *)callable)->im_func; 1473 args[-1] = Py_NewRef(method); 1474 Py_DECREF(callable); 1475 callable = method; 1476 } 1477 int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); 1478 // Check if the call can be inlined or not 1479 if (Py_TYPE(callable) == &PyFunction_Type && 1480 tstate->interp->eval_frame == NULL && 1481 ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) 1482 { 1483 int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; 1484 PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); 1485 _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( 1486 tstate, (PyFunctionObject *)callable, locals, 1487 args, positional_args, kwnames 1488 ); 1489 Py_DECREF(kwnames); 1490 // Manipulate stack directly since we leave using DISPATCH_INLINED(). 1491 STACK_SHRINK(oparg + 3); 1492 // The frame has stolen all the arguments from the stack, 1493 // so there is no need to clean them up. 1494 if (new_frame == NULL) { 1495 goto error; 1496 } 1497 assert(next_instr - this_instr == 1); 1498 frame->return_offset = 1; 1499 DISPATCH_INLINED(new_frame); 1500 } 1501 /* Callable is not a normal Python function */ 1502 res = PyObject_Vectorcall( 1503 callable, args, 1504 positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, 1505 kwnames); 1506 if (opcode == INSTRUMENTED_CALL_KW) { 1507 PyObject *arg = total_args == 0 ? 1508 &_PyInstrumentation_MISSING : args[0]; 1509 if (res == NULL) { 1510 _Py_call_instrumentation_exc2( 1511 tstate, PY_MONITORING_EVENT_C_RAISE, 1512 frame, this_instr, callable, arg); 1513 } 1514 else { 1515 int err = _Py_call_instrumentation_2args( 1516 tstate, PY_MONITORING_EVENT_C_RETURN, 1517 frame, this_instr, callable, arg); 1518 if (err < 0) { 1519 Py_CLEAR(res); 1520 } 1521 } 1522 } 1523 Py_DECREF(kwnames); 1524 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1525 Py_DECREF(callable); 1526 for (int i = 0; i < total_args; i++) { 1527 Py_DECREF(args[i]); 1528 } 1529 if (res == NULL) { stack_pointer += -3 - oparg; goto error; } 1530 stack_pointer[-3 - oparg] = res; 1531 stack_pointer += -2 - oparg; 1532 CHECK_EVAL_BREAKER(); 1533 DISPATCH(); 1534 } 1535 TARGET(CALL_LEN)1536 TARGET(CALL_LEN) { 1537 frame->instr_ptr = next_instr; 1538 next_instr += 4; 1539 INSTRUCTION_STATS(CALL_LEN); 1540 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1541 PyObject **args; 1542 PyObject *self_or_null; 1543 PyObject *callable; 1544 PyObject *res; 1545 /* Skip 1 cache entry */ 1546 /* Skip 2 cache entries */ 1547 args = &stack_pointer[-oparg]; 1548 self_or_null = stack_pointer[-1 - oparg]; 1549 callable = stack_pointer[-2 - oparg]; 1550 /* len(o) */ 1551 int total_args = oparg; 1552 if (self_or_null != NULL) { 1553 args--; 1554 total_args++; 1555 } 1556 DEOPT_IF(total_args != 1, CALL); 1557 PyInterpreterState *interp = tstate->interp; 1558 DEOPT_IF(callable != interp->callable_cache.len, CALL); 1559 STAT_INC(CALL, hit); 1560 PyObject *arg = args[0]; 1561 Py_ssize_t len_i = PyObject_Length(arg); 1562 if (len_i < 0) { 1563 goto error; 1564 } 1565 res = PyLong_FromSsize_t(len_i); 1566 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1567 if (res == NULL) { 1568 GOTO_ERROR(error); 1569 } 1570 Py_DECREF(callable); 1571 Py_DECREF(arg); 1572 stack_pointer[-2 - oparg] = res; 1573 stack_pointer += -1 - oparg; 1574 DISPATCH(); 1575 } 1576 TARGET(CALL_LIST_APPEND)1577 TARGET(CALL_LIST_APPEND) { 1578 frame->instr_ptr = next_instr; 1579 next_instr += 4; 1580 INSTRUCTION_STATS(CALL_LIST_APPEND); 1581 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1582 PyObject *arg; 1583 PyObject *self; 1584 PyObject *callable; 1585 /* Skip 1 cache entry */ 1586 /* Skip 2 cache entries */ 1587 arg = stack_pointer[-1]; 1588 self = stack_pointer[-2]; 1589 callable = stack_pointer[-3]; 1590 assert(oparg == 1); 1591 PyInterpreterState *interp = tstate->interp; 1592 DEOPT_IF(callable != interp->callable_cache.list_append, CALL); 1593 assert(self != NULL); 1594 DEOPT_IF(!PyList_Check(self), CALL); 1595 STAT_INC(CALL, hit); 1596 if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { 1597 goto pop_1_error; // Since arg is DECREF'ed already 1598 } 1599 Py_DECREF(self); 1600 Py_DECREF(callable); 1601 STACK_SHRINK(3); 1602 // Skip POP_TOP 1603 assert(next_instr->op.code == POP_TOP); 1604 SKIP_OVER(1); 1605 DISPATCH(); 1606 } 1607 TARGET(CALL_METHOD_DESCRIPTOR_FAST)1608 TARGET(CALL_METHOD_DESCRIPTOR_FAST) { 1609 frame->instr_ptr = next_instr; 1610 next_instr += 4; 1611 INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); 1612 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1613 PyObject **args; 1614 PyObject *self_or_null; 1615 PyObject *callable; 1616 PyObject *res; 1617 /* Skip 1 cache entry */ 1618 /* Skip 2 cache entries */ 1619 // _CALL_METHOD_DESCRIPTOR_FAST 1620 args = &stack_pointer[-oparg]; 1621 self_or_null = stack_pointer[-1 - oparg]; 1622 callable = stack_pointer[-2 - oparg]; 1623 { 1624 int total_args = oparg; 1625 if (self_or_null != NULL) { 1626 args--; 1627 total_args++; 1628 } 1629 PyMethodDescrObject *method = (PyMethodDescrObject *)callable; 1630 /* Builtin METH_FASTCALL methods, without keywords */ 1631 DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); 1632 PyMethodDef *meth = method->d_method; 1633 DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); 1634 PyObject *self = args[0]; 1635 DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); 1636 STAT_INC(CALL, hit); 1637 PyCFunctionFast cfunc = 1638 (PyCFunctionFast)(void(*)(void))meth->ml_meth; 1639 int nargs = total_args - 1; 1640 res = cfunc(self, args + 1, nargs); 1641 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1642 /* Clear the stack of the arguments. */ 1643 for (int i = 0; i < total_args; i++) { 1644 Py_DECREF(args[i]); 1645 } 1646 Py_DECREF(callable); 1647 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 1648 } 1649 // _CHECK_PERIODIC 1650 { 1651 } 1652 stack_pointer[-2 - oparg] = res; 1653 stack_pointer += -1 - oparg; 1654 CHECK_EVAL_BREAKER(); 1655 DISPATCH(); 1656 } 1657 TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS)1658 TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { 1659 frame->instr_ptr = next_instr; 1660 next_instr += 4; 1661 INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); 1662 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1663 PyObject **args; 1664 PyObject *self_or_null; 1665 PyObject *callable; 1666 PyObject *res; 1667 /* Skip 1 cache entry */ 1668 /* Skip 2 cache entries */ 1669 // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 1670 args = &stack_pointer[-oparg]; 1671 self_or_null = stack_pointer[-1 - oparg]; 1672 callable = stack_pointer[-2 - oparg]; 1673 { 1674 int total_args = oparg; 1675 if (self_or_null != NULL) { 1676 args--; 1677 total_args++; 1678 } 1679 PyMethodDescrObject *method = (PyMethodDescrObject *)callable; 1680 DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); 1681 PyMethodDef *meth = method->d_method; 1682 DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); 1683 PyTypeObject *d_type = method->d_common.d_type; 1684 PyObject *self = args[0]; 1685 DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); 1686 STAT_INC(CALL, hit); 1687 int nargs = total_args - 1; 1688 PyCFunctionFastWithKeywords cfunc = 1689 (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; 1690 res = cfunc(self, args + 1, nargs, NULL); 1691 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1692 /* Free the arguments. */ 1693 for (int i = 0; i < total_args; i++) { 1694 Py_DECREF(args[i]); 1695 } 1696 Py_DECREF(callable); 1697 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 1698 } 1699 // _CHECK_PERIODIC 1700 { 1701 } 1702 stack_pointer[-2 - oparg] = res; 1703 stack_pointer += -1 - oparg; 1704 CHECK_EVAL_BREAKER(); 1705 DISPATCH(); 1706 } 1707 TARGET(CALL_METHOD_DESCRIPTOR_NOARGS)1708 TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { 1709 frame->instr_ptr = next_instr; 1710 next_instr += 4; 1711 INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); 1712 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1713 PyObject **args; 1714 PyObject *self_or_null; 1715 PyObject *callable; 1716 PyObject *res; 1717 /* Skip 1 cache entry */ 1718 /* Skip 2 cache entries */ 1719 // _CALL_METHOD_DESCRIPTOR_NOARGS 1720 args = &stack_pointer[-oparg]; 1721 self_or_null = stack_pointer[-1 - oparg]; 1722 callable = stack_pointer[-2 - oparg]; 1723 { 1724 assert(oparg == 0 || oparg == 1); 1725 int total_args = oparg; 1726 if (self_or_null != NULL) { 1727 args--; 1728 total_args++; 1729 } 1730 DEOPT_IF(total_args != 1, CALL); 1731 PyMethodDescrObject *method = (PyMethodDescrObject *)callable; 1732 DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); 1733 PyMethodDef *meth = method->d_method; 1734 PyObject *self = args[0]; 1735 DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); 1736 DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); 1737 // CPython promises to check all non-vectorcall function calls. 1738 DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); 1739 STAT_INC(CALL, hit); 1740 PyCFunction cfunc = meth->ml_meth; 1741 _Py_EnterRecursiveCallTstateUnchecked(tstate); 1742 res = _PyCFunction_TrampolineCall(cfunc, self, NULL); 1743 _Py_LeaveRecursiveCallTstate(tstate); 1744 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1745 Py_DECREF(self); 1746 Py_DECREF(callable); 1747 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 1748 } 1749 // _CHECK_PERIODIC 1750 { 1751 } 1752 stack_pointer[-2 - oparg] = res; 1753 stack_pointer += -1 - oparg; 1754 CHECK_EVAL_BREAKER(); 1755 DISPATCH(); 1756 } 1757 TARGET(CALL_METHOD_DESCRIPTOR_O)1758 TARGET(CALL_METHOD_DESCRIPTOR_O) { 1759 frame->instr_ptr = next_instr; 1760 next_instr += 4; 1761 INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); 1762 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1763 PyObject **args; 1764 PyObject *self_or_null; 1765 PyObject *callable; 1766 PyObject *res; 1767 /* Skip 1 cache entry */ 1768 /* Skip 2 cache entries */ 1769 // _CALL_METHOD_DESCRIPTOR_O 1770 args = &stack_pointer[-oparg]; 1771 self_or_null = stack_pointer[-1 - oparg]; 1772 callable = stack_pointer[-2 - oparg]; 1773 { 1774 int total_args = oparg; 1775 if (self_or_null != NULL) { 1776 args--; 1777 total_args++; 1778 } 1779 PyMethodDescrObject *method = (PyMethodDescrObject *)callable; 1780 DEOPT_IF(total_args != 2, CALL); 1781 DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); 1782 PyMethodDef *meth = method->d_method; 1783 DEOPT_IF(meth->ml_flags != METH_O, CALL); 1784 // CPython promises to check all non-vectorcall function calls. 1785 DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); 1786 PyObject *arg = args[1]; 1787 PyObject *self = args[0]; 1788 DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); 1789 STAT_INC(CALL, hit); 1790 PyCFunction cfunc = meth->ml_meth; 1791 _Py_EnterRecursiveCallTstateUnchecked(tstate); 1792 res = _PyCFunction_TrampolineCall(cfunc, self, arg); 1793 _Py_LeaveRecursiveCallTstate(tstate); 1794 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1795 Py_DECREF(self); 1796 Py_DECREF(arg); 1797 Py_DECREF(callable); 1798 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 1799 } 1800 // _CHECK_PERIODIC 1801 { 1802 } 1803 stack_pointer[-2 - oparg] = res; 1804 stack_pointer += -1 - oparg; 1805 CHECK_EVAL_BREAKER(); 1806 DISPATCH(); 1807 } 1808 TARGET(CALL_NON_PY_GENERAL)1809 TARGET(CALL_NON_PY_GENERAL) { 1810 frame->instr_ptr = next_instr; 1811 next_instr += 4; 1812 INSTRUCTION_STATS(CALL_NON_PY_GENERAL); 1813 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1814 PyObject *callable; 1815 PyObject **args; 1816 PyObject *self_or_null; 1817 PyObject *res; 1818 /* Skip 1 cache entry */ 1819 /* Skip 2 cache entries */ 1820 // _CHECK_IS_NOT_PY_CALLABLE 1821 callable = stack_pointer[-2 - oparg]; 1822 { 1823 DEOPT_IF(PyFunction_Check(callable), CALL); 1824 DEOPT_IF(Py_TYPE(callable) == &PyMethod_Type, CALL); 1825 } 1826 // _CALL_NON_PY_GENERAL 1827 args = &stack_pointer[-oparg]; 1828 self_or_null = stack_pointer[-1 - oparg]; 1829 { 1830 #if TIER_ONE 1831 assert(opcode != INSTRUMENTED_CALL); 1832 #endif 1833 int total_args = oparg; 1834 if (self_or_null != NULL) { 1835 args--; 1836 total_args++; 1837 } 1838 /* Callable is not a normal Python function */ 1839 res = PyObject_Vectorcall( 1840 callable, args, 1841 total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, 1842 NULL); 1843 assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); 1844 Py_DECREF(callable); 1845 for (int i = 0; i < total_args; i++) { 1846 Py_DECREF(args[i]); 1847 } 1848 if (res == NULL) { stack_pointer += -2 - oparg; goto error; } 1849 } 1850 // _CHECK_PERIODIC 1851 { 1852 } 1853 stack_pointer[-2 - oparg] = res; 1854 stack_pointer += -1 - oparg; 1855 CHECK_EVAL_BREAKER(); 1856 DISPATCH(); 1857 } 1858 TARGET(CALL_PY_EXACT_ARGS)1859 TARGET(CALL_PY_EXACT_ARGS) { 1860 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 1861 next_instr += 4; 1862 INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); 1863 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1864 PyObject *self_or_null; 1865 PyObject *callable; 1866 PyObject **args; 1867 _PyInterpreterFrame *new_frame; 1868 /* Skip 1 cache entry */ 1869 // _CHECK_PEP_523 1870 { 1871 DEOPT_IF(tstate->interp->eval_frame, CALL); 1872 } 1873 // _CHECK_FUNCTION_EXACT_ARGS 1874 self_or_null = stack_pointer[-1 - oparg]; 1875 callable = stack_pointer[-2 - oparg]; 1876 { 1877 uint32_t func_version = read_u32(&this_instr[2].cache); 1878 DEOPT_IF(!PyFunction_Check(callable), CALL); 1879 PyFunctionObject *func = (PyFunctionObject *)callable; 1880 DEOPT_IF(func->func_version != func_version, CALL); 1881 PyCodeObject *code = (PyCodeObject *)func->func_code; 1882 DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); 1883 } 1884 // _CHECK_STACK_SPACE 1885 { 1886 PyFunctionObject *func = (PyFunctionObject *)callable; 1887 PyCodeObject *code = (PyCodeObject *)func->func_code; 1888 DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); 1889 DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); 1890 } 1891 // _INIT_CALL_PY_EXACT_ARGS 1892 args = &stack_pointer[-oparg]; 1893 self_or_null = stack_pointer[-1 - oparg]; 1894 { 1895 int has_self = (self_or_null != NULL); 1896 STAT_INC(CALL, hit); 1897 PyFunctionObject *func = (PyFunctionObject *)callable; 1898 new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); 1899 PyObject **first_non_self_local = new_frame->localsplus + has_self; 1900 new_frame->localsplus[0] = self_or_null; 1901 for (int i = 0; i < oparg; i++) { 1902 first_non_self_local[i] = args[i]; 1903 } 1904 } 1905 // _SAVE_RETURN_OFFSET 1906 { 1907 #if TIER_ONE 1908 frame->return_offset = (uint16_t)(next_instr - this_instr); 1909 #endif 1910 #if TIER_TWO 1911 frame->return_offset = oparg; 1912 #endif 1913 } 1914 // _PUSH_FRAME 1915 { 1916 // Write it out explicitly because it's subtly different. 1917 // Eventually this should be the only occurrence of this code. 1918 assert(tstate->interp->eval_frame == NULL); 1919 stack_pointer += -2 - oparg; 1920 _PyFrame_SetStackPointer(frame, stack_pointer); 1921 new_frame->previous = frame; 1922 CALL_STAT_INC(inlined_py_calls); 1923 frame = tstate->current_frame = new_frame; 1924 tstate->py_recursion_remaining--; 1925 LOAD_SP(); 1926 LOAD_IP(0); 1927 LLTRACE_RESUME_FRAME(); 1928 } 1929 DISPATCH(); 1930 } 1931 TARGET(CALL_PY_GENERAL)1932 TARGET(CALL_PY_GENERAL) { 1933 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 1934 next_instr += 4; 1935 INSTRUCTION_STATS(CALL_PY_GENERAL); 1936 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 1937 PyObject *callable; 1938 PyObject **args; 1939 PyObject *self_or_null; 1940 _PyInterpreterFrame *new_frame; 1941 /* Skip 1 cache entry */ 1942 // _CHECK_PEP_523 1943 { 1944 DEOPT_IF(tstate->interp->eval_frame, CALL); 1945 } 1946 // _CHECK_FUNCTION_VERSION 1947 callable = stack_pointer[-2 - oparg]; 1948 { 1949 uint32_t func_version = read_u32(&this_instr[2].cache); 1950 DEOPT_IF(!PyFunction_Check(callable), CALL); 1951 PyFunctionObject *func = (PyFunctionObject *)callable; 1952 DEOPT_IF(func->func_version != func_version, CALL); 1953 } 1954 // _PY_FRAME_GENERAL 1955 args = &stack_pointer[-oparg]; 1956 self_or_null = stack_pointer[-1 - oparg]; 1957 { 1958 // oparg counts all of the args, but *not* self: 1959 int total_args = oparg; 1960 if (self_or_null != NULL) { 1961 args--; 1962 total_args++; 1963 } 1964 assert(Py_TYPE(callable) == &PyFunction_Type); 1965 int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; 1966 PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); 1967 new_frame = _PyEvalFramePushAndInit( 1968 tstate, (PyFunctionObject *)callable, locals, 1969 args, total_args, NULL 1970 ); 1971 // The frame has stolen all the arguments from the stack, 1972 // so there is no need to clean them up. 1973 stack_pointer += -2 - oparg; 1974 if (new_frame == NULL) { 1975 goto error; 1976 } 1977 } 1978 // _SAVE_RETURN_OFFSET 1979 { 1980 #if TIER_ONE 1981 frame->return_offset = (uint16_t)(next_instr - this_instr); 1982 #endif 1983 #if TIER_TWO 1984 frame->return_offset = oparg; 1985 #endif 1986 } 1987 // _PUSH_FRAME 1988 { 1989 // Write it out explicitly because it's subtly different. 1990 // Eventually this should be the only occurrence of this code. 1991 assert(tstate->interp->eval_frame == NULL); 1992 _PyFrame_SetStackPointer(frame, stack_pointer); 1993 new_frame->previous = frame; 1994 CALL_STAT_INC(inlined_py_calls); 1995 frame = tstate->current_frame = new_frame; 1996 tstate->py_recursion_remaining--; 1997 LOAD_SP(); 1998 LOAD_IP(0); 1999 LLTRACE_RESUME_FRAME(); 2000 } 2001 DISPATCH(); 2002 } 2003 TARGET(CALL_STR_1)2004 TARGET(CALL_STR_1) { 2005 frame->instr_ptr = next_instr; 2006 next_instr += 4; 2007 INSTRUCTION_STATS(CALL_STR_1); 2008 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 2009 PyObject *arg; 2010 PyObject *null; 2011 PyObject *callable; 2012 PyObject *res; 2013 /* Skip 1 cache entry */ 2014 /* Skip 2 cache entries */ 2015 // _CALL_STR_1 2016 arg = stack_pointer[-1]; 2017 null = stack_pointer[-2]; 2018 callable = stack_pointer[-3]; 2019 { 2020 assert(oparg == 1); 2021 DEOPT_IF(null != NULL, CALL); 2022 DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); 2023 STAT_INC(CALL, hit); 2024 res = PyObject_Str(arg); 2025 Py_DECREF(arg); 2026 if (res == NULL) goto pop_3_error; 2027 } 2028 // _CHECK_PERIODIC 2029 { 2030 } 2031 stack_pointer[-3] = res; 2032 stack_pointer += -2; 2033 CHECK_EVAL_BREAKER(); 2034 DISPATCH(); 2035 } 2036 TARGET(CALL_TUPLE_1)2037 TARGET(CALL_TUPLE_1) { 2038 frame->instr_ptr = next_instr; 2039 next_instr += 4; 2040 INSTRUCTION_STATS(CALL_TUPLE_1); 2041 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 2042 PyObject *arg; 2043 PyObject *null; 2044 PyObject *callable; 2045 PyObject *res; 2046 /* Skip 1 cache entry */ 2047 /* Skip 2 cache entries */ 2048 // _CALL_TUPLE_1 2049 arg = stack_pointer[-1]; 2050 null = stack_pointer[-2]; 2051 callable = stack_pointer[-3]; 2052 { 2053 assert(oparg == 1); 2054 DEOPT_IF(null != NULL, CALL); 2055 DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); 2056 STAT_INC(CALL, hit); 2057 res = PySequence_Tuple(arg); 2058 Py_DECREF(arg); 2059 if (res == NULL) goto pop_3_error; 2060 } 2061 // _CHECK_PERIODIC 2062 { 2063 } 2064 stack_pointer[-3] = res; 2065 stack_pointer += -2; 2066 CHECK_EVAL_BREAKER(); 2067 DISPATCH(); 2068 } 2069 TARGET(CALL_TYPE_1)2070 TARGET(CALL_TYPE_1) { 2071 frame->instr_ptr = next_instr; 2072 next_instr += 4; 2073 INSTRUCTION_STATS(CALL_TYPE_1); 2074 static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); 2075 PyObject *arg; 2076 PyObject *null; 2077 PyObject *callable; 2078 PyObject *res; 2079 /* Skip 1 cache entry */ 2080 /* Skip 2 cache entries */ 2081 arg = stack_pointer[-1]; 2082 null = stack_pointer[-2]; 2083 callable = stack_pointer[-3]; 2084 assert(oparg == 1); 2085 DEOPT_IF(null != NULL, CALL); 2086 DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); 2087 STAT_INC(CALL, hit); 2088 res = Py_NewRef(Py_TYPE(arg)); 2089 Py_DECREF(arg); 2090 stack_pointer[-3] = res; 2091 stack_pointer += -2; 2092 DISPATCH(); 2093 } 2094 TARGET(CHECK_EG_MATCH)2095 TARGET(CHECK_EG_MATCH) { 2096 frame->instr_ptr = next_instr; 2097 next_instr += 1; 2098 INSTRUCTION_STATS(CHECK_EG_MATCH); 2099 PyObject *match_type; 2100 PyObject *exc_value; 2101 PyObject *rest; 2102 PyObject *match; 2103 match_type = stack_pointer[-1]; 2104 exc_value = stack_pointer[-2]; 2105 if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { 2106 Py_DECREF(exc_value); 2107 Py_DECREF(match_type); 2108 if (true) goto pop_2_error; 2109 } 2110 match = NULL; 2111 rest = NULL; 2112 int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, 2113 &match, &rest); 2114 Py_DECREF(exc_value); 2115 Py_DECREF(match_type); 2116 if (res < 0) goto pop_2_error; 2117 assert((match == NULL) == (rest == NULL)); 2118 if (match == NULL) goto pop_2_error; 2119 if (!Py_IsNone(match)) { 2120 PyErr_SetHandledException(match); 2121 } 2122 stack_pointer[-2] = rest; 2123 stack_pointer[-1] = match; 2124 DISPATCH(); 2125 } 2126 TARGET(CHECK_EXC_MATCH)2127 TARGET(CHECK_EXC_MATCH) { 2128 frame->instr_ptr = next_instr; 2129 next_instr += 1; 2130 INSTRUCTION_STATS(CHECK_EXC_MATCH); 2131 PyObject *right; 2132 PyObject *left; 2133 PyObject *b; 2134 right = stack_pointer[-1]; 2135 left = stack_pointer[-2]; 2136 assert(PyExceptionInstance_Check(left)); 2137 if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { 2138 Py_DECREF(right); 2139 if (true) goto pop_1_error; 2140 } 2141 int res = PyErr_GivenExceptionMatches(left, right); 2142 Py_DECREF(right); 2143 b = res ? Py_True : Py_False; 2144 stack_pointer[-1] = b; 2145 DISPATCH(); 2146 } 2147 TARGET(CLEANUP_THROW)2148 TARGET(CLEANUP_THROW) { 2149 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 2150 (void)this_instr; 2151 next_instr += 1; 2152 INSTRUCTION_STATS(CLEANUP_THROW); 2153 PyObject *exc_value; 2154 PyObject *last_sent_val; 2155 PyObject *sub_iter; 2156 PyObject *none; 2157 PyObject *value; 2158 exc_value = stack_pointer[-1]; 2159 last_sent_val = stack_pointer[-2]; 2160 sub_iter = stack_pointer[-3]; 2161 assert(throwflag); 2162 assert(exc_value && PyExceptionInstance_Check(exc_value)); 2163 if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { 2164 value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); 2165 Py_DECREF(sub_iter); 2166 Py_DECREF(last_sent_val); 2167 Py_DECREF(exc_value); 2168 none = Py_None; 2169 } 2170 else { 2171 _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); 2172 monitor_reraise(tstate, frame, this_instr); 2173 goto exception_unwind; 2174 } 2175 stack_pointer[-3] = none; 2176 stack_pointer[-2] = value; 2177 stack_pointer += -1; 2178 DISPATCH(); 2179 } 2180 TARGET(COMPARE_OP)2181 TARGET(COMPARE_OP) { 2182 frame->instr_ptr = next_instr; 2183 next_instr += 2; 2184 INSTRUCTION_STATS(COMPARE_OP); 2185 PREDICTED(COMPARE_OP); 2186 _Py_CODEUNIT *this_instr = next_instr - 2; 2187 (void)this_instr; 2188 PyObject *right; 2189 PyObject *left; 2190 PyObject *res; 2191 // _SPECIALIZE_COMPARE_OP 2192 right = stack_pointer[-1]; 2193 left = stack_pointer[-2]; 2194 { 2195 uint16_t counter = read_u16(&this_instr[1].cache); 2196 (void)counter; 2197 #if ENABLE_SPECIALIZATION 2198 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 2199 next_instr = this_instr; 2200 _Py_Specialize_CompareOp(left, right, next_instr, oparg); 2201 DISPATCH_SAME_OPARG(); 2202 } 2203 STAT_INC(COMPARE_OP, deferred); 2204 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 2205 #endif /* ENABLE_SPECIALIZATION */ 2206 } 2207 // _COMPARE_OP 2208 { 2209 assert((oparg >> 5) <= Py_GE); 2210 res = PyObject_RichCompare(left, right, oparg >> 5); 2211 Py_DECREF(left); 2212 Py_DECREF(right); 2213 if (res == NULL) goto pop_2_error; 2214 if (oparg & 16) { 2215 int res_bool = PyObject_IsTrue(res); 2216 Py_DECREF(res); 2217 if (res_bool < 0) goto pop_2_error; 2218 res = res_bool ? Py_True : Py_False; 2219 } 2220 } 2221 stack_pointer[-2] = res; 2222 stack_pointer += -1; 2223 DISPATCH(); 2224 } 2225 TARGET(COMPARE_OP_FLOAT)2226 TARGET(COMPARE_OP_FLOAT) { 2227 frame->instr_ptr = next_instr; 2228 next_instr += 2; 2229 INSTRUCTION_STATS(COMPARE_OP_FLOAT); 2230 static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); 2231 PyObject *right; 2232 PyObject *left; 2233 PyObject *res; 2234 // _GUARD_BOTH_FLOAT 2235 right = stack_pointer[-1]; 2236 left = stack_pointer[-2]; 2237 { 2238 DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); 2239 DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); 2240 } 2241 /* Skip 1 cache entry */ 2242 // _COMPARE_OP_FLOAT 2243 { 2244 STAT_INC(COMPARE_OP, hit); 2245 double dleft = PyFloat_AS_DOUBLE(left); 2246 double dright = PyFloat_AS_DOUBLE(right); 2247 // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg 2248 int sign_ish = COMPARISON_BIT(dleft, dright); 2249 _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); 2250 _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); 2251 res = (sign_ish & oparg) ? Py_True : Py_False; 2252 // It's always a bool, so we don't care about oparg & 16. 2253 } 2254 stack_pointer[-2] = res; 2255 stack_pointer += -1; 2256 DISPATCH(); 2257 } 2258 TARGET(COMPARE_OP_INT)2259 TARGET(COMPARE_OP_INT) { 2260 frame->instr_ptr = next_instr; 2261 next_instr += 2; 2262 INSTRUCTION_STATS(COMPARE_OP_INT); 2263 static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); 2264 PyObject *right; 2265 PyObject *left; 2266 PyObject *res; 2267 // _GUARD_BOTH_INT 2268 right = stack_pointer[-1]; 2269 left = stack_pointer[-2]; 2270 { 2271 DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); 2272 DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); 2273 } 2274 /* Skip 1 cache entry */ 2275 // _COMPARE_OP_INT 2276 { 2277 DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); 2278 DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); 2279 STAT_INC(COMPARE_OP, hit); 2280 assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && 2281 _PyLong_DigitCount((PyLongObject *)right) <= 1); 2282 Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); 2283 Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); 2284 // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg 2285 int sign_ish = COMPARISON_BIT(ileft, iright); 2286 _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); 2287 _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); 2288 res = (sign_ish & oparg) ? Py_True : Py_False; 2289 // It's always a bool, so we don't care about oparg & 16. 2290 } 2291 stack_pointer[-2] = res; 2292 stack_pointer += -1; 2293 DISPATCH(); 2294 } 2295 TARGET(COMPARE_OP_STR)2296 TARGET(COMPARE_OP_STR) { 2297 frame->instr_ptr = next_instr; 2298 next_instr += 2; 2299 INSTRUCTION_STATS(COMPARE_OP_STR); 2300 static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); 2301 PyObject *right; 2302 PyObject *left; 2303 PyObject *res; 2304 // _GUARD_BOTH_UNICODE 2305 right = stack_pointer[-1]; 2306 left = stack_pointer[-2]; 2307 { 2308 DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); 2309 DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); 2310 } 2311 /* Skip 1 cache entry */ 2312 // _COMPARE_OP_STR 2313 { 2314 STAT_INC(COMPARE_OP, hit); 2315 int eq = _PyUnicode_Equal(left, right); 2316 assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); 2317 _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); 2318 _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); 2319 assert(eq == 0 || eq == 1); 2320 assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); 2321 assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); 2322 res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; 2323 // It's always a bool, so we don't care about oparg & 16. 2324 } 2325 stack_pointer[-2] = res; 2326 stack_pointer += -1; 2327 DISPATCH(); 2328 } 2329 TARGET(CONTAINS_OP)2330 TARGET(CONTAINS_OP) { 2331 frame->instr_ptr = next_instr; 2332 next_instr += 2; 2333 INSTRUCTION_STATS(CONTAINS_OP); 2334 PREDICTED(CONTAINS_OP); 2335 _Py_CODEUNIT *this_instr = next_instr - 2; 2336 (void)this_instr; 2337 PyObject *right; 2338 PyObject *left; 2339 PyObject *b; 2340 // _SPECIALIZE_CONTAINS_OP 2341 right = stack_pointer[-1]; 2342 left = stack_pointer[-2]; 2343 { 2344 uint16_t counter = read_u16(&this_instr[1].cache); 2345 (void)counter; 2346 #if ENABLE_SPECIALIZATION 2347 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 2348 next_instr = this_instr; 2349 _Py_Specialize_ContainsOp(right, next_instr); 2350 DISPATCH_SAME_OPARG(); 2351 } 2352 STAT_INC(CONTAINS_OP, deferred); 2353 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 2354 #endif /* ENABLE_SPECIALIZATION */ 2355 } 2356 // _CONTAINS_OP 2357 { 2358 int res = PySequence_Contains(right, left); 2359 Py_DECREF(left); 2360 Py_DECREF(right); 2361 if (res < 0) goto pop_2_error; 2362 b = (res ^ oparg) ? Py_True : Py_False; 2363 } 2364 stack_pointer[-2] = b; 2365 stack_pointer += -1; 2366 DISPATCH(); 2367 } 2368 TARGET(CONTAINS_OP_DICT)2369 TARGET(CONTAINS_OP_DICT) { 2370 frame->instr_ptr = next_instr; 2371 next_instr += 2; 2372 INSTRUCTION_STATS(CONTAINS_OP_DICT); 2373 static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); 2374 PyObject *right; 2375 PyObject *left; 2376 PyObject *b; 2377 /* Skip 1 cache entry */ 2378 right = stack_pointer[-1]; 2379 left = stack_pointer[-2]; 2380 DEOPT_IF(!PyDict_CheckExact(right), CONTAINS_OP); 2381 STAT_INC(CONTAINS_OP, hit); 2382 int res = PyDict_Contains(right, left); 2383 Py_DECREF(left); 2384 Py_DECREF(right); 2385 if (res < 0) goto pop_2_error; 2386 b = (res ^ oparg) ? Py_True : Py_False; 2387 stack_pointer[-2] = b; 2388 stack_pointer += -1; 2389 DISPATCH(); 2390 } 2391 TARGET(CONTAINS_OP_SET)2392 TARGET(CONTAINS_OP_SET) { 2393 frame->instr_ptr = next_instr; 2394 next_instr += 2; 2395 INSTRUCTION_STATS(CONTAINS_OP_SET); 2396 static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); 2397 PyObject *right; 2398 PyObject *left; 2399 PyObject *b; 2400 /* Skip 1 cache entry */ 2401 right = stack_pointer[-1]; 2402 left = stack_pointer[-2]; 2403 DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right)), CONTAINS_OP); 2404 STAT_INC(CONTAINS_OP, hit); 2405 // Note: both set and frozenset use the same seq_contains method! 2406 int res = _PySet_Contains((PySetObject *)right, left); 2407 Py_DECREF(left); 2408 Py_DECREF(right); 2409 if (res < 0) goto pop_2_error; 2410 b = (res ^ oparg) ? Py_True : Py_False; 2411 stack_pointer[-2] = b; 2412 stack_pointer += -1; 2413 DISPATCH(); 2414 } 2415 TARGET(CONVERT_VALUE)2416 TARGET(CONVERT_VALUE) { 2417 frame->instr_ptr = next_instr; 2418 next_instr += 1; 2419 INSTRUCTION_STATS(CONVERT_VALUE); 2420 PyObject *value; 2421 PyObject *result; 2422 value = stack_pointer[-1]; 2423 conversion_func conv_fn; 2424 assert(oparg >= FVC_STR && oparg <= FVC_ASCII); 2425 conv_fn = _PyEval_ConversionFuncs[oparg]; 2426 result = conv_fn(value); 2427 Py_DECREF(value); 2428 if (result == NULL) goto pop_1_error; 2429 stack_pointer[-1] = result; 2430 DISPATCH(); 2431 } 2432 TARGET(COPY)2433 TARGET(COPY) { 2434 frame->instr_ptr = next_instr; 2435 next_instr += 1; 2436 INSTRUCTION_STATS(COPY); 2437 PyObject *bottom; 2438 PyObject *top; 2439 bottom = stack_pointer[-1 - (oparg-1)]; 2440 assert(oparg > 0); 2441 top = Py_NewRef(bottom); 2442 stack_pointer[0] = top; 2443 stack_pointer += 1; 2444 DISPATCH(); 2445 } 2446 TARGET(COPY_FREE_VARS)2447 TARGET(COPY_FREE_VARS) { 2448 frame->instr_ptr = next_instr; 2449 next_instr += 1; 2450 INSTRUCTION_STATS(COPY_FREE_VARS); 2451 /* Copy closure variables to free variables */ 2452 PyCodeObject *co = _PyFrame_GetCode(frame); 2453 assert(PyFunction_Check(frame->f_funcobj)); 2454 PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; 2455 assert(oparg == co->co_nfreevars); 2456 int offset = co->co_nlocalsplus - oparg; 2457 for (int i = 0; i < oparg; ++i) { 2458 PyObject *o = PyTuple_GET_ITEM(closure, i); 2459 frame->localsplus[offset + i] = Py_NewRef(o); 2460 } 2461 DISPATCH(); 2462 } 2463 TARGET(DELETE_ATTR)2464 TARGET(DELETE_ATTR) { 2465 frame->instr_ptr = next_instr; 2466 next_instr += 1; 2467 INSTRUCTION_STATS(DELETE_ATTR); 2468 PyObject *owner; 2469 owner = stack_pointer[-1]; 2470 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 2471 int err = PyObject_DelAttr(owner, name); 2472 Py_DECREF(owner); 2473 if (err) goto pop_1_error; 2474 stack_pointer += -1; 2475 DISPATCH(); 2476 } 2477 TARGET(DELETE_DEREF)2478 TARGET(DELETE_DEREF) { 2479 frame->instr_ptr = next_instr; 2480 next_instr += 1; 2481 INSTRUCTION_STATS(DELETE_DEREF); 2482 PyObject *cell = GETLOCAL(oparg); 2483 // Can't use ERROR_IF here. 2484 // Fortunately we don't need its superpower. 2485 PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); 2486 if (oldobj == NULL) { 2487 _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); 2488 goto error; 2489 } 2490 Py_DECREF(oldobj); 2491 DISPATCH(); 2492 } 2493 TARGET(DELETE_FAST)2494 TARGET(DELETE_FAST) { 2495 frame->instr_ptr = next_instr; 2496 next_instr += 1; 2497 INSTRUCTION_STATS(DELETE_FAST); 2498 PyObject *v = GETLOCAL(oparg); 2499 if (v == NULL) { 2500 _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, 2501 UNBOUNDLOCAL_ERROR_MSG, 2502 PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) 2503 ); 2504 if (1) goto error; 2505 } 2506 SETLOCAL(oparg, NULL); 2507 DISPATCH(); 2508 } 2509 TARGET(DELETE_GLOBAL)2510 TARGET(DELETE_GLOBAL) { 2511 frame->instr_ptr = next_instr; 2512 next_instr += 1; 2513 INSTRUCTION_STATS(DELETE_GLOBAL); 2514 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 2515 int err = PyDict_Pop(GLOBALS(), name, NULL); 2516 // Can't use ERROR_IF here. 2517 if (err < 0) { 2518 goto error; 2519 } 2520 if (err == 0) { 2521 _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, 2522 NAME_ERROR_MSG, name); 2523 goto error; 2524 } 2525 DISPATCH(); 2526 } 2527 TARGET(DELETE_NAME)2528 TARGET(DELETE_NAME) { 2529 frame->instr_ptr = next_instr; 2530 next_instr += 1; 2531 INSTRUCTION_STATS(DELETE_NAME); 2532 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 2533 PyObject *ns = LOCALS(); 2534 int err; 2535 if (ns == NULL) { 2536 _PyErr_Format(tstate, PyExc_SystemError, 2537 "no locals when deleting %R", name); 2538 goto error; 2539 } 2540 err = PyObject_DelItem(ns, name); 2541 // Can't use ERROR_IF here. 2542 if (err != 0) { 2543 _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, 2544 NAME_ERROR_MSG, 2545 name); 2546 goto error; 2547 } 2548 DISPATCH(); 2549 } 2550 TARGET(DELETE_SUBSCR)2551 TARGET(DELETE_SUBSCR) { 2552 frame->instr_ptr = next_instr; 2553 next_instr += 1; 2554 INSTRUCTION_STATS(DELETE_SUBSCR); 2555 PyObject *sub; 2556 PyObject *container; 2557 sub = stack_pointer[-1]; 2558 container = stack_pointer[-2]; 2559 /* del container[sub] */ 2560 int err = PyObject_DelItem(container, sub); 2561 Py_DECREF(container); 2562 Py_DECREF(sub); 2563 if (err) goto pop_2_error; 2564 stack_pointer += -2; 2565 DISPATCH(); 2566 } 2567 TARGET(DICT_MERGE)2568 TARGET(DICT_MERGE) { 2569 frame->instr_ptr = next_instr; 2570 next_instr += 1; 2571 INSTRUCTION_STATS(DICT_MERGE); 2572 PyObject *update; 2573 PyObject *dict; 2574 PyObject *callable; 2575 update = stack_pointer[-1]; 2576 dict = stack_pointer[-2 - (oparg - 1)]; 2577 callable = stack_pointer[-5 - (oparg - 1)]; 2578 if (_PyDict_MergeEx(dict, update, 2) < 0) { 2579 _PyEval_FormatKwargsError(tstate, callable, update); 2580 Py_DECREF(update); 2581 if (true) goto pop_1_error; 2582 } 2583 Py_DECREF(update); 2584 stack_pointer += -1; 2585 DISPATCH(); 2586 } 2587 TARGET(DICT_UPDATE)2588 TARGET(DICT_UPDATE) { 2589 frame->instr_ptr = next_instr; 2590 next_instr += 1; 2591 INSTRUCTION_STATS(DICT_UPDATE); 2592 PyObject *update; 2593 PyObject *dict; 2594 update = stack_pointer[-1]; 2595 dict = stack_pointer[-2 - (oparg - 1)]; 2596 if (PyDict_Update(dict, update) < 0) { 2597 if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { 2598 _PyErr_Format(tstate, PyExc_TypeError, 2599 "'%.200s' object is not a mapping", 2600 Py_TYPE(update)->tp_name); 2601 } 2602 Py_DECREF(update); 2603 if (true) goto pop_1_error; 2604 } 2605 Py_DECREF(update); 2606 stack_pointer += -1; 2607 DISPATCH(); 2608 } 2609 TARGET(END_ASYNC_FOR)2610 TARGET(END_ASYNC_FOR) { 2611 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 2612 (void)this_instr; 2613 next_instr += 1; 2614 INSTRUCTION_STATS(END_ASYNC_FOR); 2615 PyObject *exc; 2616 PyObject *awaitable; 2617 exc = stack_pointer[-1]; 2618 awaitable = stack_pointer[-2]; 2619 assert(exc && PyExceptionInstance_Check(exc)); 2620 if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { 2621 Py_DECREF(awaitable); 2622 Py_DECREF(exc); 2623 } 2624 else { 2625 Py_INCREF(exc); 2626 _PyErr_SetRaisedException(tstate, exc); 2627 monitor_reraise(tstate, frame, this_instr); 2628 goto exception_unwind; 2629 } 2630 stack_pointer += -2; 2631 DISPATCH(); 2632 } 2633 TARGET(END_FOR)2634 TARGET(END_FOR) { 2635 frame->instr_ptr = next_instr; 2636 next_instr += 1; 2637 INSTRUCTION_STATS(END_FOR); 2638 PyObject *value; 2639 value = stack_pointer[-1]; 2640 Py_DECREF(value); 2641 stack_pointer += -1; 2642 DISPATCH(); 2643 } 2644 TARGET(END_SEND)2645 TARGET(END_SEND) { 2646 frame->instr_ptr = next_instr; 2647 next_instr += 1; 2648 INSTRUCTION_STATS(END_SEND); 2649 PyObject *value; 2650 PyObject *receiver; 2651 value = stack_pointer[-1]; 2652 receiver = stack_pointer[-2]; 2653 Py_DECREF(receiver); 2654 stack_pointer[-2] = value; 2655 stack_pointer += -1; 2656 DISPATCH(); 2657 } 2658 TARGET(ENTER_EXECUTOR)2659 TARGET(ENTER_EXECUTOR) { 2660 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 2661 (void)this_instr; 2662 next_instr += 1; 2663 INSTRUCTION_STATS(ENTER_EXECUTOR); 2664 #ifdef _Py_TIER2 2665 PyCodeObject *code = _PyFrame_GetCode(frame); 2666 _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; 2667 assert(executor->vm_data.index == INSTR_OFFSET() - 1); 2668 assert(executor->vm_data.code == code); 2669 assert(executor->vm_data.valid); 2670 assert(tstate->previous_executor == NULL); 2671 /* If the eval breaker is set then stay in tier 1. 2672 * This avoids any potentially infinite loops 2673 * involving _RESUME_CHECK */ 2674 if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { 2675 opcode = executor->vm_data.opcode; 2676 oparg = (oparg & ~255) | executor->vm_data.oparg; 2677 next_instr = this_instr; 2678 if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { 2679 PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); 2680 } 2681 DISPATCH_GOTO(); 2682 } 2683 tstate->previous_executor = Py_None; 2684 Py_INCREF(executor); 2685 GOTO_TIER_TWO(executor); 2686 #else 2687 Py_FatalError("ENTER_EXECUTOR is not supported in this build"); 2688 #endif /* _Py_TIER2 */ 2689 DISPATCH(); 2690 } 2691 TARGET(EXIT_INIT_CHECK)2692 TARGET(EXIT_INIT_CHECK) { 2693 frame->instr_ptr = next_instr; 2694 next_instr += 1; 2695 INSTRUCTION_STATS(EXIT_INIT_CHECK); 2696 PyObject *should_be_none; 2697 should_be_none = stack_pointer[-1]; 2698 assert(STACK_LEVEL() == 2); 2699 if (should_be_none != Py_None) { 2700 PyErr_Format(PyExc_TypeError, 2701 "__init__() should return None, not '%.200s'", 2702 Py_TYPE(should_be_none)->tp_name); 2703 goto error; 2704 } 2705 stack_pointer += -1; 2706 DISPATCH(); 2707 } 2708 TARGET(EXTENDED_ARG)2709 TARGET(EXTENDED_ARG) { 2710 frame->instr_ptr = next_instr; 2711 next_instr += 1; 2712 INSTRUCTION_STATS(EXTENDED_ARG); 2713 assert(oparg); 2714 opcode = next_instr->op.code; 2715 oparg = oparg << 8 | next_instr->op.arg; 2716 PRE_DISPATCH_GOTO(); 2717 DISPATCH_GOTO(); 2718 } 2719 TARGET(FORMAT_SIMPLE)2720 TARGET(FORMAT_SIMPLE) { 2721 frame->instr_ptr = next_instr; 2722 next_instr += 1; 2723 INSTRUCTION_STATS(FORMAT_SIMPLE); 2724 PyObject *value; 2725 PyObject *res; 2726 value = stack_pointer[-1]; 2727 /* If value is a unicode object, then we know the result 2728 * of format(value) is value itself. */ 2729 if (!PyUnicode_CheckExact(value)) { 2730 res = PyObject_Format(value, NULL); 2731 Py_DECREF(value); 2732 if (res == NULL) goto pop_1_error; 2733 } 2734 else { 2735 res = value; 2736 } 2737 stack_pointer[-1] = res; 2738 DISPATCH(); 2739 } 2740 TARGET(FORMAT_WITH_SPEC)2741 TARGET(FORMAT_WITH_SPEC) { 2742 frame->instr_ptr = next_instr; 2743 next_instr += 1; 2744 INSTRUCTION_STATS(FORMAT_WITH_SPEC); 2745 PyObject *fmt_spec; 2746 PyObject *value; 2747 PyObject *res; 2748 fmt_spec = stack_pointer[-1]; 2749 value = stack_pointer[-2]; 2750 res = PyObject_Format(value, fmt_spec); 2751 Py_DECREF(value); 2752 Py_DECREF(fmt_spec); 2753 if (res == NULL) goto pop_2_error; 2754 stack_pointer[-2] = res; 2755 stack_pointer += -1; 2756 DISPATCH(); 2757 } 2758 TARGET(FOR_ITER)2759 TARGET(FOR_ITER) { 2760 frame->instr_ptr = next_instr; 2761 next_instr += 2; 2762 INSTRUCTION_STATS(FOR_ITER); 2763 PREDICTED(FOR_ITER); 2764 _Py_CODEUNIT *this_instr = next_instr - 2; 2765 (void)this_instr; 2766 PyObject *iter; 2767 PyObject *next; 2768 // _SPECIALIZE_FOR_ITER 2769 iter = stack_pointer[-1]; 2770 { 2771 uint16_t counter = read_u16(&this_instr[1].cache); 2772 (void)counter; 2773 #if ENABLE_SPECIALIZATION 2774 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 2775 next_instr = this_instr; 2776 _Py_Specialize_ForIter(iter, next_instr, oparg); 2777 DISPATCH_SAME_OPARG(); 2778 } 2779 STAT_INC(FOR_ITER, deferred); 2780 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 2781 #endif /* ENABLE_SPECIALIZATION */ 2782 } 2783 // _FOR_ITER 2784 { 2785 /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ 2786 next = (*Py_TYPE(iter)->tp_iternext)(iter); 2787 if (next == NULL) { 2788 if (_PyErr_Occurred(tstate)) { 2789 if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { 2790 goto error; 2791 } 2792 _PyEval_MonitorRaise(tstate, frame, this_instr); 2793 _PyErr_Clear(tstate); 2794 } 2795 /* iterator ended normally */ 2796 assert(next_instr[oparg].op.code == END_FOR || 2797 next_instr[oparg].op.code == INSTRUMENTED_END_FOR); 2798 Py_DECREF(iter); 2799 STACK_SHRINK(1); 2800 /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ 2801 JUMPBY(oparg + 2); 2802 DISPATCH(); 2803 } 2804 // Common case: no jump, leave it to the code generator 2805 } 2806 stack_pointer[0] = next; 2807 stack_pointer += 1; 2808 DISPATCH(); 2809 } 2810 TARGET(FOR_ITER_GEN)2811 TARGET(FOR_ITER_GEN) { 2812 frame->instr_ptr = next_instr; 2813 next_instr += 2; 2814 INSTRUCTION_STATS(FOR_ITER_GEN); 2815 static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); 2816 PyObject *iter; 2817 _PyInterpreterFrame *gen_frame; 2818 _PyInterpreterFrame *new_frame; 2819 /* Skip 1 cache entry */ 2820 // _CHECK_PEP_523 2821 { 2822 DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); 2823 } 2824 // _FOR_ITER_GEN_FRAME 2825 iter = stack_pointer[-1]; 2826 { 2827 PyGenObject *gen = (PyGenObject *)iter; 2828 DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); 2829 DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); 2830 STAT_INC(FOR_ITER, hit); 2831 gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; 2832 _PyFrame_StackPush(gen_frame, Py_None); 2833 gen->gi_frame_state = FRAME_EXECUTING; 2834 gen->gi_exc_state.previous_item = tstate->exc_info; 2835 tstate->exc_info = &gen->gi_exc_state; 2836 // oparg is the return offset from the next instruction. 2837 frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); 2838 } 2839 // _PUSH_FRAME 2840 new_frame = gen_frame; 2841 { 2842 // Write it out explicitly because it's subtly different. 2843 // Eventually this should be the only occurrence of this code. 2844 assert(tstate->interp->eval_frame == NULL); 2845 _PyFrame_SetStackPointer(frame, stack_pointer); 2846 new_frame->previous = frame; 2847 CALL_STAT_INC(inlined_py_calls); 2848 frame = tstate->current_frame = new_frame; 2849 tstate->py_recursion_remaining--; 2850 LOAD_SP(); 2851 LOAD_IP(0); 2852 LLTRACE_RESUME_FRAME(); 2853 } 2854 DISPATCH(); 2855 } 2856 TARGET(FOR_ITER_LIST)2857 TARGET(FOR_ITER_LIST) { 2858 frame->instr_ptr = next_instr; 2859 next_instr += 2; 2860 INSTRUCTION_STATS(FOR_ITER_LIST); 2861 static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); 2862 PyObject *iter; 2863 PyObject *next; 2864 /* Skip 1 cache entry */ 2865 // _ITER_CHECK_LIST 2866 iter = stack_pointer[-1]; 2867 { 2868 DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); 2869 } 2870 // _ITER_JUMP_LIST 2871 { 2872 _PyListIterObject *it = (_PyListIterObject *)iter; 2873 assert(Py_TYPE(iter) == &PyListIter_Type); 2874 STAT_INC(FOR_ITER, hit); 2875 PyListObject *seq = it->it_seq; 2876 if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { 2877 it->it_index = -1; 2878 #ifndef Py_GIL_DISABLED 2879 if (seq != NULL) { 2880 it->it_seq = NULL; 2881 Py_DECREF(seq); 2882 } 2883 #endif 2884 Py_DECREF(iter); 2885 STACK_SHRINK(1); 2886 /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ 2887 JUMPBY(oparg + 2); 2888 DISPATCH(); 2889 } 2890 } 2891 // _ITER_NEXT_LIST 2892 { 2893 _PyListIterObject *it = (_PyListIterObject *)iter; 2894 assert(Py_TYPE(iter) == &PyListIter_Type); 2895 PyListObject *seq = it->it_seq; 2896 assert(seq); 2897 assert(it->it_index < PyList_GET_SIZE(seq)); 2898 next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); 2899 } 2900 stack_pointer[0] = next; 2901 stack_pointer += 1; 2902 DISPATCH(); 2903 } 2904 TARGET(FOR_ITER_RANGE)2905 TARGET(FOR_ITER_RANGE) { 2906 frame->instr_ptr = next_instr; 2907 next_instr += 2; 2908 INSTRUCTION_STATS(FOR_ITER_RANGE); 2909 static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); 2910 PyObject *iter; 2911 PyObject *next; 2912 /* Skip 1 cache entry */ 2913 // _ITER_CHECK_RANGE 2914 iter = stack_pointer[-1]; 2915 { 2916 _PyRangeIterObject *r = (_PyRangeIterObject *)iter; 2917 DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); 2918 } 2919 // _ITER_JUMP_RANGE 2920 { 2921 _PyRangeIterObject *r = (_PyRangeIterObject *)iter; 2922 assert(Py_TYPE(r) == &PyRangeIter_Type); 2923 STAT_INC(FOR_ITER, hit); 2924 if (r->len <= 0) { 2925 STACK_SHRINK(1); 2926 Py_DECREF(r); 2927 // Jump over END_FOR and POP_TOP instructions. 2928 JUMPBY(oparg + 2); 2929 DISPATCH(); 2930 } 2931 } 2932 // _ITER_NEXT_RANGE 2933 { 2934 _PyRangeIterObject *r = (_PyRangeIterObject *)iter; 2935 assert(Py_TYPE(r) == &PyRangeIter_Type); 2936 assert(r->len > 0); 2937 long value = r->start; 2938 r->start = value + r->step; 2939 r->len--; 2940 next = PyLong_FromLong(value); 2941 if (next == NULL) goto error; 2942 } 2943 stack_pointer[0] = next; 2944 stack_pointer += 1; 2945 DISPATCH(); 2946 } 2947 TARGET(FOR_ITER_TUPLE)2948 TARGET(FOR_ITER_TUPLE) { 2949 frame->instr_ptr = next_instr; 2950 next_instr += 2; 2951 INSTRUCTION_STATS(FOR_ITER_TUPLE); 2952 static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); 2953 PyObject *iter; 2954 PyObject *next; 2955 /* Skip 1 cache entry */ 2956 // _ITER_CHECK_TUPLE 2957 iter = stack_pointer[-1]; 2958 { 2959 DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); 2960 } 2961 // _ITER_JUMP_TUPLE 2962 { 2963 _PyTupleIterObject *it = (_PyTupleIterObject *)iter; 2964 assert(Py_TYPE(iter) == &PyTupleIter_Type); 2965 STAT_INC(FOR_ITER, hit); 2966 PyTupleObject *seq = it->it_seq; 2967 if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { 2968 if (seq != NULL) { 2969 it->it_seq = NULL; 2970 Py_DECREF(seq); 2971 } 2972 Py_DECREF(iter); 2973 STACK_SHRINK(1); 2974 /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ 2975 JUMPBY(oparg + 2); 2976 DISPATCH(); 2977 } 2978 } 2979 // _ITER_NEXT_TUPLE 2980 { 2981 _PyTupleIterObject *it = (_PyTupleIterObject *)iter; 2982 assert(Py_TYPE(iter) == &PyTupleIter_Type); 2983 PyTupleObject *seq = it->it_seq; 2984 assert(seq); 2985 assert(it->it_index < PyTuple_GET_SIZE(seq)); 2986 next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); 2987 } 2988 stack_pointer[0] = next; 2989 stack_pointer += 1; 2990 DISPATCH(); 2991 } 2992 TARGET(GET_AITER)2993 TARGET(GET_AITER) { 2994 frame->instr_ptr = next_instr; 2995 next_instr += 1; 2996 INSTRUCTION_STATS(GET_AITER); 2997 PyObject *obj; 2998 PyObject *iter; 2999 obj = stack_pointer[-1]; 3000 unaryfunc getter = NULL; 3001 PyTypeObject *type = Py_TYPE(obj); 3002 if (type->tp_as_async != NULL) { 3003 getter = type->tp_as_async->am_aiter; 3004 } 3005 if (getter == NULL) { 3006 _PyErr_Format(tstate, PyExc_TypeError, 3007 "'async for' requires an object with " 3008 "__aiter__ method, got %.100s", 3009 type->tp_name); 3010 Py_DECREF(obj); 3011 if (true) goto pop_1_error; 3012 } 3013 iter = (*getter)(obj); 3014 Py_DECREF(obj); 3015 if (iter == NULL) goto pop_1_error; 3016 if (Py_TYPE(iter)->tp_as_async == NULL || 3017 Py_TYPE(iter)->tp_as_async->am_anext == NULL) { 3018 _PyErr_Format(tstate, PyExc_TypeError, 3019 "'async for' received an object from __aiter__ " 3020 "that does not implement __anext__: %.100s", 3021 Py_TYPE(iter)->tp_name); 3022 Py_DECREF(iter); 3023 if (true) goto pop_1_error; 3024 } 3025 stack_pointer[-1] = iter; 3026 DISPATCH(); 3027 } 3028 TARGET(GET_ANEXT)3029 TARGET(GET_ANEXT) { 3030 frame->instr_ptr = next_instr; 3031 next_instr += 1; 3032 INSTRUCTION_STATS(GET_ANEXT); 3033 PyObject *aiter; 3034 PyObject *awaitable; 3035 aiter = stack_pointer[-1]; 3036 unaryfunc getter = NULL; 3037 PyObject *next_iter = NULL; 3038 PyTypeObject *type = Py_TYPE(aiter); 3039 if (PyAsyncGen_CheckExact(aiter)) { 3040 awaitable = type->tp_as_async->am_anext(aiter); 3041 if (awaitable == NULL) { 3042 goto error; 3043 } 3044 } else { 3045 if (type->tp_as_async != NULL){ 3046 getter = type->tp_as_async->am_anext; 3047 } 3048 if (getter != NULL) { 3049 next_iter = (*getter)(aiter); 3050 if (next_iter == NULL) { 3051 goto error; 3052 } 3053 } 3054 else { 3055 _PyErr_Format(tstate, PyExc_TypeError, 3056 "'async for' requires an iterator with " 3057 "__anext__ method, got %.100s", 3058 type->tp_name); 3059 goto error; 3060 } 3061 awaitable = _PyCoro_GetAwaitableIter(next_iter); 3062 if (awaitable == NULL) { 3063 _PyErr_FormatFromCause( 3064 PyExc_TypeError, 3065 "'async for' received an invalid object " 3066 "from __anext__: %.100s", 3067 Py_TYPE(next_iter)->tp_name); 3068 Py_DECREF(next_iter); 3069 goto error; 3070 } else { 3071 Py_DECREF(next_iter); 3072 } 3073 } 3074 stack_pointer[0] = awaitable; 3075 stack_pointer += 1; 3076 DISPATCH(); 3077 } 3078 TARGET(GET_AWAITABLE)3079 TARGET(GET_AWAITABLE) { 3080 frame->instr_ptr = next_instr; 3081 next_instr += 1; 3082 INSTRUCTION_STATS(GET_AWAITABLE); 3083 PyObject *iterable; 3084 PyObject *iter; 3085 iterable = stack_pointer[-1]; 3086 iter = _PyCoro_GetAwaitableIter(iterable); 3087 if (iter == NULL) { 3088 _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); 3089 } 3090 Py_DECREF(iterable); 3091 if (iter != NULL && PyCoro_CheckExact(iter)) { 3092 PyObject *yf = _PyGen_yf((PyGenObject*)iter); 3093 if (yf != NULL) { 3094 /* `iter` is a coroutine object that is being 3095 awaited, `yf` is a pointer to the current awaitable 3096 being awaited on. */ 3097 Py_DECREF(yf); 3098 Py_CLEAR(iter); 3099 _PyErr_SetString(tstate, PyExc_RuntimeError, 3100 "coroutine is being awaited already"); 3101 /* The code below jumps to `error` if `iter` is NULL. */ 3102 } 3103 } 3104 if (iter == NULL) goto pop_1_error; 3105 stack_pointer[-1] = iter; 3106 DISPATCH(); 3107 } 3108 TARGET(GET_ITER)3109 TARGET(GET_ITER) { 3110 frame->instr_ptr = next_instr; 3111 next_instr += 1; 3112 INSTRUCTION_STATS(GET_ITER); 3113 PyObject *iterable; 3114 PyObject *iter; 3115 iterable = stack_pointer[-1]; 3116 /* before: [obj]; after [getiter(obj)] */ 3117 iter = PyObject_GetIter(iterable); 3118 Py_DECREF(iterable); 3119 if (iter == NULL) goto pop_1_error; 3120 stack_pointer[-1] = iter; 3121 DISPATCH(); 3122 } 3123 TARGET(GET_LEN)3124 TARGET(GET_LEN) { 3125 frame->instr_ptr = next_instr; 3126 next_instr += 1; 3127 INSTRUCTION_STATS(GET_LEN); 3128 PyObject *obj; 3129 PyObject *len_o; 3130 obj = stack_pointer[-1]; 3131 // PUSH(len(TOS)) 3132 Py_ssize_t len_i = PyObject_Length(obj); 3133 if (len_i < 0) goto error; 3134 len_o = PyLong_FromSsize_t(len_i); 3135 if (len_o == NULL) goto error; 3136 stack_pointer[0] = len_o; 3137 stack_pointer += 1; 3138 DISPATCH(); 3139 } 3140 TARGET(GET_YIELD_FROM_ITER)3141 TARGET(GET_YIELD_FROM_ITER) { 3142 frame->instr_ptr = next_instr; 3143 next_instr += 1; 3144 INSTRUCTION_STATS(GET_YIELD_FROM_ITER); 3145 PyObject *iterable; 3146 PyObject *iter; 3147 iterable = stack_pointer[-1]; 3148 /* before: [obj]; after [getiter(obj)] */ 3149 if (PyCoro_CheckExact(iterable)) { 3150 /* `iterable` is a coroutine */ 3151 if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { 3152 /* and it is used in a 'yield from' expression of a 3153 regular generator. */ 3154 _PyErr_SetString(tstate, PyExc_TypeError, 3155 "cannot 'yield from' a coroutine object " 3156 "in a non-coroutine generator"); 3157 goto error; 3158 } 3159 iter = iterable; 3160 } 3161 else if (PyGen_CheckExact(iterable)) { 3162 iter = iterable; 3163 } 3164 else { 3165 /* `iterable` is not a generator. */ 3166 iter = PyObject_GetIter(iterable); 3167 if (iter == NULL) { 3168 goto error; 3169 } 3170 Py_DECREF(iterable); 3171 } 3172 stack_pointer[-1] = iter; 3173 DISPATCH(); 3174 } 3175 TARGET(IMPORT_FROM)3176 TARGET(IMPORT_FROM) { 3177 frame->instr_ptr = next_instr; 3178 next_instr += 1; 3179 INSTRUCTION_STATS(IMPORT_FROM); 3180 PyObject *from; 3181 PyObject *res; 3182 from = stack_pointer[-1]; 3183 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 3184 res = import_from(tstate, from, name); 3185 if (res == NULL) goto error; 3186 stack_pointer[0] = res; 3187 stack_pointer += 1; 3188 DISPATCH(); 3189 } 3190 TARGET(IMPORT_NAME)3191 TARGET(IMPORT_NAME) { 3192 frame->instr_ptr = next_instr; 3193 next_instr += 1; 3194 INSTRUCTION_STATS(IMPORT_NAME); 3195 PyObject *fromlist; 3196 PyObject *level; 3197 PyObject *res; 3198 fromlist = stack_pointer[-1]; 3199 level = stack_pointer[-2]; 3200 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 3201 res = import_name(tstate, frame, name, fromlist, level); 3202 Py_DECREF(level); 3203 Py_DECREF(fromlist); 3204 if (res == NULL) goto pop_2_error; 3205 stack_pointer[-2] = res; 3206 stack_pointer += -1; 3207 DISPATCH(); 3208 } 3209 TARGET(INSTRUMENTED_CALL)3210 TARGET(INSTRUMENTED_CALL) { 3211 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3212 (void)this_instr; 3213 next_instr += 4; 3214 INSTRUCTION_STATS(INSTRUMENTED_CALL); 3215 /* Skip 3 cache entries */ 3216 int is_meth = PEEK(oparg + 1) != NULL; 3217 int total_args = oparg + is_meth; 3218 PyObject *function = PEEK(oparg + 2); 3219 PyObject *arg = total_args == 0 ? 3220 &_PyInstrumentation_MISSING : PEEK(total_args); 3221 int err = _Py_call_instrumentation_2args( 3222 tstate, PY_MONITORING_EVENT_CALL, 3223 frame, this_instr, function, arg); 3224 if (err) goto error; 3225 PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); 3226 GO_TO_INSTRUCTION(CALL); 3227 } 3228 TARGET(INSTRUMENTED_CALL_FUNCTION_EX)3229 TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { 3230 frame->instr_ptr = next_instr; 3231 next_instr += 1; 3232 INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); 3233 GO_TO_INSTRUCTION(CALL_FUNCTION_EX); 3234 } 3235 TARGET(INSTRUMENTED_CALL_KW)3236 TARGET(INSTRUMENTED_CALL_KW) { 3237 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3238 (void)this_instr; 3239 next_instr += 1; 3240 INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); 3241 int is_meth = PEEK(oparg + 2) != NULL; 3242 int total_args = oparg + is_meth; 3243 PyObject *function = PEEK(oparg + 3); 3244 PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING 3245 : PEEK(total_args + 1); 3246 int err = _Py_call_instrumentation_2args( 3247 tstate, PY_MONITORING_EVENT_CALL, 3248 frame, this_instr, function, arg); 3249 if (err) goto error; 3250 GO_TO_INSTRUCTION(CALL_KW); 3251 } 3252 TARGET(INSTRUMENTED_END_FOR)3253 TARGET(INSTRUMENTED_END_FOR) { 3254 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3255 (void)this_instr; 3256 next_instr += 1; 3257 INSTRUCTION_STATS(INSTRUMENTED_END_FOR); 3258 PyObject *value; 3259 PyObject *receiver; 3260 value = stack_pointer[-1]; 3261 receiver = stack_pointer[-2]; 3262 /* Need to create a fake StopIteration error here, 3263 * to conform to PEP 380 */ 3264 if (PyGen_Check(receiver)) { 3265 if (monitor_stop_iteration(tstate, frame, this_instr, value)) { 3266 goto error; 3267 } 3268 } 3269 Py_DECREF(value); 3270 stack_pointer += -1; 3271 DISPATCH(); 3272 } 3273 TARGET(INSTRUMENTED_END_SEND)3274 TARGET(INSTRUMENTED_END_SEND) { 3275 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3276 (void)this_instr; 3277 next_instr += 1; 3278 INSTRUCTION_STATS(INSTRUMENTED_END_SEND); 3279 PyObject *value; 3280 PyObject *receiver; 3281 value = stack_pointer[-1]; 3282 receiver = stack_pointer[-2]; 3283 if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { 3284 if (monitor_stop_iteration(tstate, frame, this_instr, value)) { 3285 goto error; 3286 } 3287 } 3288 Py_DECREF(receiver); 3289 stack_pointer[-2] = value; 3290 stack_pointer += -1; 3291 DISPATCH(); 3292 } 3293 TARGET(INSTRUMENTED_FOR_ITER)3294 TARGET(INSTRUMENTED_FOR_ITER) { 3295 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3296 (void)this_instr; 3297 next_instr += 2; 3298 INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); 3299 /* Skip 1 cache entry */ 3300 _Py_CODEUNIT *target; 3301 PyObject *iter = TOP(); 3302 PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); 3303 if (next != NULL) { 3304 PUSH(next); 3305 target = next_instr; 3306 } 3307 else { 3308 if (_PyErr_Occurred(tstate)) { 3309 if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { 3310 goto error; 3311 } 3312 _PyEval_MonitorRaise(tstate, frame, this_instr); 3313 _PyErr_Clear(tstate); 3314 } 3315 /* iterator ended normally */ 3316 assert(next_instr[oparg].op.code == END_FOR || 3317 next_instr[oparg].op.code == INSTRUMENTED_END_FOR); 3318 STACK_SHRINK(1); 3319 Py_DECREF(iter); 3320 /* Skip END_FOR and POP_TOP */ 3321 target = next_instr + oparg + 2; 3322 } 3323 INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); 3324 DISPATCH(); 3325 } 3326 TARGET(INSTRUMENTED_INSTRUCTION)3327 TARGET(INSTRUMENTED_INSTRUCTION) { 3328 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3329 (void)this_instr; 3330 next_instr += 1; 3331 INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); 3332 int next_opcode = _Py_call_instrumentation_instruction( 3333 tstate, frame, this_instr); 3334 if (next_opcode < 0) goto error; 3335 next_instr = this_instr; 3336 if (_PyOpcode_Caches[next_opcode]) { 3337 PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); 3338 } 3339 assert(next_opcode > 0 && next_opcode < 256); 3340 opcode = next_opcode; 3341 DISPATCH_GOTO(); 3342 } 3343 TARGET(INSTRUMENTED_JUMP_BACKWARD)3344 TARGET(INSTRUMENTED_JUMP_BACKWARD) { 3345 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3346 (void)this_instr; 3347 next_instr += 2; 3348 INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); 3349 /* Skip 1 cache entry */ 3350 CHECK_EVAL_BREAKER(); 3351 INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); 3352 DISPATCH(); 3353 } 3354 TARGET(INSTRUMENTED_JUMP_FORWARD)3355 TARGET(INSTRUMENTED_JUMP_FORWARD) { 3356 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3357 (void)this_instr; 3358 next_instr += 1; 3359 INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); 3360 INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); 3361 DISPATCH(); 3362 } 3363 TARGET(INSTRUMENTED_LOAD_SUPER_ATTR)3364 TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { 3365 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3366 (void)this_instr; 3367 next_instr += 2; 3368 INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); 3369 /* Skip 1 cache entry */ 3370 // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we 3371 // don't want to specialize instrumented instructions 3372 PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); 3373 GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); 3374 } 3375 TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE)3376 TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { 3377 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3378 (void)this_instr; 3379 next_instr += 2; 3380 INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); 3381 /* Skip 1 cache entry */ 3382 PyObject *cond = POP(); 3383 assert(PyBool_Check(cond)); 3384 int flag = Py_IsFalse(cond); 3385 int offset = flag * oparg; 3386 #if ENABLE_SPECIALIZATION 3387 this_instr[1].cache = (this_instr[1].cache << 1) | flag; 3388 #endif 3389 INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); 3390 DISPATCH(); 3391 } 3392 TARGET(INSTRUMENTED_POP_JUMP_IF_NONE)3393 TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { 3394 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3395 (void)this_instr; 3396 next_instr += 2; 3397 INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); 3398 /* Skip 1 cache entry */ 3399 PyObject *value = POP(); 3400 int flag = Py_IsNone(value); 3401 int offset; 3402 if (flag) { 3403 offset = oparg; 3404 } 3405 else { 3406 Py_DECREF(value); 3407 offset = 0; 3408 } 3409 #if ENABLE_SPECIALIZATION 3410 this_instr[1].cache = (this_instr[1].cache << 1) | flag; 3411 #endif 3412 INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); 3413 DISPATCH(); 3414 } 3415 TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE)3416 TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { 3417 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3418 (void)this_instr; 3419 next_instr += 2; 3420 INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); 3421 /* Skip 1 cache entry */ 3422 PyObject *value = POP(); 3423 int offset; 3424 int nflag = Py_IsNone(value); 3425 if (nflag) { 3426 offset = 0; 3427 } 3428 else { 3429 Py_DECREF(value); 3430 offset = oparg; 3431 } 3432 #if ENABLE_SPECIALIZATION 3433 this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; 3434 #endif 3435 INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); 3436 DISPATCH(); 3437 } 3438 TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE)3439 TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { 3440 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3441 (void)this_instr; 3442 next_instr += 2; 3443 INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); 3444 /* Skip 1 cache entry */ 3445 PyObject *cond = POP(); 3446 assert(PyBool_Check(cond)); 3447 int flag = Py_IsTrue(cond); 3448 int offset = flag * oparg; 3449 #if ENABLE_SPECIALIZATION 3450 this_instr[1].cache = (this_instr[1].cache << 1) | flag; 3451 #endif 3452 INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); 3453 DISPATCH(); 3454 } 3455 TARGET(INSTRUMENTED_RESUME)3456 TARGET(INSTRUMENTED_RESUME) { 3457 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3458 (void)this_instr; 3459 next_instr += 1; 3460 INSTRUCTION_STATS(INSTRUMENTED_RESUME); 3461 uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; 3462 uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); 3463 if (code_version != global_version && tstate->tracing == 0) { 3464 if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { 3465 goto error; 3466 } 3467 next_instr = this_instr; 3468 } 3469 else { 3470 if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { 3471 CHECK_EVAL_BREAKER(); 3472 } 3473 _PyFrame_SetStackPointer(frame, stack_pointer); 3474 int err = _Py_call_instrumentation( 3475 tstate, oparg > 0, frame, this_instr); 3476 stack_pointer = _PyFrame_GetStackPointer(frame); 3477 if (err) goto error; 3478 if (frame->instr_ptr != this_instr) { 3479 /* Instrumentation has jumped */ 3480 next_instr = frame->instr_ptr; 3481 DISPATCH(); 3482 } 3483 } 3484 DISPATCH(); 3485 } 3486 TARGET(INSTRUMENTED_RETURN_CONST)3487 TARGET(INSTRUMENTED_RETURN_CONST) { 3488 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3489 (void)this_instr; 3490 next_instr += 1; 3491 INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); 3492 PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); 3493 int err = _Py_call_instrumentation_arg( 3494 tstate, PY_MONITORING_EVENT_PY_RETURN, 3495 frame, this_instr, retval); 3496 if (err) goto error; 3497 Py_INCREF(retval); 3498 assert(EMPTY()); 3499 _PyFrame_SetStackPointer(frame, stack_pointer); 3500 _Py_LeaveRecursiveCallPy(tstate); 3501 assert(frame != &entry_frame); 3502 // GH-99729: We need to unlink the frame *before* clearing it: 3503 _PyInterpreterFrame *dying = frame; 3504 frame = tstate->current_frame = dying->previous; 3505 _PyEval_FrameClearAndPop(tstate, dying); 3506 _PyFrame_StackPush(frame, retval); 3507 LOAD_IP(frame->return_offset); 3508 goto resume_frame; 3509 } 3510 TARGET(INSTRUMENTED_RETURN_VALUE)3511 TARGET(INSTRUMENTED_RETURN_VALUE) { 3512 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3513 (void)this_instr; 3514 next_instr += 1; 3515 INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); 3516 PyObject *retval; 3517 retval = stack_pointer[-1]; 3518 int err = _Py_call_instrumentation_arg( 3519 tstate, PY_MONITORING_EVENT_PY_RETURN, 3520 frame, this_instr, retval); 3521 if (err) goto error; 3522 STACK_SHRINK(1); 3523 assert(EMPTY()); 3524 _PyFrame_SetStackPointer(frame, stack_pointer); 3525 _Py_LeaveRecursiveCallPy(tstate); 3526 assert(frame != &entry_frame); 3527 // GH-99729: We need to unlink the frame *before* clearing it: 3528 _PyInterpreterFrame *dying = frame; 3529 frame = tstate->current_frame = dying->previous; 3530 _PyEval_FrameClearAndPop(tstate, dying); 3531 _PyFrame_StackPush(frame, retval); 3532 LOAD_IP(frame->return_offset); 3533 goto resume_frame; 3534 } 3535 TARGET(INSTRUMENTED_YIELD_VALUE)3536 TARGET(INSTRUMENTED_YIELD_VALUE) { 3537 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3538 (void)this_instr; 3539 next_instr += 1; 3540 INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); 3541 PyObject *retval; 3542 retval = stack_pointer[-1]; 3543 assert(frame != &entry_frame); 3544 frame->instr_ptr = next_instr; 3545 PyGenObject *gen = _PyFrame_GetGenerator(frame); 3546 assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); 3547 assert(oparg == 0 || oparg == 1); 3548 gen->gi_frame_state = FRAME_SUSPENDED + oparg; 3549 _PyFrame_SetStackPointer(frame, stack_pointer - 1); 3550 int err = _Py_call_instrumentation_arg( 3551 tstate, PY_MONITORING_EVENT_PY_YIELD, 3552 frame, this_instr, retval); 3553 if (err) goto error; 3554 tstate->exc_info = gen->gi_exc_state.previous_item; 3555 gen->gi_exc_state.previous_item = NULL; 3556 _Py_LeaveRecursiveCallPy(tstate); 3557 _PyInterpreterFrame *gen_frame = frame; 3558 frame = tstate->current_frame = frame->previous; 3559 gen_frame->previous = NULL; 3560 _PyFrame_StackPush(frame, retval); 3561 /* We don't know which of these is relevant here, so keep them equal */ 3562 assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); 3563 LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); 3564 goto resume_frame; 3565 } 3566 TARGET(INTERPRETER_EXIT)3567 TARGET(INTERPRETER_EXIT) { 3568 frame->instr_ptr = next_instr; 3569 next_instr += 1; 3570 INSTRUCTION_STATS(INTERPRETER_EXIT); 3571 PyObject *retval; 3572 retval = stack_pointer[-1]; 3573 assert(frame == &entry_frame); 3574 assert(_PyFrame_IsIncomplete(frame)); 3575 /* Restore previous frame and return. */ 3576 tstate->current_frame = frame->previous; 3577 assert(!_PyErr_Occurred(tstate)); 3578 tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; 3579 return retval; 3580 } 3581 TARGET(IS_OP)3582 TARGET(IS_OP) { 3583 frame->instr_ptr = next_instr; 3584 next_instr += 1; 3585 INSTRUCTION_STATS(IS_OP); 3586 PyObject *right; 3587 PyObject *left; 3588 PyObject *b; 3589 right = stack_pointer[-1]; 3590 left = stack_pointer[-2]; 3591 int res = Py_Is(left, right) ^ oparg; 3592 Py_DECREF(left); 3593 Py_DECREF(right); 3594 b = res ? Py_True : Py_False; 3595 stack_pointer[-2] = b; 3596 stack_pointer += -1; 3597 DISPATCH(); 3598 } 3599 TARGET(JUMP_BACKWARD)3600 TARGET(JUMP_BACKWARD) { 3601 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3602 (void)this_instr; 3603 next_instr += 2; 3604 INSTRUCTION_STATS(JUMP_BACKWARD); 3605 /* Skip 1 cache entry */ 3606 CHECK_EVAL_BREAKER(); 3607 assert(oparg <= INSTR_OFFSET()); 3608 JUMPBY(-oparg); 3609 #ifdef _Py_TIER2 3610 #if ENABLE_SPECIALIZATION 3611 _Py_BackoffCounter counter = this_instr[1].counter; 3612 if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { 3613 _Py_CODEUNIT *start = this_instr; 3614 /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ 3615 while (oparg > 255) { 3616 oparg >>= 8; 3617 start--; 3618 } 3619 _PyExecutorObject *executor; 3620 int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); 3621 if (optimized < 0) goto error; 3622 if (optimized) { 3623 assert(tstate->previous_executor == NULL); 3624 tstate->previous_executor = Py_None; 3625 GOTO_TIER_TWO(executor); 3626 } 3627 else { 3628 this_instr[1].counter = restart_backoff_counter(counter); 3629 } 3630 } 3631 else { 3632 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 3633 } 3634 #endif /* ENABLE_SPECIALIZATION */ 3635 #endif /* _Py_TIER2 */ 3636 DISPATCH(); 3637 } 3638 TARGET(JUMP_BACKWARD_NO_INTERRUPT)3639 TARGET(JUMP_BACKWARD_NO_INTERRUPT) { 3640 frame->instr_ptr = next_instr; 3641 next_instr += 1; 3642 INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); 3643 /* This bytecode is used in the `yield from` or `await` loop. 3644 * If there is an interrupt, we want it handled in the innermost 3645 * generator or coroutine, so we deliberately do not check it here. 3646 * (see bpo-30039). 3647 */ 3648 JUMPBY(-oparg); 3649 DISPATCH(); 3650 } 3651 TARGET(JUMP_FORWARD)3652 TARGET(JUMP_FORWARD) { 3653 frame->instr_ptr = next_instr; 3654 next_instr += 1; 3655 INSTRUCTION_STATS(JUMP_FORWARD); 3656 JUMPBY(oparg); 3657 DISPATCH(); 3658 } 3659 TARGET(LIST_APPEND)3660 TARGET(LIST_APPEND) { 3661 frame->instr_ptr = next_instr; 3662 next_instr += 1; 3663 INSTRUCTION_STATS(LIST_APPEND); 3664 PyObject *v; 3665 PyObject *list; 3666 v = stack_pointer[-1]; 3667 list = stack_pointer[-2 - (oparg-1)]; 3668 if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; 3669 stack_pointer += -1; 3670 DISPATCH(); 3671 } 3672 TARGET(LIST_EXTEND)3673 TARGET(LIST_EXTEND) { 3674 frame->instr_ptr = next_instr; 3675 next_instr += 1; 3676 INSTRUCTION_STATS(LIST_EXTEND); 3677 PyObject *iterable; 3678 PyObject *list; 3679 iterable = stack_pointer[-1]; 3680 list = stack_pointer[-2 - (oparg-1)]; 3681 PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); 3682 if (none_val == NULL) { 3683 if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && 3684 (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) 3685 { 3686 _PyErr_Clear(tstate); 3687 _PyErr_Format(tstate, PyExc_TypeError, 3688 "Value after * must be an iterable, not %.200s", 3689 Py_TYPE(iterable)->tp_name); 3690 } 3691 Py_DECREF(iterable); 3692 if (true) goto pop_1_error; 3693 } 3694 assert(Py_IsNone(none_val)); 3695 Py_DECREF(iterable); 3696 stack_pointer += -1; 3697 DISPATCH(); 3698 } 3699 TARGET(LOAD_ASSERTION_ERROR)3700 TARGET(LOAD_ASSERTION_ERROR) { 3701 frame->instr_ptr = next_instr; 3702 next_instr += 1; 3703 INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); 3704 PyObject *value; 3705 value = Py_NewRef(PyExc_AssertionError); 3706 stack_pointer[0] = value; 3707 stack_pointer += 1; 3708 DISPATCH(); 3709 } 3710 TARGET(LOAD_ATTR)3711 TARGET(LOAD_ATTR) { 3712 frame->instr_ptr = next_instr; 3713 next_instr += 10; 3714 INSTRUCTION_STATS(LOAD_ATTR); 3715 PREDICTED(LOAD_ATTR); 3716 _Py_CODEUNIT *this_instr = next_instr - 10; 3717 (void)this_instr; 3718 PyObject *owner; 3719 PyObject *attr; 3720 PyObject *self_or_null = NULL; 3721 // _SPECIALIZE_LOAD_ATTR 3722 owner = stack_pointer[-1]; 3723 { 3724 uint16_t counter = read_u16(&this_instr[1].cache); 3725 (void)counter; 3726 #if ENABLE_SPECIALIZATION 3727 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 3728 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); 3729 next_instr = this_instr; 3730 _Py_Specialize_LoadAttr(owner, next_instr, name); 3731 DISPATCH_SAME_OPARG(); 3732 } 3733 STAT_INC(LOAD_ATTR, deferred); 3734 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 3735 #endif /* ENABLE_SPECIALIZATION */ 3736 } 3737 /* Skip 8 cache entries */ 3738 // _LOAD_ATTR 3739 { 3740 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); 3741 if (oparg & 1) { 3742 /* Designed to work in tandem with CALL, pushes two values. */ 3743 attr = NULL; 3744 if (_PyObject_GetMethod(owner, name, &attr)) { 3745 /* We can bypass temporary bound method object. 3746 meth is unbound method and obj is self. 3747 meth | self | arg1 | ... | argN 3748 */ 3749 assert(attr != NULL); // No errors on this branch 3750 self_or_null = owner; // Transfer ownership 3751 } 3752 else { 3753 /* meth is not an unbound method (but a regular attr, or 3754 something was returned by a descriptor protocol). Set 3755 the second element of the stack to NULL, to signal 3756 CALL that it's not a method call. 3757 meth | NULL | arg1 | ... | argN 3758 */ 3759 Py_DECREF(owner); 3760 if (attr == NULL) goto pop_1_error; 3761 self_or_null = NULL; 3762 } 3763 } 3764 else { 3765 /* Classic, pushes one value. */ 3766 attr = PyObject_GetAttr(owner, name); 3767 Py_DECREF(owner); 3768 if (attr == NULL) goto pop_1_error; 3769 } 3770 } 3771 stack_pointer[-1] = attr; 3772 if (oparg & 1) stack_pointer[0] = self_or_null; 3773 stack_pointer += (oparg & 1); 3774 DISPATCH(); 3775 } 3776 TARGET(LOAD_ATTR_CLASS)3777 TARGET(LOAD_ATTR_CLASS) { 3778 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3779 next_instr += 10; 3780 INSTRUCTION_STATS(LOAD_ATTR_CLASS); 3781 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 3782 PyObject *owner; 3783 PyObject *attr; 3784 PyObject *null = NULL; 3785 /* Skip 1 cache entry */ 3786 // _CHECK_ATTR_CLASS 3787 owner = stack_pointer[-1]; 3788 { 3789 uint32_t type_version = read_u32(&this_instr[2].cache); 3790 DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); 3791 assert(type_version != 0); 3792 DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); 3793 } 3794 /* Skip 2 cache entries */ 3795 // _LOAD_ATTR_CLASS 3796 { 3797 PyObject *descr = read_obj(&this_instr[6].cache); 3798 STAT_INC(LOAD_ATTR, hit); 3799 assert(descr != NULL); 3800 attr = Py_NewRef(descr); 3801 null = NULL; 3802 Py_DECREF(owner); 3803 } 3804 stack_pointer[-1] = attr; 3805 if (oparg & 1) stack_pointer[0] = null; 3806 stack_pointer += (oparg & 1); 3807 DISPATCH(); 3808 } 3809 TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN)3810 TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { 3811 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3812 next_instr += 10; 3813 INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); 3814 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 3815 PyObject *owner; 3816 /* Skip 1 cache entry */ 3817 owner = stack_pointer[-1]; 3818 uint32_t type_version = read_u32(&this_instr[2].cache); 3819 uint32_t func_version = read_u32(&this_instr[4].cache); 3820 PyObject *getattribute = read_obj(&this_instr[6].cache); 3821 assert((oparg & 1) == 0); 3822 DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); 3823 PyTypeObject *cls = Py_TYPE(owner); 3824 assert(type_version != 0); 3825 DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); 3826 assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); 3827 PyFunctionObject *f = (PyFunctionObject *)getattribute; 3828 assert(func_version != 0); 3829 DEOPT_IF(f->func_version != func_version, LOAD_ATTR); 3830 PyCodeObject *code = (PyCodeObject *)f->func_code; 3831 assert(code->co_argcount == 2); 3832 DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); 3833 STAT_INC(LOAD_ATTR, hit); 3834 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); 3835 Py_INCREF(f); 3836 _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); 3837 // Manipulate stack directly because we exit with DISPATCH_INLINED(). 3838 STACK_SHRINK(1); 3839 new_frame->localsplus[0] = owner; 3840 new_frame->localsplus[1] = Py_NewRef(name); 3841 frame->return_offset = (uint16_t)(next_instr - this_instr); 3842 DISPATCH_INLINED(new_frame); 3843 } 3844 TARGET(LOAD_ATTR_INSTANCE_VALUE)3845 TARGET(LOAD_ATTR_INSTANCE_VALUE) { 3846 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3847 next_instr += 10; 3848 INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); 3849 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 3850 PyObject *owner; 3851 PyObject *attr; 3852 PyObject *null = NULL; 3853 /* Skip 1 cache entry */ 3854 // _GUARD_TYPE_VERSION 3855 owner = stack_pointer[-1]; 3856 { 3857 uint32_t type_version = read_u32(&this_instr[2].cache); 3858 PyTypeObject *tp = Py_TYPE(owner); 3859 assert(type_version != 0); 3860 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); 3861 } 3862 // _CHECK_MANAGED_OBJECT_HAS_VALUES 3863 { 3864 assert(Py_TYPE(owner)->tp_dictoffset < 0); 3865 assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); 3866 DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); 3867 } 3868 // _LOAD_ATTR_INSTANCE_VALUE 3869 { 3870 uint16_t index = read_u16(&this_instr[4].cache); 3871 attr = _PyObject_InlineValues(owner)->values[index]; 3872 DEOPT_IF(attr == NULL, LOAD_ATTR); 3873 STAT_INC(LOAD_ATTR, hit); 3874 Py_INCREF(attr); 3875 null = NULL; 3876 Py_DECREF(owner); 3877 } 3878 /* Skip 5 cache entries */ 3879 stack_pointer[-1] = attr; 3880 if (oparg & 1) stack_pointer[0] = null; 3881 stack_pointer += (oparg & 1); 3882 DISPATCH(); 3883 } 3884 TARGET(LOAD_ATTR_METHOD_LAZY_DICT)3885 TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { 3886 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3887 next_instr += 10; 3888 INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); 3889 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 3890 PyObject *owner; 3891 PyObject *attr; 3892 PyObject *self = NULL; 3893 /* Skip 1 cache entry */ 3894 // _GUARD_TYPE_VERSION 3895 owner = stack_pointer[-1]; 3896 { 3897 uint32_t type_version = read_u32(&this_instr[2].cache); 3898 PyTypeObject *tp = Py_TYPE(owner); 3899 assert(type_version != 0); 3900 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); 3901 } 3902 // _CHECK_ATTR_METHOD_LAZY_DICT 3903 { 3904 uint16_t dictoffset = read_u16(&this_instr[4].cache); 3905 char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; 3906 PyObject *dict = *(PyObject **)ptr; 3907 /* This object has a __dict__, just not yet created */ 3908 DEOPT_IF(dict != NULL, LOAD_ATTR); 3909 } 3910 /* Skip 1 cache entry */ 3911 // _LOAD_ATTR_METHOD_LAZY_DICT 3912 { 3913 PyObject *descr = read_obj(&this_instr[6].cache); 3914 assert(oparg & 1); 3915 STAT_INC(LOAD_ATTR, hit); 3916 assert(descr != NULL); 3917 assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); 3918 attr = Py_NewRef(descr); 3919 self = owner; 3920 } 3921 stack_pointer[-1] = attr; 3922 stack_pointer[0] = self; 3923 stack_pointer += 1; 3924 DISPATCH(); 3925 } 3926 TARGET(LOAD_ATTR_METHOD_NO_DICT)3927 TARGET(LOAD_ATTR_METHOD_NO_DICT) { 3928 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3929 next_instr += 10; 3930 INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); 3931 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 3932 PyObject *owner; 3933 PyObject *attr; 3934 PyObject *self = NULL; 3935 /* Skip 1 cache entry */ 3936 // _GUARD_TYPE_VERSION 3937 owner = stack_pointer[-1]; 3938 { 3939 uint32_t type_version = read_u32(&this_instr[2].cache); 3940 PyTypeObject *tp = Py_TYPE(owner); 3941 assert(type_version != 0); 3942 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); 3943 } 3944 /* Skip 2 cache entries */ 3945 // _LOAD_ATTR_METHOD_NO_DICT 3946 { 3947 PyObject *descr = read_obj(&this_instr[6].cache); 3948 assert(oparg & 1); 3949 assert(Py_TYPE(owner)->tp_dictoffset == 0); 3950 STAT_INC(LOAD_ATTR, hit); 3951 assert(descr != NULL); 3952 assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); 3953 attr = Py_NewRef(descr); 3954 self = owner; 3955 } 3956 stack_pointer[-1] = attr; 3957 stack_pointer[0] = self; 3958 stack_pointer += 1; 3959 DISPATCH(); 3960 } 3961 TARGET(LOAD_ATTR_METHOD_WITH_VALUES)3962 TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { 3963 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 3964 next_instr += 10; 3965 INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); 3966 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 3967 PyObject *owner; 3968 PyObject *attr; 3969 PyObject *self = NULL; 3970 /* Skip 1 cache entry */ 3971 // _GUARD_TYPE_VERSION 3972 owner = stack_pointer[-1]; 3973 { 3974 uint32_t type_version = read_u32(&this_instr[2].cache); 3975 PyTypeObject *tp = Py_TYPE(owner); 3976 assert(type_version != 0); 3977 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); 3978 } 3979 // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 3980 { 3981 assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); 3982 DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); 3983 } 3984 // _GUARD_KEYS_VERSION 3985 { 3986 uint32_t keys_version = read_u32(&this_instr[4].cache); 3987 PyTypeObject *owner_cls = Py_TYPE(owner); 3988 PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; 3989 DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); 3990 } 3991 // _LOAD_ATTR_METHOD_WITH_VALUES 3992 { 3993 PyObject *descr = read_obj(&this_instr[6].cache); 3994 assert(oparg & 1); 3995 /* Cached method object */ 3996 STAT_INC(LOAD_ATTR, hit); 3997 assert(descr != NULL); 3998 attr = Py_NewRef(descr); 3999 assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); 4000 self = owner; 4001 } 4002 stack_pointer[-1] = attr; 4003 stack_pointer[0] = self; 4004 stack_pointer += 1; 4005 DISPATCH(); 4006 } 4007 TARGET(LOAD_ATTR_MODULE)4008 TARGET(LOAD_ATTR_MODULE) { 4009 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4010 next_instr += 10; 4011 INSTRUCTION_STATS(LOAD_ATTR_MODULE); 4012 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 4013 PyObject *owner; 4014 PyObject *attr; 4015 PyObject *null = NULL; 4016 /* Skip 1 cache entry */ 4017 // _CHECK_ATTR_MODULE 4018 owner = stack_pointer[-1]; 4019 { 4020 uint32_t dict_version = read_u32(&this_instr[2].cache); 4021 DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); 4022 PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; 4023 assert(dict != NULL); 4024 DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR); 4025 } 4026 // _LOAD_ATTR_MODULE 4027 { 4028 uint16_t index = read_u16(&this_instr[4].cache); 4029 PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; 4030 assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); 4031 assert(index < dict->ma_keys->dk_nentries); 4032 PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; 4033 attr = ep->me_value; 4034 DEOPT_IF(attr == NULL, LOAD_ATTR); 4035 STAT_INC(LOAD_ATTR, hit); 4036 Py_INCREF(attr); 4037 null = NULL; 4038 Py_DECREF(owner); 4039 } 4040 /* Skip 5 cache entries */ 4041 stack_pointer[-1] = attr; 4042 if (oparg & 1) stack_pointer[0] = null; 4043 stack_pointer += (oparg & 1); 4044 DISPATCH(); 4045 } 4046 TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT)4047 TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { 4048 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4049 next_instr += 10; 4050 INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); 4051 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 4052 PyObject *owner; 4053 PyObject *attr; 4054 /* Skip 1 cache entry */ 4055 // _GUARD_TYPE_VERSION 4056 owner = stack_pointer[-1]; 4057 { 4058 uint32_t type_version = read_u32(&this_instr[2].cache); 4059 PyTypeObject *tp = Py_TYPE(owner); 4060 assert(type_version != 0); 4061 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); 4062 } 4063 /* Skip 2 cache entries */ 4064 // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 4065 { 4066 PyObject *descr = read_obj(&this_instr[6].cache); 4067 assert((oparg & 1) == 0); 4068 assert(Py_TYPE(owner)->tp_dictoffset == 0); 4069 STAT_INC(LOAD_ATTR, hit); 4070 assert(descr != NULL); 4071 Py_DECREF(owner); 4072 attr = Py_NewRef(descr); 4073 } 4074 stack_pointer[-1] = attr; 4075 DISPATCH(); 4076 } 4077 TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES)4078 TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { 4079 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4080 next_instr += 10; 4081 INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); 4082 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 4083 PyObject *owner; 4084 PyObject *attr; 4085 /* Skip 1 cache entry */ 4086 // _GUARD_TYPE_VERSION 4087 owner = stack_pointer[-1]; 4088 { 4089 uint32_t type_version = read_u32(&this_instr[2].cache); 4090 PyTypeObject *tp = Py_TYPE(owner); 4091 assert(type_version != 0); 4092 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); 4093 } 4094 // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 4095 { 4096 assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); 4097 DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); 4098 } 4099 // _GUARD_KEYS_VERSION 4100 { 4101 uint32_t keys_version = read_u32(&this_instr[4].cache); 4102 PyTypeObject *owner_cls = Py_TYPE(owner); 4103 PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; 4104 DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); 4105 } 4106 // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 4107 { 4108 PyObject *descr = read_obj(&this_instr[6].cache); 4109 assert((oparg & 1) == 0); 4110 STAT_INC(LOAD_ATTR, hit); 4111 assert(descr != NULL); 4112 Py_DECREF(owner); 4113 attr = Py_NewRef(descr); 4114 } 4115 stack_pointer[-1] = attr; 4116 DISPATCH(); 4117 } 4118 TARGET(LOAD_ATTR_PROPERTY)4119 TARGET(LOAD_ATTR_PROPERTY) { 4120 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4121 next_instr += 10; 4122 INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); 4123 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 4124 PyObject *owner; 4125 /* Skip 1 cache entry */ 4126 owner = stack_pointer[-1]; 4127 uint32_t type_version = read_u32(&this_instr[2].cache); 4128 uint32_t func_version = read_u32(&this_instr[4].cache); 4129 PyObject *fget = read_obj(&this_instr[6].cache); 4130 assert((oparg & 1) == 0); 4131 DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); 4132 PyTypeObject *cls = Py_TYPE(owner); 4133 assert(type_version != 0); 4134 DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); 4135 assert(Py_IS_TYPE(fget, &PyFunction_Type)); 4136 PyFunctionObject *f = (PyFunctionObject *)fget; 4137 assert(func_version != 0); 4138 DEOPT_IF(f->func_version != func_version, LOAD_ATTR); 4139 PyCodeObject *code = (PyCodeObject *)f->func_code; 4140 assert(code->co_argcount == 1); 4141 DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); 4142 STAT_INC(LOAD_ATTR, hit); 4143 Py_INCREF(fget); 4144 _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); 4145 // Manipulate stack directly because we exit with DISPATCH_INLINED(). 4146 STACK_SHRINK(1); 4147 new_frame->localsplus[0] = owner; 4148 frame->return_offset = (uint16_t)(next_instr - this_instr); 4149 DISPATCH_INLINED(new_frame); 4150 } 4151 TARGET(LOAD_ATTR_SLOT)4152 TARGET(LOAD_ATTR_SLOT) { 4153 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4154 next_instr += 10; 4155 INSTRUCTION_STATS(LOAD_ATTR_SLOT); 4156 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 4157 PyObject *owner; 4158 PyObject *attr; 4159 PyObject *null = NULL; 4160 /* Skip 1 cache entry */ 4161 // _GUARD_TYPE_VERSION 4162 owner = stack_pointer[-1]; 4163 { 4164 uint32_t type_version = read_u32(&this_instr[2].cache); 4165 PyTypeObject *tp = Py_TYPE(owner); 4166 assert(type_version != 0); 4167 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); 4168 } 4169 // _LOAD_ATTR_SLOT 4170 { 4171 uint16_t index = read_u16(&this_instr[4].cache); 4172 char *addr = (char *)owner + index; 4173 attr = *(PyObject **)addr; 4174 DEOPT_IF(attr == NULL, LOAD_ATTR); 4175 STAT_INC(LOAD_ATTR, hit); 4176 Py_INCREF(attr); 4177 null = NULL; 4178 Py_DECREF(owner); 4179 } 4180 /* Skip 5 cache entries */ 4181 stack_pointer[-1] = attr; 4182 if (oparg & 1) stack_pointer[0] = null; 4183 stack_pointer += (oparg & 1); 4184 DISPATCH(); 4185 } 4186 TARGET(LOAD_ATTR_WITH_HINT)4187 TARGET(LOAD_ATTR_WITH_HINT) { 4188 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4189 next_instr += 10; 4190 INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); 4191 static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); 4192 PyObject *owner; 4193 PyObject *attr; 4194 PyObject *null = NULL; 4195 /* Skip 1 cache entry */ 4196 // _GUARD_TYPE_VERSION 4197 owner = stack_pointer[-1]; 4198 { 4199 uint32_t type_version = read_u32(&this_instr[2].cache); 4200 PyTypeObject *tp = Py_TYPE(owner); 4201 assert(type_version != 0); 4202 DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); 4203 } 4204 // _CHECK_ATTR_WITH_HINT 4205 { 4206 assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); 4207 PyDictObject *dict = _PyObject_GetManagedDict(owner); 4208 DEOPT_IF(dict == NULL, LOAD_ATTR); 4209 assert(PyDict_CheckExact((PyObject *)dict)); 4210 } 4211 // _LOAD_ATTR_WITH_HINT 4212 { 4213 uint16_t hint = read_u16(&this_instr[4].cache); 4214 PyDictObject *dict = _PyObject_GetManagedDict(owner); 4215 DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); 4216 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); 4217 if (DK_IS_UNICODE(dict->ma_keys)) { 4218 PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; 4219 DEOPT_IF(ep->me_key != name, LOAD_ATTR); 4220 attr = ep->me_value; 4221 } 4222 else { 4223 PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; 4224 DEOPT_IF(ep->me_key != name, LOAD_ATTR); 4225 attr = ep->me_value; 4226 } 4227 DEOPT_IF(attr == NULL, LOAD_ATTR); 4228 STAT_INC(LOAD_ATTR, hit); 4229 Py_INCREF(attr); 4230 null = NULL; 4231 Py_DECREF(owner); 4232 } 4233 /* Skip 5 cache entries */ 4234 stack_pointer[-1] = attr; 4235 if (oparg & 1) stack_pointer[0] = null; 4236 stack_pointer += (oparg & 1); 4237 DISPATCH(); 4238 } 4239 TARGET(LOAD_BUILD_CLASS)4240 TARGET(LOAD_BUILD_CLASS) { 4241 frame->instr_ptr = next_instr; 4242 next_instr += 1; 4243 INSTRUCTION_STATS(LOAD_BUILD_CLASS); 4244 PyObject *bc; 4245 if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; 4246 if (bc == NULL) { 4247 _PyErr_SetString(tstate, PyExc_NameError, 4248 "__build_class__ not found"); 4249 if (true) goto error; 4250 } 4251 stack_pointer[0] = bc; 4252 stack_pointer += 1; 4253 DISPATCH(); 4254 } 4255 TARGET(LOAD_CONST)4256 TARGET(LOAD_CONST) { 4257 frame->instr_ptr = next_instr; 4258 next_instr += 1; 4259 INSTRUCTION_STATS(LOAD_CONST); 4260 PyObject *value; 4261 value = GETITEM(FRAME_CO_CONSTS, oparg); 4262 Py_INCREF(value); 4263 stack_pointer[0] = value; 4264 stack_pointer += 1; 4265 DISPATCH(); 4266 } 4267 TARGET(LOAD_DEREF)4268 TARGET(LOAD_DEREF) { 4269 frame->instr_ptr = next_instr; 4270 next_instr += 1; 4271 INSTRUCTION_STATS(LOAD_DEREF); 4272 PyObject *value; 4273 PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); 4274 value = PyCell_GetRef(cell); 4275 if (value == NULL) { 4276 _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); 4277 if (true) goto error; 4278 } 4279 stack_pointer[0] = value; 4280 stack_pointer += 1; 4281 DISPATCH(); 4282 } 4283 TARGET(LOAD_FAST)4284 TARGET(LOAD_FAST) { 4285 frame->instr_ptr = next_instr; 4286 next_instr += 1; 4287 INSTRUCTION_STATS(LOAD_FAST); 4288 PyObject *value; 4289 value = GETLOCAL(oparg); 4290 assert(value != NULL); 4291 Py_INCREF(value); 4292 stack_pointer[0] = value; 4293 stack_pointer += 1; 4294 DISPATCH(); 4295 } 4296 TARGET(LOAD_FAST_AND_CLEAR)4297 TARGET(LOAD_FAST_AND_CLEAR) { 4298 frame->instr_ptr = next_instr; 4299 next_instr += 1; 4300 INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); 4301 PyObject *value; 4302 value = GETLOCAL(oparg); 4303 // do not use SETLOCAL here, it decrefs the old value 4304 GETLOCAL(oparg) = NULL; 4305 stack_pointer[0] = value; 4306 stack_pointer += 1; 4307 DISPATCH(); 4308 } 4309 TARGET(LOAD_FAST_CHECK)4310 TARGET(LOAD_FAST_CHECK) { 4311 frame->instr_ptr = next_instr; 4312 next_instr += 1; 4313 INSTRUCTION_STATS(LOAD_FAST_CHECK); 4314 PyObject *value; 4315 value = GETLOCAL(oparg); 4316 if (value == NULL) { 4317 _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, 4318 UNBOUNDLOCAL_ERROR_MSG, 4319 PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) 4320 ); 4321 if (1) goto error; 4322 } 4323 Py_INCREF(value); 4324 stack_pointer[0] = value; 4325 stack_pointer += 1; 4326 DISPATCH(); 4327 } 4328 TARGET(LOAD_FAST_LOAD_FAST)4329 TARGET(LOAD_FAST_LOAD_FAST) { 4330 frame->instr_ptr = next_instr; 4331 next_instr += 1; 4332 INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); 4333 PyObject *value1; 4334 PyObject *value2; 4335 uint32_t oparg1 = oparg >> 4; 4336 uint32_t oparg2 = oparg & 15; 4337 value1 = GETLOCAL(oparg1); 4338 value2 = GETLOCAL(oparg2); 4339 Py_INCREF(value1); 4340 Py_INCREF(value2); 4341 stack_pointer[0] = value1; 4342 stack_pointer[1] = value2; 4343 stack_pointer += 2; 4344 DISPATCH(); 4345 } 4346 TARGET(LOAD_FROM_DICT_OR_DEREF)4347 TARGET(LOAD_FROM_DICT_OR_DEREF) { 4348 frame->instr_ptr = next_instr; 4349 next_instr += 1; 4350 INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); 4351 PyObject *class_dict; 4352 PyObject *value; 4353 class_dict = stack_pointer[-1]; 4354 PyObject *name; 4355 assert(class_dict); 4356 assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); 4357 name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); 4358 if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { 4359 goto error; 4360 } 4361 if (!value) { 4362 PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); 4363 value = PyCell_GetRef(cell); 4364 if (value == NULL) { 4365 _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); 4366 goto error; 4367 } 4368 } 4369 Py_DECREF(class_dict); 4370 stack_pointer[-1] = value; 4371 DISPATCH(); 4372 } 4373 TARGET(LOAD_FROM_DICT_OR_GLOBALS)4374 TARGET(LOAD_FROM_DICT_OR_GLOBALS) { 4375 frame->instr_ptr = next_instr; 4376 next_instr += 1; 4377 INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); 4378 PyObject *mod_or_class_dict; 4379 PyObject *v; 4380 mod_or_class_dict = stack_pointer[-1]; 4381 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 4382 if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { 4383 goto error; 4384 } 4385 if (v == NULL) { 4386 if (PyDict_CheckExact(GLOBALS()) 4387 && PyDict_CheckExact(BUILTINS())) 4388 { 4389 v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), 4390 (PyDictObject *)BUILTINS(), 4391 name); 4392 if (v == NULL) { 4393 if (!_PyErr_Occurred(tstate)) { 4394 /* _PyDict_LoadGlobal() returns NULL without raising 4395 * an exception if the key doesn't exist */ 4396 _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, 4397 NAME_ERROR_MSG, name); 4398 } 4399 goto error; 4400 } 4401 } 4402 else { 4403 /* Slow-path if globals or builtins is not a dict */ 4404 /* namespace 1: globals */ 4405 if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto pop_1_error; 4406 if (v == NULL) { 4407 /* namespace 2: builtins */ 4408 if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto pop_1_error; 4409 if (v == NULL) { 4410 _PyEval_FormatExcCheckArg( 4411 tstate, PyExc_NameError, 4412 NAME_ERROR_MSG, name); 4413 if (true) goto pop_1_error; 4414 } 4415 } 4416 } 4417 } 4418 Py_DECREF(mod_or_class_dict); 4419 stack_pointer[-1] = v; 4420 DISPATCH(); 4421 } 4422 TARGET(LOAD_GLOBAL)4423 TARGET(LOAD_GLOBAL) { 4424 frame->instr_ptr = next_instr; 4425 next_instr += 5; 4426 INSTRUCTION_STATS(LOAD_GLOBAL); 4427 PREDICTED(LOAD_GLOBAL); 4428 _Py_CODEUNIT *this_instr = next_instr - 5; 4429 (void)this_instr; 4430 PyObject *res; 4431 PyObject *null = NULL; 4432 // _SPECIALIZE_LOAD_GLOBAL 4433 { 4434 uint16_t counter = read_u16(&this_instr[1].cache); 4435 (void)counter; 4436 #if ENABLE_SPECIALIZATION 4437 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 4438 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); 4439 next_instr = this_instr; 4440 _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); 4441 DISPATCH_SAME_OPARG(); 4442 } 4443 STAT_INC(LOAD_GLOBAL, deferred); 4444 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 4445 #endif /* ENABLE_SPECIALIZATION */ 4446 } 4447 /* Skip 1 cache entry */ 4448 /* Skip 1 cache entry */ 4449 /* Skip 1 cache entry */ 4450 // _LOAD_GLOBAL 4451 { 4452 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); 4453 if (PyDict_CheckExact(GLOBALS()) 4454 && PyDict_CheckExact(BUILTINS())) 4455 { 4456 res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), 4457 (PyDictObject *)BUILTINS(), 4458 name); 4459 if (res == NULL) { 4460 if (!_PyErr_Occurred(tstate)) { 4461 /* _PyDict_LoadGlobal() returns NULL without raising 4462 * an exception if the key doesn't exist */ 4463 _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, 4464 NAME_ERROR_MSG, name); 4465 } 4466 if (true) goto error; 4467 } 4468 } 4469 else { 4470 /* Slow-path if globals or builtins is not a dict */ 4471 /* namespace 1: globals */ 4472 if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; 4473 if (res == NULL) { 4474 /* namespace 2: builtins */ 4475 if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; 4476 if (res == NULL) { 4477 _PyEval_FormatExcCheckArg( 4478 tstate, PyExc_NameError, 4479 NAME_ERROR_MSG, name); 4480 if (true) goto error; 4481 } 4482 } 4483 } 4484 null = NULL; 4485 } 4486 stack_pointer[0] = res; 4487 if (oparg & 1) stack_pointer[1] = null; 4488 stack_pointer += 1 + (oparg & 1); 4489 DISPATCH(); 4490 } 4491 TARGET(LOAD_GLOBAL_BUILTIN)4492 TARGET(LOAD_GLOBAL_BUILTIN) { 4493 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4494 next_instr += 5; 4495 INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); 4496 static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); 4497 PyObject *res; 4498 PyObject *null = NULL; 4499 /* Skip 1 cache entry */ 4500 // _GUARD_GLOBALS_VERSION 4501 { 4502 uint16_t version = read_u16(&this_instr[2].cache); 4503 PyDictObject *dict = (PyDictObject *)GLOBALS(); 4504 DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); 4505 DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); 4506 assert(DK_IS_UNICODE(dict->ma_keys)); 4507 } 4508 // _GUARD_BUILTINS_VERSION 4509 { 4510 uint16_t version = read_u16(&this_instr[3].cache); 4511 PyDictObject *dict = (PyDictObject *)BUILTINS(); 4512 DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); 4513 DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); 4514 assert(DK_IS_UNICODE(dict->ma_keys)); 4515 } 4516 // _LOAD_GLOBAL_BUILTINS 4517 { 4518 uint16_t index = read_u16(&this_instr[4].cache); 4519 PyDictObject *bdict = (PyDictObject *)BUILTINS(); 4520 PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); 4521 res = entries[index].me_value; 4522 DEOPT_IF(res == NULL, LOAD_GLOBAL); 4523 Py_INCREF(res); 4524 STAT_INC(LOAD_GLOBAL, hit); 4525 null = NULL; 4526 } 4527 stack_pointer[0] = res; 4528 if (oparg & 1) stack_pointer[1] = null; 4529 stack_pointer += 1 + (oparg & 1); 4530 DISPATCH(); 4531 } 4532 TARGET(LOAD_GLOBAL_MODULE)4533 TARGET(LOAD_GLOBAL_MODULE) { 4534 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4535 next_instr += 5; 4536 INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); 4537 static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); 4538 PyObject *res; 4539 PyObject *null = NULL; 4540 /* Skip 1 cache entry */ 4541 // _GUARD_GLOBALS_VERSION 4542 { 4543 uint16_t version = read_u16(&this_instr[2].cache); 4544 PyDictObject *dict = (PyDictObject *)GLOBALS(); 4545 DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); 4546 DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); 4547 assert(DK_IS_UNICODE(dict->ma_keys)); 4548 } 4549 /* Skip 1 cache entry */ 4550 // _LOAD_GLOBAL_MODULE 4551 { 4552 uint16_t index = read_u16(&this_instr[4].cache); 4553 PyDictObject *dict = (PyDictObject *)GLOBALS(); 4554 PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); 4555 res = entries[index].me_value; 4556 DEOPT_IF(res == NULL, LOAD_GLOBAL); 4557 Py_INCREF(res); 4558 STAT_INC(LOAD_GLOBAL, hit); 4559 null = NULL; 4560 } 4561 stack_pointer[0] = res; 4562 if (oparg & 1) stack_pointer[1] = null; 4563 stack_pointer += 1 + (oparg & 1); 4564 DISPATCH(); 4565 } 4566 TARGET(LOAD_LOCALS)4567 TARGET(LOAD_LOCALS) { 4568 frame->instr_ptr = next_instr; 4569 next_instr += 1; 4570 INSTRUCTION_STATS(LOAD_LOCALS); 4571 PyObject *locals; 4572 locals = LOCALS(); 4573 if (locals == NULL) { 4574 _PyErr_SetString(tstate, PyExc_SystemError, 4575 "no locals found"); 4576 if (true) goto error; 4577 } 4578 Py_INCREF(locals); 4579 stack_pointer[0] = locals; 4580 stack_pointer += 1; 4581 DISPATCH(); 4582 } 4583 TARGET(LOAD_NAME)4584 TARGET(LOAD_NAME) { 4585 frame->instr_ptr = next_instr; 4586 next_instr += 1; 4587 INSTRUCTION_STATS(LOAD_NAME); 4588 PyObject *v; 4589 PyObject *mod_or_class_dict = LOCALS(); 4590 if (mod_or_class_dict == NULL) { 4591 _PyErr_SetString(tstate, PyExc_SystemError, 4592 "no locals found"); 4593 if (true) goto error; 4594 } 4595 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 4596 if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { 4597 goto error; 4598 } 4599 if (v == NULL) { 4600 if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { 4601 goto error; 4602 } 4603 if (v == NULL) { 4604 if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { 4605 goto error; 4606 } 4607 if (v == NULL) { 4608 _PyEval_FormatExcCheckArg( 4609 tstate, PyExc_NameError, 4610 NAME_ERROR_MSG, name); 4611 goto error; 4612 } 4613 } 4614 } 4615 stack_pointer[0] = v; 4616 stack_pointer += 1; 4617 DISPATCH(); 4618 } 4619 TARGET(LOAD_SUPER_ATTR)4620 TARGET(LOAD_SUPER_ATTR) { 4621 frame->instr_ptr = next_instr; 4622 next_instr += 2; 4623 INSTRUCTION_STATS(LOAD_SUPER_ATTR); 4624 PREDICTED(LOAD_SUPER_ATTR); 4625 _Py_CODEUNIT *this_instr = next_instr - 2; 4626 (void)this_instr; 4627 PyObject *class; 4628 PyObject *global_super; 4629 PyObject *self; 4630 PyObject *attr; 4631 PyObject *null = NULL; 4632 // _SPECIALIZE_LOAD_SUPER_ATTR 4633 class = stack_pointer[-2]; 4634 global_super = stack_pointer[-3]; 4635 { 4636 uint16_t counter = read_u16(&this_instr[1].cache); 4637 (void)counter; 4638 #if ENABLE_SPECIALIZATION 4639 int load_method = oparg & 1; 4640 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 4641 next_instr = this_instr; 4642 _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); 4643 DISPATCH_SAME_OPARG(); 4644 } 4645 STAT_INC(LOAD_SUPER_ATTR, deferred); 4646 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 4647 #endif /* ENABLE_SPECIALIZATION */ 4648 } 4649 // _LOAD_SUPER_ATTR 4650 self = stack_pointer[-1]; 4651 { 4652 if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { 4653 PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; 4654 int err = _Py_call_instrumentation_2args( 4655 tstate, PY_MONITORING_EVENT_CALL, 4656 frame, this_instr, global_super, arg); 4657 if (err) goto pop_3_error; 4658 } 4659 // we make no attempt to optimize here; specializations should 4660 // handle any case whose performance we care about 4661 PyObject *stack[] = {class, self}; 4662 PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); 4663 if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { 4664 PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; 4665 if (super == NULL) { 4666 _Py_call_instrumentation_exc2( 4667 tstate, PY_MONITORING_EVENT_C_RAISE, 4668 frame, this_instr, global_super, arg); 4669 } 4670 else { 4671 int err = _Py_call_instrumentation_2args( 4672 tstate, PY_MONITORING_EVENT_C_RETURN, 4673 frame, this_instr, global_super, arg); 4674 if (err < 0) { 4675 Py_CLEAR(super); 4676 } 4677 } 4678 } 4679 Py_DECREF(global_super); 4680 Py_DECREF(class); 4681 Py_DECREF(self); 4682 if (super == NULL) goto pop_3_error; 4683 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); 4684 attr = PyObject_GetAttr(super, name); 4685 Py_DECREF(super); 4686 if (attr == NULL) goto pop_3_error; 4687 null = NULL; 4688 } 4689 stack_pointer[-3] = attr; 4690 if (oparg & 1) stack_pointer[-2] = null; 4691 stack_pointer += -2 + (oparg & 1); 4692 DISPATCH(); 4693 } 4694 TARGET(LOAD_SUPER_ATTR_ATTR)4695 TARGET(LOAD_SUPER_ATTR_ATTR) { 4696 frame->instr_ptr = next_instr; 4697 next_instr += 2; 4698 INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); 4699 static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); 4700 PyObject *self; 4701 PyObject *class; 4702 PyObject *global_super; 4703 PyObject *attr; 4704 /* Skip 1 cache entry */ 4705 self = stack_pointer[-1]; 4706 class = stack_pointer[-2]; 4707 global_super = stack_pointer[-3]; 4708 assert(!(oparg & 1)); 4709 DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); 4710 DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); 4711 STAT_INC(LOAD_SUPER_ATTR, hit); 4712 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); 4713 attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); 4714 Py_DECREF(global_super); 4715 Py_DECREF(class); 4716 Py_DECREF(self); 4717 if (attr == NULL) goto pop_3_error; 4718 stack_pointer[-3] = attr; 4719 stack_pointer += -2; 4720 DISPATCH(); 4721 } 4722 TARGET(LOAD_SUPER_ATTR_METHOD)4723 TARGET(LOAD_SUPER_ATTR_METHOD) { 4724 frame->instr_ptr = next_instr; 4725 next_instr += 2; 4726 INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); 4727 static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); 4728 PyObject *self; 4729 PyObject *class; 4730 PyObject *global_super; 4731 PyObject *attr; 4732 PyObject *self_or_null; 4733 /* Skip 1 cache entry */ 4734 self = stack_pointer[-1]; 4735 class = stack_pointer[-2]; 4736 global_super = stack_pointer[-3]; 4737 assert(oparg & 1); 4738 DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); 4739 DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); 4740 STAT_INC(LOAD_SUPER_ATTR, hit); 4741 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); 4742 PyTypeObject *cls = (PyTypeObject *)class; 4743 int method_found = 0; 4744 attr = _PySuper_Lookup(cls, self, name, 4745 Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); 4746 Py_DECREF(global_super); 4747 Py_DECREF(class); 4748 if (attr == NULL) { 4749 Py_DECREF(self); 4750 if (true) goto pop_3_error; 4751 } 4752 if (method_found) { 4753 self_or_null = self; // transfer ownership 4754 } else { 4755 Py_DECREF(self); 4756 self_or_null = NULL; 4757 } 4758 stack_pointer[-3] = attr; 4759 stack_pointer[-2] = self_or_null; 4760 stack_pointer += -1; 4761 DISPATCH(); 4762 } 4763 TARGET(MAKE_CELL)4764 TARGET(MAKE_CELL) { 4765 frame->instr_ptr = next_instr; 4766 next_instr += 1; 4767 INSTRUCTION_STATS(MAKE_CELL); 4768 // "initial" is probably NULL but not if it's an arg (or set 4769 // via the f_locals proxy before MAKE_CELL has run). 4770 PyObject *initial = GETLOCAL(oparg); 4771 PyObject *cell = PyCell_New(initial); 4772 if (cell == NULL) { 4773 goto error; 4774 } 4775 SETLOCAL(oparg, cell); 4776 DISPATCH(); 4777 } 4778 TARGET(MAKE_FUNCTION)4779 TARGET(MAKE_FUNCTION) { 4780 frame->instr_ptr = next_instr; 4781 next_instr += 1; 4782 INSTRUCTION_STATS(MAKE_FUNCTION); 4783 PyObject *codeobj; 4784 PyObject *func; 4785 codeobj = stack_pointer[-1]; 4786 PyFunctionObject *func_obj = (PyFunctionObject *) 4787 PyFunction_New(codeobj, GLOBALS()); 4788 Py_DECREF(codeobj); 4789 if (func_obj == NULL) { 4790 goto error; 4791 } 4792 _PyFunction_SetVersion( 4793 func_obj, ((PyCodeObject *)codeobj)->co_version); 4794 func = (PyObject *)func_obj; 4795 stack_pointer[-1] = func; 4796 DISPATCH(); 4797 } 4798 TARGET(MAP_ADD)4799 TARGET(MAP_ADD) { 4800 frame->instr_ptr = next_instr; 4801 next_instr += 1; 4802 INSTRUCTION_STATS(MAP_ADD); 4803 PyObject *value; 4804 PyObject *key; 4805 PyObject *dict; 4806 value = stack_pointer[-1]; 4807 key = stack_pointer[-2]; 4808 dict = stack_pointer[-3 - (oparg - 1)]; 4809 assert(PyDict_CheckExact(dict)); 4810 /* dict[key] = value */ 4811 // Do not DECREF INPUTS because the function steals the references 4812 if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; 4813 stack_pointer += -2; 4814 DISPATCH(); 4815 } 4816 TARGET(MATCH_CLASS)4817 TARGET(MATCH_CLASS) { 4818 frame->instr_ptr = next_instr; 4819 next_instr += 1; 4820 INSTRUCTION_STATS(MATCH_CLASS); 4821 PyObject *names; 4822 PyObject *type; 4823 PyObject *subject; 4824 PyObject *attrs; 4825 names = stack_pointer[-1]; 4826 type = stack_pointer[-2]; 4827 subject = stack_pointer[-3]; 4828 // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or 4829 // None on failure. 4830 assert(PyTuple_CheckExact(names)); 4831 attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); 4832 Py_DECREF(subject); 4833 Py_DECREF(type); 4834 Py_DECREF(names); 4835 if (attrs) { 4836 assert(PyTuple_CheckExact(attrs)); // Success! 4837 } 4838 else { 4839 if (_PyErr_Occurred(tstate)) goto pop_3_error; 4840 // Error! 4841 attrs = Py_None; // Failure! 4842 } 4843 stack_pointer[-3] = attrs; 4844 stack_pointer += -2; 4845 DISPATCH(); 4846 } 4847 TARGET(MATCH_KEYS)4848 TARGET(MATCH_KEYS) { 4849 frame->instr_ptr = next_instr; 4850 next_instr += 1; 4851 INSTRUCTION_STATS(MATCH_KEYS); 4852 PyObject *keys; 4853 PyObject *subject; 4854 PyObject *values_or_none; 4855 keys = stack_pointer[-1]; 4856 subject = stack_pointer[-2]; 4857 // On successful match, PUSH(values). Otherwise, PUSH(None). 4858 values_or_none = _PyEval_MatchKeys(tstate, subject, keys); 4859 if (values_or_none == NULL) goto error; 4860 stack_pointer[0] = values_or_none; 4861 stack_pointer += 1; 4862 DISPATCH(); 4863 } 4864 TARGET(MATCH_MAPPING)4865 TARGET(MATCH_MAPPING) { 4866 frame->instr_ptr = next_instr; 4867 next_instr += 1; 4868 INSTRUCTION_STATS(MATCH_MAPPING); 4869 PyObject *subject; 4870 PyObject *res; 4871 subject = stack_pointer[-1]; 4872 int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; 4873 res = match ? Py_True : Py_False; 4874 stack_pointer[0] = res; 4875 stack_pointer += 1; 4876 DISPATCH(); 4877 } 4878 TARGET(MATCH_SEQUENCE)4879 TARGET(MATCH_SEQUENCE) { 4880 frame->instr_ptr = next_instr; 4881 next_instr += 1; 4882 INSTRUCTION_STATS(MATCH_SEQUENCE); 4883 PyObject *subject; 4884 PyObject *res; 4885 subject = stack_pointer[-1]; 4886 int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; 4887 res = match ? Py_True : Py_False; 4888 stack_pointer[0] = res; 4889 stack_pointer += 1; 4890 DISPATCH(); 4891 } 4892 TARGET(NOP)4893 TARGET(NOP) { 4894 frame->instr_ptr = next_instr; 4895 next_instr += 1; 4896 INSTRUCTION_STATS(NOP); 4897 DISPATCH(); 4898 } 4899 TARGET(POP_EXCEPT)4900 TARGET(POP_EXCEPT) { 4901 frame->instr_ptr = next_instr; 4902 next_instr += 1; 4903 INSTRUCTION_STATS(POP_EXCEPT); 4904 PyObject *exc_value; 4905 exc_value = stack_pointer[-1]; 4906 _PyErr_StackItem *exc_info = tstate->exc_info; 4907 Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); 4908 stack_pointer += -1; 4909 DISPATCH(); 4910 } 4911 TARGET(POP_JUMP_IF_FALSE)4912 TARGET(POP_JUMP_IF_FALSE) { 4913 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4914 (void)this_instr; 4915 next_instr += 2; 4916 INSTRUCTION_STATS(POP_JUMP_IF_FALSE); 4917 PyObject *cond; 4918 /* Skip 1 cache entry */ 4919 cond = stack_pointer[-1]; 4920 assert(PyBool_Check(cond)); 4921 int flag = Py_IsFalse(cond); 4922 #if ENABLE_SPECIALIZATION 4923 this_instr[1].cache = (this_instr[1].cache << 1) | flag; 4924 #endif 4925 JUMPBY(oparg * flag); 4926 stack_pointer += -1; 4927 DISPATCH(); 4928 } 4929 TARGET(POP_JUMP_IF_NONE)4930 TARGET(POP_JUMP_IF_NONE) { 4931 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4932 (void)this_instr; 4933 next_instr += 2; 4934 INSTRUCTION_STATS(POP_JUMP_IF_NONE); 4935 PyObject *value; 4936 PyObject *b; 4937 PyObject *cond; 4938 /* Skip 1 cache entry */ 4939 // _IS_NONE 4940 value = stack_pointer[-1]; 4941 { 4942 if (Py_IsNone(value)) { 4943 b = Py_True; 4944 } 4945 else { 4946 b = Py_False; 4947 Py_DECREF(value); 4948 } 4949 } 4950 // _POP_JUMP_IF_TRUE 4951 cond = b; 4952 { 4953 assert(PyBool_Check(cond)); 4954 int flag = Py_IsTrue(cond); 4955 #if ENABLE_SPECIALIZATION 4956 this_instr[1].cache = (this_instr[1].cache << 1) | flag; 4957 #endif 4958 JUMPBY(oparg * flag); 4959 } 4960 stack_pointer += -1; 4961 DISPATCH(); 4962 } 4963 TARGET(POP_JUMP_IF_NOT_NONE)4964 TARGET(POP_JUMP_IF_NOT_NONE) { 4965 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 4966 (void)this_instr; 4967 next_instr += 2; 4968 INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); 4969 PyObject *value; 4970 PyObject *b; 4971 PyObject *cond; 4972 /* Skip 1 cache entry */ 4973 // _IS_NONE 4974 value = stack_pointer[-1]; 4975 { 4976 if (Py_IsNone(value)) { 4977 b = Py_True; 4978 } 4979 else { 4980 b = Py_False; 4981 Py_DECREF(value); 4982 } 4983 } 4984 // _POP_JUMP_IF_FALSE 4985 cond = b; 4986 { 4987 assert(PyBool_Check(cond)); 4988 int flag = Py_IsFalse(cond); 4989 #if ENABLE_SPECIALIZATION 4990 this_instr[1].cache = (this_instr[1].cache << 1) | flag; 4991 #endif 4992 JUMPBY(oparg * flag); 4993 } 4994 stack_pointer += -1; 4995 DISPATCH(); 4996 } 4997 TARGET(POP_JUMP_IF_TRUE)4998 TARGET(POP_JUMP_IF_TRUE) { 4999 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 5000 (void)this_instr; 5001 next_instr += 2; 5002 INSTRUCTION_STATS(POP_JUMP_IF_TRUE); 5003 PyObject *cond; 5004 /* Skip 1 cache entry */ 5005 cond = stack_pointer[-1]; 5006 assert(PyBool_Check(cond)); 5007 int flag = Py_IsTrue(cond); 5008 #if ENABLE_SPECIALIZATION 5009 this_instr[1].cache = (this_instr[1].cache << 1) | flag; 5010 #endif 5011 JUMPBY(oparg * flag); 5012 stack_pointer += -1; 5013 DISPATCH(); 5014 } 5015 TARGET(POP_TOP)5016 TARGET(POP_TOP) { 5017 frame->instr_ptr = next_instr; 5018 next_instr += 1; 5019 INSTRUCTION_STATS(POP_TOP); 5020 PyObject *value; 5021 value = stack_pointer[-1]; 5022 Py_DECREF(value); 5023 stack_pointer += -1; 5024 DISPATCH(); 5025 } 5026 TARGET(PUSH_EXC_INFO)5027 TARGET(PUSH_EXC_INFO) { 5028 frame->instr_ptr = next_instr; 5029 next_instr += 1; 5030 INSTRUCTION_STATS(PUSH_EXC_INFO); 5031 PyObject *new_exc; 5032 PyObject *prev_exc; 5033 new_exc = stack_pointer[-1]; 5034 _PyErr_StackItem *exc_info = tstate->exc_info; 5035 if (exc_info->exc_value != NULL) { 5036 prev_exc = exc_info->exc_value; 5037 } 5038 else { 5039 prev_exc = Py_None; 5040 } 5041 assert(PyExceptionInstance_Check(new_exc)); 5042 exc_info->exc_value = Py_NewRef(new_exc); 5043 stack_pointer[-1] = prev_exc; 5044 stack_pointer[0] = new_exc; 5045 stack_pointer += 1; 5046 DISPATCH(); 5047 } 5048 TARGET(PUSH_NULL)5049 TARGET(PUSH_NULL) { 5050 frame->instr_ptr = next_instr; 5051 next_instr += 1; 5052 INSTRUCTION_STATS(PUSH_NULL); 5053 PyObject *res; 5054 res = NULL; 5055 stack_pointer[0] = res; 5056 stack_pointer += 1; 5057 DISPATCH(); 5058 } 5059 TARGET(RAISE_VARARGS)5060 TARGET(RAISE_VARARGS) { 5061 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 5062 (void)this_instr; 5063 next_instr += 1; 5064 INSTRUCTION_STATS(RAISE_VARARGS); 5065 PyObject **args; 5066 args = &stack_pointer[-oparg]; 5067 PyObject *cause = NULL, *exc = NULL; 5068 switch (oparg) { 5069 case 2: 5070 cause = args[1]; 5071 /* fall through */ 5072 case 1: 5073 exc = args[0]; 5074 /* fall through */ 5075 case 0: 5076 if (do_raise(tstate, exc, cause)) { 5077 assert(oparg == 0); 5078 monitor_reraise(tstate, frame, this_instr); 5079 goto exception_unwind; 5080 } 5081 break; 5082 default: 5083 _PyErr_SetString(tstate, PyExc_SystemError, 5084 "bad RAISE_VARARGS oparg"); 5085 break; 5086 } 5087 if (true) { stack_pointer += -oparg; goto error; } 5088 } 5089 TARGET(RERAISE)5090 TARGET(RERAISE) { 5091 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 5092 (void)this_instr; 5093 next_instr += 1; 5094 INSTRUCTION_STATS(RERAISE); 5095 PyObject *exc; 5096 PyObject **values; 5097 exc = stack_pointer[-1]; 5098 values = &stack_pointer[-1 - oparg]; 5099 assert(oparg >= 0 && oparg <= 2); 5100 if (oparg) { 5101 PyObject *lasti = values[0]; 5102 if (PyLong_Check(lasti)) { 5103 frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); 5104 assert(!_PyErr_Occurred(tstate)); 5105 } 5106 else { 5107 assert(PyLong_Check(lasti)); 5108 _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); 5109 goto error; 5110 } 5111 } 5112 assert(exc && PyExceptionInstance_Check(exc)); 5113 Py_INCREF(exc); 5114 _PyErr_SetRaisedException(tstate, exc); 5115 monitor_reraise(tstate, frame, this_instr); 5116 goto exception_unwind; 5117 } 5118 TARGET(RESERVED)5119 TARGET(RESERVED) { 5120 frame->instr_ptr = next_instr; 5121 next_instr += 1; 5122 INSTRUCTION_STATS(RESERVED); 5123 assert(0 && "Executing RESERVED instruction."); 5124 Py_FatalError("Executing RESERVED instruction."); 5125 DISPATCH(); 5126 } 5127 TARGET(RESUME)5128 TARGET(RESUME) { 5129 frame->instr_ptr = next_instr; 5130 next_instr += 1; 5131 INSTRUCTION_STATS(RESUME); 5132 PREDICTED(RESUME); 5133 _Py_CODEUNIT *this_instr = next_instr - 1; 5134 (void)this_instr; 5135 assert(frame == tstate->current_frame); 5136 if (tstate->tracing == 0) { 5137 uintptr_t global_version = 5138 _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & 5139 ~_PY_EVAL_EVENTS_MASK; 5140 PyCodeObject* code = _PyFrame_GetCode(frame); 5141 uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); 5142 assert((code_version & 255) == 0); 5143 if (code_version != global_version) { 5144 int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); 5145 if (err) goto error; 5146 next_instr = this_instr; 5147 DISPATCH(); 5148 } 5149 assert(this_instr->op.code == RESUME || 5150 this_instr->op.code == RESUME_CHECK || 5151 this_instr->op.code == INSTRUMENTED_RESUME || 5152 this_instr->op.code == ENTER_EXECUTOR); 5153 if (this_instr->op.code == RESUME) { 5154 #if ENABLE_SPECIALIZATION 5155 FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); 5156 #endif /* ENABLE_SPECIALIZATION */ 5157 } 5158 } 5159 if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { 5160 CHECK_EVAL_BREAKER(); 5161 } 5162 DISPATCH(); 5163 } 5164 TARGET(RESUME_CHECK)5165 TARGET(RESUME_CHECK) { 5166 frame->instr_ptr = next_instr; 5167 next_instr += 1; 5168 INSTRUCTION_STATS(RESUME_CHECK); 5169 static_assert(0 == 0, "incorrect cache size"); 5170 #if defined(__EMSCRIPTEN__) 5171 DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); 5172 _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; 5173 #endif 5174 uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); 5175 uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); 5176 assert((version & _PY_EVAL_EVENTS_MASK) == 0); 5177 DEOPT_IF(eval_breaker != version, RESUME); 5178 DISPATCH(); 5179 } 5180 TARGET(RETURN_CONST)5181 TARGET(RETURN_CONST) { 5182 frame->instr_ptr = next_instr; 5183 next_instr += 1; 5184 INSTRUCTION_STATS(RETURN_CONST); 5185 PyObject *value; 5186 PyObject *retval; 5187 // _LOAD_CONST 5188 { 5189 value = GETITEM(FRAME_CO_CONSTS, oparg); 5190 Py_INCREF(value); 5191 } 5192 // _POP_FRAME 5193 retval = value; 5194 { 5195 #if TIER_ONE 5196 assert(frame != &entry_frame); 5197 #endif 5198 _PyFrame_SetStackPointer(frame, stack_pointer); 5199 assert(EMPTY()); 5200 _Py_LeaveRecursiveCallPy(tstate); 5201 // GH-99729: We need to unlink the frame *before* clearing it: 5202 _PyInterpreterFrame *dying = frame; 5203 frame = tstate->current_frame = dying->previous; 5204 _PyEval_FrameClearAndPop(tstate, dying); 5205 _PyFrame_StackPush(frame, retval); 5206 LOAD_SP(); 5207 LOAD_IP(frame->return_offset); 5208 LLTRACE_RESUME_FRAME(); 5209 } 5210 DISPATCH(); 5211 } 5212 TARGET(RETURN_GENERATOR)5213 TARGET(RETURN_GENERATOR) { 5214 frame->instr_ptr = next_instr; 5215 next_instr += 1; 5216 INSTRUCTION_STATS(RETURN_GENERATOR); 5217 PyObject *res; 5218 assert(PyFunction_Check(frame->f_funcobj)); 5219 PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; 5220 PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); 5221 if (gen == NULL) { 5222 goto error; 5223 } 5224 assert(EMPTY()); 5225 _PyFrame_SetStackPointer(frame, stack_pointer); 5226 _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; 5227 frame->instr_ptr++; 5228 _PyFrame_Copy(frame, gen_frame); 5229 assert(frame->frame_obj == NULL); 5230 gen->gi_frame_state = FRAME_CREATED; 5231 gen_frame->owner = FRAME_OWNED_BY_GENERATOR; 5232 _Py_LeaveRecursiveCallPy(tstate); 5233 res = (PyObject *)gen; 5234 _PyInterpreterFrame *prev = frame->previous; 5235 _PyThreadState_PopFrame(tstate, frame); 5236 frame = tstate->current_frame = prev; 5237 LOAD_IP(frame->return_offset); 5238 LOAD_SP(); 5239 LLTRACE_RESUME_FRAME(); 5240 stack_pointer[0] = res; 5241 stack_pointer += 1; 5242 DISPATCH(); 5243 } 5244 TARGET(RETURN_VALUE)5245 TARGET(RETURN_VALUE) { 5246 frame->instr_ptr = next_instr; 5247 next_instr += 1; 5248 INSTRUCTION_STATS(RETURN_VALUE); 5249 PyObject *retval; 5250 retval = stack_pointer[-1]; 5251 #if TIER_ONE 5252 assert(frame != &entry_frame); 5253 #endif 5254 stack_pointer += -1; 5255 _PyFrame_SetStackPointer(frame, stack_pointer); 5256 assert(EMPTY()); 5257 _Py_LeaveRecursiveCallPy(tstate); 5258 // GH-99729: We need to unlink the frame *before* clearing it: 5259 _PyInterpreterFrame *dying = frame; 5260 frame = tstate->current_frame = dying->previous; 5261 _PyEval_FrameClearAndPop(tstate, dying); 5262 _PyFrame_StackPush(frame, retval); 5263 LOAD_SP(); 5264 LOAD_IP(frame->return_offset); 5265 LLTRACE_RESUME_FRAME(); 5266 DISPATCH(); 5267 } 5268 TARGET(SEND)5269 TARGET(SEND) { 5270 frame->instr_ptr = next_instr; 5271 next_instr += 2; 5272 INSTRUCTION_STATS(SEND); 5273 PREDICTED(SEND); 5274 _Py_CODEUNIT *this_instr = next_instr - 2; 5275 (void)this_instr; 5276 PyObject *receiver; 5277 PyObject *v; 5278 PyObject *retval; 5279 // _SPECIALIZE_SEND 5280 receiver = stack_pointer[-2]; 5281 { 5282 uint16_t counter = read_u16(&this_instr[1].cache); 5283 (void)counter; 5284 #if ENABLE_SPECIALIZATION 5285 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 5286 next_instr = this_instr; 5287 _Py_Specialize_Send(receiver, next_instr); 5288 DISPATCH_SAME_OPARG(); 5289 } 5290 STAT_INC(SEND, deferred); 5291 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 5292 #endif /* ENABLE_SPECIALIZATION */ 5293 } 5294 // _SEND 5295 v = stack_pointer[-1]; 5296 { 5297 assert(frame != &entry_frame); 5298 if ((tstate->interp->eval_frame == NULL) && 5299 (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && 5300 ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) 5301 { 5302 PyGenObject *gen = (PyGenObject *)receiver; 5303 _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; 5304 STACK_SHRINK(1); 5305 _PyFrame_StackPush(gen_frame, v); 5306 gen->gi_frame_state = FRAME_EXECUTING; 5307 gen->gi_exc_state.previous_item = tstate->exc_info; 5308 tstate->exc_info = &gen->gi_exc_state; 5309 assert(next_instr - this_instr + oparg <= UINT16_MAX); 5310 frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); 5311 DISPATCH_INLINED(gen_frame); 5312 } 5313 if (Py_IsNone(v) && PyIter_Check(receiver)) { 5314 retval = Py_TYPE(receiver)->tp_iternext(receiver); 5315 } 5316 else { 5317 retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); 5318 } 5319 if (retval == NULL) { 5320 if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) 5321 ) { 5322 _PyEval_MonitorRaise(tstate, frame, this_instr); 5323 } 5324 if (_PyGen_FetchStopIterationValue(&retval) == 0) { 5325 assert(retval != NULL); 5326 JUMPBY(oparg); 5327 } 5328 else { 5329 goto error; 5330 } 5331 } 5332 Py_DECREF(v); 5333 } 5334 stack_pointer[-1] = retval; 5335 DISPATCH(); 5336 } 5337 TARGET(SEND_GEN)5338 TARGET(SEND_GEN) { 5339 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 5340 next_instr += 2; 5341 INSTRUCTION_STATS(SEND_GEN); 5342 static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); 5343 PyObject *v; 5344 PyObject *receiver; 5345 /* Skip 1 cache entry */ 5346 v = stack_pointer[-1]; 5347 receiver = stack_pointer[-2]; 5348 DEOPT_IF(tstate->interp->eval_frame, SEND); 5349 PyGenObject *gen = (PyGenObject *)receiver; 5350 DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); 5351 DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); 5352 STAT_INC(SEND, hit); 5353 _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; 5354 STACK_SHRINK(1); 5355 _PyFrame_StackPush(gen_frame, v); 5356 gen->gi_frame_state = FRAME_EXECUTING; 5357 gen->gi_exc_state.previous_item = tstate->exc_info; 5358 tstate->exc_info = &gen->gi_exc_state; 5359 assert(next_instr - this_instr + oparg <= UINT16_MAX); 5360 frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); 5361 DISPATCH_INLINED(gen_frame); 5362 } 5363 TARGET(SETUP_ANNOTATIONS)5364 TARGET(SETUP_ANNOTATIONS) { 5365 frame->instr_ptr = next_instr; 5366 next_instr += 1; 5367 INSTRUCTION_STATS(SETUP_ANNOTATIONS); 5368 int err; 5369 PyObject *ann_dict; 5370 if (LOCALS() == NULL) { 5371 _PyErr_Format(tstate, PyExc_SystemError, 5372 "no locals found when setting up annotations"); 5373 if (true) goto error; 5374 } 5375 /* check if __annotations__ in locals()... */ 5376 if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; 5377 if (ann_dict == NULL) { 5378 ann_dict = PyDict_New(); 5379 if (ann_dict == NULL) goto error; 5380 err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), 5381 ann_dict); 5382 Py_DECREF(ann_dict); 5383 if (err) goto error; 5384 } 5385 else { 5386 Py_DECREF(ann_dict); 5387 } 5388 DISPATCH(); 5389 } 5390 TARGET(SET_ADD)5391 TARGET(SET_ADD) { 5392 frame->instr_ptr = next_instr; 5393 next_instr += 1; 5394 INSTRUCTION_STATS(SET_ADD); 5395 PyObject *v; 5396 PyObject *set; 5397 v = stack_pointer[-1]; 5398 set = stack_pointer[-2 - (oparg-1)]; 5399 int err = PySet_Add(set, v); 5400 Py_DECREF(v); 5401 if (err) goto pop_1_error; 5402 stack_pointer += -1; 5403 DISPATCH(); 5404 } 5405 TARGET(SET_FUNCTION_ATTRIBUTE)5406 TARGET(SET_FUNCTION_ATTRIBUTE) { 5407 frame->instr_ptr = next_instr; 5408 next_instr += 1; 5409 INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); 5410 PyObject *func; 5411 PyObject *attr; 5412 func = stack_pointer[-1]; 5413 attr = stack_pointer[-2]; 5414 assert(PyFunction_Check(func)); 5415 PyFunctionObject *func_obj = (PyFunctionObject *)func; 5416 switch(oparg) { 5417 case MAKE_FUNCTION_CLOSURE: 5418 assert(func_obj->func_closure == NULL); 5419 func_obj->func_closure = attr; 5420 break; 5421 case MAKE_FUNCTION_ANNOTATIONS: 5422 assert(func_obj->func_annotations == NULL); 5423 func_obj->func_annotations = attr; 5424 break; 5425 case MAKE_FUNCTION_KWDEFAULTS: 5426 assert(PyDict_CheckExact(attr)); 5427 assert(func_obj->func_kwdefaults == NULL); 5428 func_obj->func_kwdefaults = attr; 5429 break; 5430 case MAKE_FUNCTION_DEFAULTS: 5431 assert(PyTuple_CheckExact(attr)); 5432 assert(func_obj->func_defaults == NULL); 5433 func_obj->func_defaults = attr; 5434 break; 5435 default: 5436 Py_UNREACHABLE(); 5437 } 5438 stack_pointer[-2] = func; 5439 stack_pointer += -1; 5440 DISPATCH(); 5441 } 5442 TARGET(SET_UPDATE)5443 TARGET(SET_UPDATE) { 5444 frame->instr_ptr = next_instr; 5445 next_instr += 1; 5446 INSTRUCTION_STATS(SET_UPDATE); 5447 PyObject *iterable; 5448 PyObject *set; 5449 iterable = stack_pointer[-1]; 5450 set = stack_pointer[-2 - (oparg-1)]; 5451 int err = _PySet_Update(set, iterable); 5452 Py_DECREF(iterable); 5453 if (err < 0) goto pop_1_error; 5454 stack_pointer += -1; 5455 DISPATCH(); 5456 } 5457 TARGET(STORE_ATTR)5458 TARGET(STORE_ATTR) { 5459 frame->instr_ptr = next_instr; 5460 next_instr += 5; 5461 INSTRUCTION_STATS(STORE_ATTR); 5462 PREDICTED(STORE_ATTR); 5463 _Py_CODEUNIT *this_instr = next_instr - 5; 5464 (void)this_instr; 5465 PyObject *owner; 5466 PyObject *v; 5467 // _SPECIALIZE_STORE_ATTR 5468 owner = stack_pointer[-1]; 5469 { 5470 uint16_t counter = read_u16(&this_instr[1].cache); 5471 (void)counter; 5472 #if ENABLE_SPECIALIZATION 5473 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 5474 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 5475 next_instr = this_instr; 5476 _Py_Specialize_StoreAttr(owner, next_instr, name); 5477 DISPATCH_SAME_OPARG(); 5478 } 5479 STAT_INC(STORE_ATTR, deferred); 5480 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 5481 #endif /* ENABLE_SPECIALIZATION */ 5482 } 5483 /* Skip 3 cache entries */ 5484 // _STORE_ATTR 5485 v = stack_pointer[-2]; 5486 { 5487 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 5488 int err = PyObject_SetAttr(owner, name, v); 5489 Py_DECREF(v); 5490 Py_DECREF(owner); 5491 if (err) goto pop_2_error; 5492 } 5493 stack_pointer += -2; 5494 DISPATCH(); 5495 } 5496 TARGET(STORE_ATTR_INSTANCE_VALUE)5497 TARGET(STORE_ATTR_INSTANCE_VALUE) { 5498 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 5499 next_instr += 5; 5500 INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); 5501 static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); 5502 PyObject *owner; 5503 PyObject *value; 5504 /* Skip 1 cache entry */ 5505 // _GUARD_TYPE_VERSION 5506 owner = stack_pointer[-1]; 5507 { 5508 uint32_t type_version = read_u32(&this_instr[2].cache); 5509 PyTypeObject *tp = Py_TYPE(owner); 5510 assert(type_version != 0); 5511 DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); 5512 } 5513 // _GUARD_DORV_NO_DICT 5514 { 5515 assert(Py_TYPE(owner)->tp_dictoffset < 0); 5516 assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); 5517 DEOPT_IF(_PyObject_GetManagedDict(owner), STORE_ATTR); 5518 DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR); 5519 } 5520 // _STORE_ATTR_INSTANCE_VALUE 5521 value = stack_pointer[-2]; 5522 { 5523 uint16_t index = read_u16(&this_instr[4].cache); 5524 STAT_INC(STORE_ATTR, hit); 5525 assert(_PyObject_GetManagedDict(owner) == NULL); 5526 PyDictValues *values = _PyObject_InlineValues(owner); 5527 PyObject *old_value = values->values[index]; 5528 values->values[index] = value; 5529 if (old_value == NULL) { 5530 _PyDictValues_AddToInsertionOrder(values, index); 5531 } 5532 else { 5533 Py_DECREF(old_value); 5534 } 5535 Py_DECREF(owner); 5536 } 5537 stack_pointer += -2; 5538 DISPATCH(); 5539 } 5540 TARGET(STORE_ATTR_SLOT)5541 TARGET(STORE_ATTR_SLOT) { 5542 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 5543 next_instr += 5; 5544 INSTRUCTION_STATS(STORE_ATTR_SLOT); 5545 static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); 5546 PyObject *owner; 5547 PyObject *value; 5548 /* Skip 1 cache entry */ 5549 // _GUARD_TYPE_VERSION 5550 owner = stack_pointer[-1]; 5551 { 5552 uint32_t type_version = read_u32(&this_instr[2].cache); 5553 PyTypeObject *tp = Py_TYPE(owner); 5554 assert(type_version != 0); 5555 DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); 5556 } 5557 // _STORE_ATTR_SLOT 5558 value = stack_pointer[-2]; 5559 { 5560 uint16_t index = read_u16(&this_instr[4].cache); 5561 char *addr = (char *)owner + index; 5562 STAT_INC(STORE_ATTR, hit); 5563 PyObject *old_value = *(PyObject **)addr; 5564 *(PyObject **)addr = value; 5565 Py_XDECREF(old_value); 5566 Py_DECREF(owner); 5567 } 5568 stack_pointer += -2; 5569 DISPATCH(); 5570 } 5571 TARGET(STORE_ATTR_WITH_HINT)5572 TARGET(STORE_ATTR_WITH_HINT) { 5573 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 5574 next_instr += 5; 5575 INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); 5576 static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); 5577 PyObject *owner; 5578 PyObject *value; 5579 /* Skip 1 cache entry */ 5580 owner = stack_pointer[-1]; 5581 value = stack_pointer[-2]; 5582 uint32_t type_version = read_u32(&this_instr[2].cache); 5583 uint16_t hint = read_u16(&this_instr[4].cache); 5584 PyTypeObject *tp = Py_TYPE(owner); 5585 assert(type_version != 0); 5586 DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); 5587 assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); 5588 PyDictObject *dict = _PyObject_GetManagedDict(owner); 5589 DEOPT_IF(dict == NULL, STORE_ATTR); 5590 assert(PyDict_CheckExact((PyObject *)dict)); 5591 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 5592 DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); 5593 PyObject *old_value; 5594 uint64_t new_version; 5595 if (DK_IS_UNICODE(dict->ma_keys)) { 5596 PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; 5597 DEOPT_IF(ep->me_key != name, STORE_ATTR); 5598 old_value = ep->me_value; 5599 DEOPT_IF(old_value == NULL, STORE_ATTR); 5600 new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); 5601 ep->me_value = value; 5602 } 5603 else { 5604 PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; 5605 DEOPT_IF(ep->me_key != name, STORE_ATTR); 5606 old_value = ep->me_value; 5607 DEOPT_IF(old_value == NULL, STORE_ATTR); 5608 new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); 5609 ep->me_value = value; 5610 } 5611 /* Ensure dict is GC tracked if it needs to be */ 5612 if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { 5613 _PyObject_GC_TRACK(dict); 5614 } 5615 dict->ma_version_tag = new_version; // PEP 509 5616 // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, 5617 // when dict only holds the strong reference to value in ep->me_value. 5618 Py_DECREF(old_value); 5619 STAT_INC(STORE_ATTR, hit); 5620 Py_DECREF(owner); 5621 stack_pointer += -2; 5622 DISPATCH(); 5623 } 5624 TARGET(STORE_DEREF)5625 TARGET(STORE_DEREF) { 5626 frame->instr_ptr = next_instr; 5627 next_instr += 1; 5628 INSTRUCTION_STATS(STORE_DEREF); 5629 PyObject *v; 5630 v = stack_pointer[-1]; 5631 PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); 5632 PyCell_SetTakeRef(cell, v); 5633 stack_pointer += -1; 5634 DISPATCH(); 5635 } 5636 TARGET(STORE_FAST)5637 TARGET(STORE_FAST) { 5638 frame->instr_ptr = next_instr; 5639 next_instr += 1; 5640 INSTRUCTION_STATS(STORE_FAST); 5641 PyObject *value; 5642 value = stack_pointer[-1]; 5643 SETLOCAL(oparg, value); 5644 stack_pointer += -1; 5645 DISPATCH(); 5646 } 5647 TARGET(STORE_FAST_LOAD_FAST)5648 TARGET(STORE_FAST_LOAD_FAST) { 5649 frame->instr_ptr = next_instr; 5650 next_instr += 1; 5651 INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); 5652 PyObject *value1; 5653 PyObject *value2; 5654 value1 = stack_pointer[-1]; 5655 uint32_t oparg1 = oparg >> 4; 5656 uint32_t oparg2 = oparg & 15; 5657 SETLOCAL(oparg1, value1); 5658 value2 = GETLOCAL(oparg2); 5659 Py_INCREF(value2); 5660 stack_pointer[-1] = value2; 5661 DISPATCH(); 5662 } 5663 TARGET(STORE_FAST_STORE_FAST)5664 TARGET(STORE_FAST_STORE_FAST) { 5665 frame->instr_ptr = next_instr; 5666 next_instr += 1; 5667 INSTRUCTION_STATS(STORE_FAST_STORE_FAST); 5668 PyObject *value1; 5669 PyObject *value2; 5670 value1 = stack_pointer[-1]; 5671 value2 = stack_pointer[-2]; 5672 uint32_t oparg1 = oparg >> 4; 5673 uint32_t oparg2 = oparg & 15; 5674 SETLOCAL(oparg1, value1); 5675 SETLOCAL(oparg2, value2); 5676 stack_pointer += -2; 5677 DISPATCH(); 5678 } 5679 TARGET(STORE_GLOBAL)5680 TARGET(STORE_GLOBAL) { 5681 frame->instr_ptr = next_instr; 5682 next_instr += 1; 5683 INSTRUCTION_STATS(STORE_GLOBAL); 5684 PyObject *v; 5685 v = stack_pointer[-1]; 5686 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 5687 int err = PyDict_SetItem(GLOBALS(), name, v); 5688 Py_DECREF(v); 5689 if (err) goto pop_1_error; 5690 stack_pointer += -1; 5691 DISPATCH(); 5692 } 5693 TARGET(STORE_NAME)5694 TARGET(STORE_NAME) { 5695 frame->instr_ptr = next_instr; 5696 next_instr += 1; 5697 INSTRUCTION_STATS(STORE_NAME); 5698 PyObject *v; 5699 v = stack_pointer[-1]; 5700 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); 5701 PyObject *ns = LOCALS(); 5702 int err; 5703 if (ns == NULL) { 5704 _PyErr_Format(tstate, PyExc_SystemError, 5705 "no locals found when storing %R", name); 5706 Py_DECREF(v); 5707 if (true) goto pop_1_error; 5708 } 5709 if (PyDict_CheckExact(ns)) 5710 err = PyDict_SetItem(ns, name, v); 5711 else 5712 err = PyObject_SetItem(ns, name, v); 5713 Py_DECREF(v); 5714 if (err) goto pop_1_error; 5715 stack_pointer += -1; 5716 DISPATCH(); 5717 } 5718 TARGET(STORE_SLICE)5719 TARGET(STORE_SLICE) { 5720 frame->instr_ptr = next_instr; 5721 next_instr += 1; 5722 INSTRUCTION_STATS(STORE_SLICE); 5723 PyObject *stop; 5724 PyObject *start; 5725 PyObject *container; 5726 PyObject *v; 5727 stop = stack_pointer[-1]; 5728 start = stack_pointer[-2]; 5729 container = stack_pointer[-3]; 5730 v = stack_pointer[-4]; 5731 PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); 5732 int err; 5733 if (slice == NULL) { 5734 err = 1; 5735 } 5736 else { 5737 err = PyObject_SetItem(container, slice, v); 5738 Py_DECREF(slice); 5739 } 5740 Py_DECREF(v); 5741 Py_DECREF(container); 5742 if (err) goto pop_4_error; 5743 stack_pointer += -4; 5744 DISPATCH(); 5745 } 5746 TARGET(STORE_SUBSCR)5747 TARGET(STORE_SUBSCR) { 5748 frame->instr_ptr = next_instr; 5749 next_instr += 2; 5750 INSTRUCTION_STATS(STORE_SUBSCR); 5751 PREDICTED(STORE_SUBSCR); 5752 _Py_CODEUNIT *this_instr = next_instr - 2; 5753 (void)this_instr; 5754 PyObject *sub; 5755 PyObject *container; 5756 PyObject *v; 5757 // _SPECIALIZE_STORE_SUBSCR 5758 sub = stack_pointer[-1]; 5759 container = stack_pointer[-2]; 5760 { 5761 uint16_t counter = read_u16(&this_instr[1].cache); 5762 (void)counter; 5763 #if ENABLE_SPECIALIZATION 5764 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 5765 next_instr = this_instr; 5766 _Py_Specialize_StoreSubscr(container, sub, next_instr); 5767 DISPATCH_SAME_OPARG(); 5768 } 5769 STAT_INC(STORE_SUBSCR, deferred); 5770 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 5771 #endif /* ENABLE_SPECIALIZATION */ 5772 } 5773 // _STORE_SUBSCR 5774 v = stack_pointer[-3]; 5775 { 5776 /* container[sub] = v */ 5777 int err = PyObject_SetItem(container, sub, v); 5778 Py_DECREF(v); 5779 Py_DECREF(container); 5780 Py_DECREF(sub); 5781 if (err) goto pop_3_error; 5782 } 5783 stack_pointer += -3; 5784 DISPATCH(); 5785 } 5786 TARGET(STORE_SUBSCR_DICT)5787 TARGET(STORE_SUBSCR_DICT) { 5788 frame->instr_ptr = next_instr; 5789 next_instr += 2; 5790 INSTRUCTION_STATS(STORE_SUBSCR_DICT); 5791 static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); 5792 PyObject *sub; 5793 PyObject *dict; 5794 PyObject *value; 5795 /* Skip 1 cache entry */ 5796 sub = stack_pointer[-1]; 5797 dict = stack_pointer[-2]; 5798 value = stack_pointer[-3]; 5799 DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); 5800 STAT_INC(STORE_SUBSCR, hit); 5801 int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); 5802 Py_DECREF(dict); 5803 if (err) goto pop_3_error; 5804 stack_pointer += -3; 5805 DISPATCH(); 5806 } 5807 TARGET(STORE_SUBSCR_LIST_INT)5808 TARGET(STORE_SUBSCR_LIST_INT) { 5809 frame->instr_ptr = next_instr; 5810 next_instr += 2; 5811 INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); 5812 static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); 5813 PyObject *sub; 5814 PyObject *list; 5815 PyObject *value; 5816 /* Skip 1 cache entry */ 5817 sub = stack_pointer[-1]; 5818 list = stack_pointer[-2]; 5819 value = stack_pointer[-3]; 5820 DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); 5821 DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); 5822 // Ensure nonnegative, zero-or-one-digit ints. 5823 DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); 5824 Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; 5825 // Ensure index < len(list) 5826 DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); 5827 STAT_INC(STORE_SUBSCR, hit); 5828 PyObject *old_value = PyList_GET_ITEM(list, index); 5829 PyList_SET_ITEM(list, index, value); 5830 assert(old_value != NULL); 5831 Py_DECREF(old_value); 5832 _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); 5833 Py_DECREF(list); 5834 stack_pointer += -3; 5835 DISPATCH(); 5836 } 5837 TARGET(SWAP)5838 TARGET(SWAP) { 5839 frame->instr_ptr = next_instr; 5840 next_instr += 1; 5841 INSTRUCTION_STATS(SWAP); 5842 PyObject *top; 5843 PyObject *bottom; 5844 top = stack_pointer[-1]; 5845 bottom = stack_pointer[-2 - (oparg-2)]; 5846 assert(oparg >= 2); 5847 stack_pointer[-2 - (oparg-2)] = top; 5848 stack_pointer[-1] = bottom; 5849 DISPATCH(); 5850 } 5851 TARGET(TO_BOOL)5852 TARGET(TO_BOOL) { 5853 frame->instr_ptr = next_instr; 5854 next_instr += 4; 5855 INSTRUCTION_STATS(TO_BOOL); 5856 PREDICTED(TO_BOOL); 5857 _Py_CODEUNIT *this_instr = next_instr - 4; 5858 (void)this_instr; 5859 PyObject *value; 5860 PyObject *res; 5861 // _SPECIALIZE_TO_BOOL 5862 value = stack_pointer[-1]; 5863 { 5864 uint16_t counter = read_u16(&this_instr[1].cache); 5865 (void)counter; 5866 #if ENABLE_SPECIALIZATION 5867 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 5868 next_instr = this_instr; 5869 _Py_Specialize_ToBool(value, next_instr); 5870 DISPATCH_SAME_OPARG(); 5871 } 5872 STAT_INC(TO_BOOL, deferred); 5873 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 5874 #endif /* ENABLE_SPECIALIZATION */ 5875 } 5876 /* Skip 2 cache entries */ 5877 // _TO_BOOL 5878 { 5879 int err = PyObject_IsTrue(value); 5880 Py_DECREF(value); 5881 if (err < 0) goto pop_1_error; 5882 res = err ? Py_True : Py_False; 5883 } 5884 stack_pointer[-1] = res; 5885 DISPATCH(); 5886 } 5887 TARGET(TO_BOOL_ALWAYS_TRUE)5888 TARGET(TO_BOOL_ALWAYS_TRUE) { 5889 _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; 5890 next_instr += 4; 5891 INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); 5892 static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); 5893 PyObject *owner; 5894 PyObject *value; 5895 PyObject *res; 5896 /* Skip 1 cache entry */ 5897 // _GUARD_TYPE_VERSION 5898 owner = stack_pointer[-1]; 5899 { 5900 uint32_t type_version = read_u32(&this_instr[2].cache); 5901 PyTypeObject *tp = Py_TYPE(owner); 5902 assert(type_version != 0); 5903 DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); 5904 } 5905 // _REPLACE_WITH_TRUE 5906 value = owner; 5907 { 5908 Py_DECREF(value); 5909 res = Py_True; 5910 } 5911 stack_pointer[-1] = res; 5912 DISPATCH(); 5913 } 5914 TARGET(TO_BOOL_BOOL)5915 TARGET(TO_BOOL_BOOL) { 5916 frame->instr_ptr = next_instr; 5917 next_instr += 4; 5918 INSTRUCTION_STATS(TO_BOOL_BOOL); 5919 static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); 5920 PyObject *value; 5921 /* Skip 1 cache entry */ 5922 /* Skip 2 cache entries */ 5923 value = stack_pointer[-1]; 5924 DEOPT_IF(!PyBool_Check(value), TO_BOOL); 5925 STAT_INC(TO_BOOL, hit); 5926 DISPATCH(); 5927 } 5928 TARGET(TO_BOOL_INT)5929 TARGET(TO_BOOL_INT) { 5930 frame->instr_ptr = next_instr; 5931 next_instr += 4; 5932 INSTRUCTION_STATS(TO_BOOL_INT); 5933 static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); 5934 PyObject *value; 5935 PyObject *res; 5936 /* Skip 1 cache entry */ 5937 /* Skip 2 cache entries */ 5938 value = stack_pointer[-1]; 5939 DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); 5940 STAT_INC(TO_BOOL, hit); 5941 if (_PyLong_IsZero((PyLongObject *)value)) { 5942 assert(_Py_IsImmortalLoose(value)); 5943 res = Py_False; 5944 } 5945 else { 5946 Py_DECREF(value); 5947 res = Py_True; 5948 } 5949 stack_pointer[-1] = res; 5950 DISPATCH(); 5951 } 5952 TARGET(TO_BOOL_LIST)5953 TARGET(TO_BOOL_LIST) { 5954 frame->instr_ptr = next_instr; 5955 next_instr += 4; 5956 INSTRUCTION_STATS(TO_BOOL_LIST); 5957 static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); 5958 PyObject *value; 5959 PyObject *res; 5960 /* Skip 1 cache entry */ 5961 /* Skip 2 cache entries */ 5962 value = stack_pointer[-1]; 5963 DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); 5964 STAT_INC(TO_BOOL, hit); 5965 res = Py_SIZE(value) ? Py_True : Py_False; 5966 Py_DECREF(value); 5967 stack_pointer[-1] = res; 5968 DISPATCH(); 5969 } 5970 TARGET(TO_BOOL_NONE)5971 TARGET(TO_BOOL_NONE) { 5972 frame->instr_ptr = next_instr; 5973 next_instr += 4; 5974 INSTRUCTION_STATS(TO_BOOL_NONE); 5975 static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); 5976 PyObject *value; 5977 PyObject *res; 5978 /* Skip 1 cache entry */ 5979 /* Skip 2 cache entries */ 5980 value = stack_pointer[-1]; 5981 // This one is a bit weird, because we expect *some* failures: 5982 DEOPT_IF(!Py_IsNone(value), TO_BOOL); 5983 STAT_INC(TO_BOOL, hit); 5984 res = Py_False; 5985 stack_pointer[-1] = res; 5986 DISPATCH(); 5987 } 5988 TARGET(TO_BOOL_STR)5989 TARGET(TO_BOOL_STR) { 5990 frame->instr_ptr = next_instr; 5991 next_instr += 4; 5992 INSTRUCTION_STATS(TO_BOOL_STR); 5993 static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); 5994 PyObject *value; 5995 PyObject *res; 5996 /* Skip 1 cache entry */ 5997 /* Skip 2 cache entries */ 5998 value = stack_pointer[-1]; 5999 DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); 6000 STAT_INC(TO_BOOL, hit); 6001 if (value == &_Py_STR(empty)) { 6002 assert(_Py_IsImmortalLoose(value)); 6003 res = Py_False; 6004 } 6005 else { 6006 assert(Py_SIZE(value)); 6007 Py_DECREF(value); 6008 res = Py_True; 6009 } 6010 stack_pointer[-1] = res; 6011 DISPATCH(); 6012 } 6013 TARGET(UNARY_INVERT)6014 TARGET(UNARY_INVERT) { 6015 frame->instr_ptr = next_instr; 6016 next_instr += 1; 6017 INSTRUCTION_STATS(UNARY_INVERT); 6018 PyObject *value; 6019 PyObject *res; 6020 value = stack_pointer[-1]; 6021 res = PyNumber_Invert(value); 6022 Py_DECREF(value); 6023 if (res == NULL) goto pop_1_error; 6024 stack_pointer[-1] = res; 6025 DISPATCH(); 6026 } 6027 TARGET(UNARY_NEGATIVE)6028 TARGET(UNARY_NEGATIVE) { 6029 frame->instr_ptr = next_instr; 6030 next_instr += 1; 6031 INSTRUCTION_STATS(UNARY_NEGATIVE); 6032 PyObject *value; 6033 PyObject *res; 6034 value = stack_pointer[-1]; 6035 res = PyNumber_Negative(value); 6036 Py_DECREF(value); 6037 if (res == NULL) goto pop_1_error; 6038 stack_pointer[-1] = res; 6039 DISPATCH(); 6040 } 6041 TARGET(UNARY_NOT)6042 TARGET(UNARY_NOT) { 6043 frame->instr_ptr = next_instr; 6044 next_instr += 1; 6045 INSTRUCTION_STATS(UNARY_NOT); 6046 PyObject *value; 6047 PyObject *res; 6048 value = stack_pointer[-1]; 6049 assert(PyBool_Check(value)); 6050 res = Py_IsFalse(value) ? Py_True : Py_False; 6051 stack_pointer[-1] = res; 6052 DISPATCH(); 6053 } 6054 TARGET(UNPACK_EX)6055 TARGET(UNPACK_EX) { 6056 frame->instr_ptr = next_instr; 6057 next_instr += 1; 6058 INSTRUCTION_STATS(UNPACK_EX); 6059 PyObject *seq; 6060 seq = stack_pointer[-1]; 6061 int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); 6062 PyObject **top = stack_pointer + totalargs - 1; 6063 int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); 6064 Py_DECREF(seq); 6065 if (res == 0) goto pop_1_error; 6066 stack_pointer += (oparg >> 8) + (oparg & 0xFF); 6067 DISPATCH(); 6068 } 6069 TARGET(UNPACK_SEQUENCE)6070 TARGET(UNPACK_SEQUENCE) { 6071 frame->instr_ptr = next_instr; 6072 next_instr += 2; 6073 INSTRUCTION_STATS(UNPACK_SEQUENCE); 6074 PREDICTED(UNPACK_SEQUENCE); 6075 _Py_CODEUNIT *this_instr = next_instr - 2; 6076 (void)this_instr; 6077 PyObject *seq; 6078 // _SPECIALIZE_UNPACK_SEQUENCE 6079 seq = stack_pointer[-1]; 6080 { 6081 uint16_t counter = read_u16(&this_instr[1].cache); 6082 (void)counter; 6083 #if ENABLE_SPECIALIZATION 6084 if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { 6085 next_instr = this_instr; 6086 _Py_Specialize_UnpackSequence(seq, next_instr, oparg); 6087 DISPATCH_SAME_OPARG(); 6088 } 6089 STAT_INC(UNPACK_SEQUENCE, deferred); 6090 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); 6091 #endif /* ENABLE_SPECIALIZATION */ 6092 (void)seq; 6093 (void)counter; 6094 } 6095 // _UNPACK_SEQUENCE 6096 { 6097 PyObject **top = stack_pointer + oparg - 1; 6098 int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); 6099 Py_DECREF(seq); 6100 if (res == 0) goto pop_1_error; 6101 } 6102 stack_pointer += -1 + oparg; 6103 DISPATCH(); 6104 } 6105 TARGET(UNPACK_SEQUENCE_LIST)6106 TARGET(UNPACK_SEQUENCE_LIST) { 6107 frame->instr_ptr = next_instr; 6108 next_instr += 2; 6109 INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); 6110 static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); 6111 PyObject *seq; 6112 PyObject **values; 6113 /* Skip 1 cache entry */ 6114 seq = stack_pointer[-1]; 6115 values = &stack_pointer[-1]; 6116 DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); 6117 DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); 6118 STAT_INC(UNPACK_SEQUENCE, hit); 6119 PyObject **items = _PyList_ITEMS(seq); 6120 for (int i = oparg; --i >= 0; ) { 6121 *values++ = Py_NewRef(items[i]); 6122 } 6123 Py_DECREF(seq); 6124 stack_pointer += -1 + oparg; 6125 DISPATCH(); 6126 } 6127 TARGET(UNPACK_SEQUENCE_TUPLE)6128 TARGET(UNPACK_SEQUENCE_TUPLE) { 6129 frame->instr_ptr = next_instr; 6130 next_instr += 2; 6131 INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); 6132 static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); 6133 PyObject *seq; 6134 PyObject **values; 6135 /* Skip 1 cache entry */ 6136 seq = stack_pointer[-1]; 6137 values = &stack_pointer[-1]; 6138 DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); 6139 DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); 6140 STAT_INC(UNPACK_SEQUENCE, hit); 6141 PyObject **items = _PyTuple_ITEMS(seq); 6142 for (int i = oparg; --i >= 0; ) { 6143 *values++ = Py_NewRef(items[i]); 6144 } 6145 Py_DECREF(seq); 6146 stack_pointer += -1 + oparg; 6147 DISPATCH(); 6148 } 6149 TARGET(UNPACK_SEQUENCE_TWO_TUPLE)6150 TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { 6151 frame->instr_ptr = next_instr; 6152 next_instr += 2; 6153 INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); 6154 static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); 6155 PyObject *seq; 6156 PyObject *val1; 6157 PyObject *val0; 6158 /* Skip 1 cache entry */ 6159 seq = stack_pointer[-1]; 6160 assert(oparg == 2); 6161 DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); 6162 DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); 6163 STAT_INC(UNPACK_SEQUENCE, hit); 6164 val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); 6165 val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); 6166 Py_DECREF(seq); 6167 stack_pointer[-1] = val1; 6168 stack_pointer[0] = val0; 6169 stack_pointer += 1; 6170 DISPATCH(); 6171 } 6172 TARGET(WITH_EXCEPT_START)6173 TARGET(WITH_EXCEPT_START) { 6174 frame->instr_ptr = next_instr; 6175 next_instr += 1; 6176 INSTRUCTION_STATS(WITH_EXCEPT_START); 6177 PyObject *val; 6178 PyObject *lasti; 6179 PyObject *exit_func; 6180 PyObject *res; 6181 val = stack_pointer[-1]; 6182 lasti = stack_pointer[-3]; 6183 exit_func = stack_pointer[-4]; 6184 /* At the top of the stack are 4 values: 6185 - val: TOP = exc_info() 6186 - unused: SECOND = previous exception 6187 - lasti: THIRD = lasti of exception in exc_info() 6188 - exit_func: FOURTH = the context.__exit__ bound method 6189 We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). 6190 Then we push the __exit__ return value. 6191 */ 6192 PyObject *exc, *tb; 6193 assert(val && PyExceptionInstance_Check(val)); 6194 exc = PyExceptionInstance_Class(val); 6195 tb = PyException_GetTraceback(val); 6196 if (tb == NULL) { 6197 tb = Py_None; 6198 } 6199 else { 6200 Py_DECREF(tb); 6201 } 6202 assert(PyLong_Check(lasti)); 6203 (void)lasti; // Shut up compiler warning if asserts are off 6204 PyObject *stack[4] = {NULL, exc, val, tb}; 6205 res = PyObject_Vectorcall(exit_func, stack + 1, 6206 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); 6207 if (res == NULL) goto error; 6208 stack_pointer[0] = res; 6209 stack_pointer += 1; 6210 DISPATCH(); 6211 } 6212 TARGET(YIELD_VALUE)6213 TARGET(YIELD_VALUE) { 6214 frame->instr_ptr = next_instr; 6215 next_instr += 1; 6216 INSTRUCTION_STATS(YIELD_VALUE); 6217 PyObject *retval; 6218 PyObject *value; 6219 retval = stack_pointer[-1]; 6220 // NOTE: It's important that YIELD_VALUE never raises an exception! 6221 // The compiler treats any exception raised here as a failed close() 6222 // or throw() call. 6223 #if TIER_ONE 6224 assert(frame != &entry_frame); 6225 #endif 6226 frame->instr_ptr++; 6227 PyGenObject *gen = _PyFrame_GetGenerator(frame); 6228 assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); 6229 assert(oparg == 0 || oparg == 1); 6230 gen->gi_frame_state = FRAME_SUSPENDED + oparg; 6231 stack_pointer += -1; 6232 _PyFrame_SetStackPointer(frame, stack_pointer); 6233 tstate->exc_info = gen->gi_exc_state.previous_item; 6234 gen->gi_exc_state.previous_item = NULL; 6235 _Py_LeaveRecursiveCallPy(tstate); 6236 _PyInterpreterFrame *gen_frame = frame; 6237 frame = tstate->current_frame = frame->previous; 6238 gen_frame->previous = NULL; 6239 /* We don't know which of these is relevant here, so keep them equal */ 6240 assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); 6241 #if TIER_ONE 6242 assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || 6243 frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || 6244 _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || 6245 _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || 6246 _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || 6247 _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); 6248 #endif 6249 LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); 6250 LOAD_SP(); 6251 value = retval; 6252 LLTRACE_RESUME_FRAME(); 6253 stack_pointer[0] = value; 6254 stack_pointer += 1; 6255 DISPATCH(); 6256 } 6257 #undef TIER_ONE 6258