1 #include "Python.h"
2
3 #include "opcode.h"
4
5 #include "pycore_code.h"
6 #include "pycore_descrobject.h" // _PyMethodWrapper_Type
7 #include "pycore_dict.h" // DICT_KEYS_UNICODE
8 #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()
9 #include "pycore_long.h" // _PyLong_IsNonNegativeCompact()
10 #include "pycore_moduleobject.h"
11 #include "pycore_object.h"
12 #include "pycore_opcode_metadata.h" // _PyOpcode_Caches
13 #include "pycore_uop_metadata.h" // _PyOpcode_uop_name
14 #include "pycore_uop_ids.h" // MAX_UOP_ID
15 #include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START
16 #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock()
17 #include "pycore_runtime.h" // _Py_ID()
18
19 #include <stdlib.h> // rand()
20
21 extern const char *_PyUOpName(int index);
22
23 /* For guidance on adding or extending families of instructions see
24 * ./adaptive.md
25 */
26
27 #ifdef Py_STATS
28 GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 };
29 static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats };
30 PyStats *_Py_stats = NULL;
31
32 #if PYSTATS_MAX_UOP_ID < MAX_UOP_ID
33 #error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID"
34 #endif
35
36 #define ADD_STAT_TO_DICT(res, field) \
37 do { \
38 PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \
39 if (val == NULL) { \
40 Py_DECREF(res); \
41 return NULL; \
42 } \
43 if (PyDict_SetItemString(res, #field, val) == -1) { \
44 Py_DECREF(res); \
45 Py_DECREF(val); \
46 return NULL; \
47 } \
48 Py_DECREF(val); \
49 } while(0);
50
51 static PyObject*
stats_to_dict(SpecializationStats * stats)52 stats_to_dict(SpecializationStats *stats)
53 {
54 PyObject *res = PyDict_New();
55 if (res == NULL) {
56 return NULL;
57 }
58 ADD_STAT_TO_DICT(res, success);
59 ADD_STAT_TO_DICT(res, failure);
60 ADD_STAT_TO_DICT(res, hit);
61 ADD_STAT_TO_DICT(res, deferred);
62 ADD_STAT_TO_DICT(res, miss);
63 ADD_STAT_TO_DICT(res, deopt);
64 PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS);
65 if (failure_kinds == NULL) {
66 Py_DECREF(res);
67 return NULL;
68 }
69 for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) {
70 PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]);
71 if (stat == NULL) {
72 Py_DECREF(res);
73 Py_DECREF(failure_kinds);
74 return NULL;
75 }
76 PyTuple_SET_ITEM(failure_kinds, i, stat);
77 }
78 if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) {
79 Py_DECREF(res);
80 Py_DECREF(failure_kinds);
81 return NULL;
82 }
83 Py_DECREF(failure_kinds);
84 return res;
85 }
86 #undef ADD_STAT_TO_DICT
87
88 static int
add_stat_dict(PyObject * res,int opcode,const char * name)89 add_stat_dict(
90 PyObject *res,
91 int opcode,
92 const char *name) {
93
94 SpecializationStats *stats = &_Py_stats_struct.opcode_stats[opcode].specialization;
95 PyObject *d = stats_to_dict(stats);
96 if (d == NULL) {
97 return -1;
98 }
99 int err = PyDict_SetItemString(res, name, d);
100 Py_DECREF(d);
101 return err;
102 }
103
104 PyObject*
_Py_GetSpecializationStats(void)105 _Py_GetSpecializationStats(void) {
106 PyObject *stats = PyDict_New();
107 if (stats == NULL) {
108 return NULL;
109 }
110 int err = 0;
111 err += add_stat_dict(stats, CONTAINS_OP, "contains_op");
112 err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr");
113 err += add_stat_dict(stats, LOAD_ATTR, "load_attr");
114 err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
115 err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
116 err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");
117 err += add_stat_dict(stats, STORE_ATTR, "store_attr");
118 err += add_stat_dict(stats, CALL, "call");
119 err += add_stat_dict(stats, BINARY_OP, "binary_op");
120 err += add_stat_dict(stats, COMPARE_OP, "compare_op");
121 err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence");
122 err += add_stat_dict(stats, FOR_ITER, "for_iter");
123 err += add_stat_dict(stats, TO_BOOL, "to_bool");
124 err += add_stat_dict(stats, SEND, "send");
125 if (err < 0) {
126 Py_DECREF(stats);
127 return NULL;
128 }
129 return stats;
130 }
131
132
133 #define PRINT_STAT(i, field) \
134 if (stats[i].field) { \
135 fprintf(out, " opcode[%s]." #field " : %" PRIu64 "\n", _PyOpcode_OpName[i], stats[i].field); \
136 }
137
138 static void
print_spec_stats(FILE * out,OpcodeStats * stats)139 print_spec_stats(FILE *out, OpcodeStats *stats)
140 {
141 /* Mark some opcodes as specializable for stats,
142 * even though we don't specialize them yet. */
143 fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n");
144 fprintf(out, "opcode[STORE_SLICE].specializable : 1\n");
145 for (int i = 0; i < 256; i++) {
146 if (_PyOpcode_Caches[i] && i != JUMP_BACKWARD) {
147 fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]);
148 }
149 PRINT_STAT(i, specialization.success);
150 PRINT_STAT(i, specialization.failure);
151 PRINT_STAT(i, specialization.hit);
152 PRINT_STAT(i, specialization.deferred);
153 PRINT_STAT(i, specialization.miss);
154 PRINT_STAT(i, specialization.deopt);
155 PRINT_STAT(i, execution_count);
156 for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) {
157 uint64_t val = stats[i].specialization.failure_kinds[j];
158 if (val) {
159 fprintf(out, " opcode[%s].specialization.failure_kinds[%d] : %"
160 PRIu64 "\n", _PyOpcode_OpName[i], j, val);
161 }
162 }
163 for (int j = 0; j < 256; j++) {
164 if (stats[i].pair_count[j]) {
165 fprintf(out, "opcode[%s].pair_count[%s] : %" PRIu64 "\n",
166 _PyOpcode_OpName[i], _PyOpcode_OpName[j], stats[i].pair_count[j]);
167 }
168 }
169 }
170 }
171 #undef PRINT_STAT
172
173
174 static void
print_call_stats(FILE * out,CallStats * stats)175 print_call_stats(FILE *out, CallStats *stats)
176 {
177 fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls);
178 fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls);
179 fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed);
180 fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created);
181 for (int i = 0; i < EVAL_CALL_KINDS; i++) {
182 fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]);
183 }
184 }
185
186 static void
print_object_stats(FILE * out,ObjectStats * stats)187 print_object_stats(FILE *out, ObjectStats *stats)
188 {
189 fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist);
190 fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist);
191 fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations);
192 fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512);
193 fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k);
194 fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
195 fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
196 fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values);
197 fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs);
198 fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs);
199 fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
200 fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs);
201 fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
202 fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
203 fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
204 fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass);
205 fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits);
206 fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses);
207 fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions);
208 fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits);
209 fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses);
210 }
211
212 static void
print_gc_stats(FILE * out,GCStats * stats)213 print_gc_stats(FILE *out, GCStats *stats)
214 {
215 for (int i = 0; i < NUM_GENERATIONS; i++) {
216 fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections);
217 fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits);
218 fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected);
219 }
220 }
221
222 #ifdef _Py_TIER2
223 static void
print_histogram(FILE * out,const char * name,uint64_t hist[_Py_UOP_HIST_SIZE])224 print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE])
225 {
226 for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) {
227 fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]);
228 }
229 }
230
231 static void
print_optimization_stats(FILE * out,OptimizationStats * stats)232 print_optimization_stats(FILE *out, OptimizationStats *stats)
233 {
234 fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts);
235 fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created);
236 fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed);
237 fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed);
238 fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow);
239 fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow);
240 fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long);
241 fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short);
242 fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop);
243 fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call);
244 fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence);
245 fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated);
246
247 print_histogram(out, "Trace length", stats->trace_length_hist);
248 print_histogram(out, "Trace run length", stats->trace_run_length_hist);
249 print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist);
250
251 fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts);
252 fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes);
253 fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n",
254 stats->optimizer_failure_reason_no_memory);
255 fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed);
256 fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys);
257 for (int i = 0; i <= MAX_UOP_ID; i++) {
258 if (stats->opcode[i].execution_count) {
259 fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count);
260 }
261 if (stats->opcode[i].miss) {
262 fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss);
263 }
264 }
265 for (int i = 0; i < 256; i++) {
266 if (stats->unsupported_opcode[i]) {
267 fprintf(
268 out,
269 "unsupported_opcode[%s].count : %" PRIu64 "\n",
270 _PyOpcode_OpName[i],
271 stats->unsupported_opcode[i]
272 );
273 }
274 }
275
276 for (int i = 1; i <= MAX_UOP_ID; i++){
277 for (int j = 1; j <= MAX_UOP_ID; j++) {
278 if (stats->opcode[i].pair_count[j]) {
279 fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n",
280 _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]);
281 }
282 }
283 }
284 for (int i = 0; i < MAX_UOP_ID; i++) {
285 if (stats->error_in_opcode[i]) {
286 fprintf(
287 out,
288 "error_in_opcode[%s].count : %" PRIu64 "\n",
289 _PyUOpName(i),
290 stats->error_in_opcode[i]
291 );
292 }
293 }
294 }
295 #endif
296
297 static void
print_rare_event_stats(FILE * out,RareEventStats * stats)298 print_rare_event_stats(FILE *out, RareEventStats *stats)
299 {
300 fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class);
301 fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases);
302 fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func);
303 fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict);
304 fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification);
305 fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification);
306 fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification);
307 }
308
309 static void
print_stats(FILE * out,PyStats * stats)310 print_stats(FILE *out, PyStats *stats)
311 {
312 print_spec_stats(out, stats->opcode_stats);
313 print_call_stats(out, &stats->call_stats);
314 print_object_stats(out, &stats->object_stats);
315 print_gc_stats(out, stats->gc_stats);
316 #ifdef _Py_TIER2
317 print_optimization_stats(out, &stats->optimization_stats);
318 #endif
319 print_rare_event_stats(out, &stats->rare_event_stats);
320 }
321
322 void
_Py_StatsOn(void)323 _Py_StatsOn(void)
324 {
325 _Py_stats = &_Py_stats_struct;
326 }
327
328 void
_Py_StatsOff(void)329 _Py_StatsOff(void)
330 {
331 _Py_stats = NULL;
332 }
333
334 void
_Py_StatsClear(void)335 _Py_StatsClear(void)
336 {
337 memset(&_py_gc_stats, 0, sizeof(_py_gc_stats));
338 memset(&_Py_stats_struct, 0, sizeof(_Py_stats_struct));
339 _Py_stats_struct.gc_stats = _py_gc_stats;
340 }
341
342 static int
mem_is_zero(unsigned char * ptr,size_t size)343 mem_is_zero(unsigned char *ptr, size_t size)
344 {
345 for (size_t i=0; i < size; i++) {
346 if (*ptr != 0) {
347 return 0;
348 }
349 ptr++;
350 }
351 return 1;
352 }
353
354 int
_Py_PrintSpecializationStats(int to_file)355 _Py_PrintSpecializationStats(int to_file)
356 {
357 PyStats *stats = &_Py_stats_struct;
358 #define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA)))
359 int is_zero = (
360 MEM_IS_ZERO(stats->gc_stats) // is a pointer
361 && MEM_IS_ZERO(&stats->opcode_stats)
362 && MEM_IS_ZERO(&stats->call_stats)
363 && MEM_IS_ZERO(&stats->object_stats)
364 );
365 #undef MEM_IS_ZERO
366 if (is_zero) {
367 // gh-108753: -X pystats command line was used, but then _stats_off()
368 // and _stats_clear() have been called: in this case, avoid printing
369 // useless "all zeros" statistics.
370 return 0;
371 }
372
373 FILE *out = stderr;
374 if (to_file) {
375 /* Write to a file instead of stderr. */
376 # ifdef MS_WINDOWS
377 const char *dirname = "c:\\temp\\py_stats\\";
378 # else
379 const char *dirname = "/tmp/py_stats/";
380 # endif
381 /* Use random 160 bit number as file name,
382 * to avoid both accidental collisions and
383 * symlink attacks. */
384 unsigned char rand[20];
385 char hex_name[41];
386 _PyOS_URandomNonblock(rand, 20);
387 for (int i = 0; i < 20; i++) {
388 hex_name[2*i] = Py_hexdigits[rand[i]&15];
389 hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15];
390 }
391 hex_name[40] = '\0';
392 char buf[64];
393 assert(strlen(dirname) + 40 + strlen(".txt") < 64);
394 sprintf(buf, "%s%s.txt", dirname, hex_name);
395 FILE *fout = fopen(buf, "w");
396 if (fout) {
397 out = fout;
398 }
399 }
400 else {
401 fprintf(out, "Specialization stats:\n");
402 }
403 print_stats(out, stats);
404 if (out != stderr) {
405 fclose(out);
406 }
407 return 1;
408 }
409
410 #define SPECIALIZATION_FAIL(opcode, kind) \
411 do { \
412 if (_Py_stats) { \
413 _Py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \
414 } \
415 } while (0)
416
417 #endif // Py_STATS
418
419
420 #ifndef SPECIALIZATION_FAIL
421 # define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
422 #endif
423
424 // Initialize warmup counters and insert superinstructions. This cannot fail.
425 void
_PyCode_Quicken(PyCodeObject * code)426 _PyCode_Quicken(PyCodeObject *code)
427 {
428 #if ENABLE_SPECIALIZATION
429 int opcode = 0;
430 _Py_CODEUNIT *instructions = _PyCode_CODE(code);
431 for (int i = 0; i < Py_SIZE(code); i++) {
432 opcode = _Py_GetBaseOpcode(code, i);
433 assert(opcode < MIN_INSTRUMENTED_OPCODE);
434 int caches = _PyOpcode_Caches[opcode];
435 if (caches) {
436 // The initial value depends on the opcode
437 switch (opcode) {
438 case JUMP_BACKWARD:
439 instructions[i + 1].counter = initial_jump_backoff_counter();
440 break;
441 case POP_JUMP_IF_FALSE:
442 case POP_JUMP_IF_TRUE:
443 case POP_JUMP_IF_NONE:
444 case POP_JUMP_IF_NOT_NONE:
445 instructions[i + 1].cache = 0x5555; // Alternating 0, 1 bits
446 break;
447 default:
448 instructions[i + 1].counter = adaptive_counter_warmup();
449 break;
450 }
451 i += caches;
452 }
453 }
454 #endif /* ENABLE_SPECIALIZATION */
455 }
456
457 #define SIMPLE_FUNCTION 0
458
459 /* Common */
460
461 #define SPEC_FAIL_OTHER 0
462 #define SPEC_FAIL_NO_DICT 1
463 #define SPEC_FAIL_OVERRIDDEN 2
464 #define SPEC_FAIL_OUT_OF_VERSIONS 3
465 #define SPEC_FAIL_OUT_OF_RANGE 4
466 #define SPEC_FAIL_EXPECTED_ERROR 5
467 #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6
468 #define SPEC_FAIL_CODE_COMPLEX_PARAMETERS 7
469 #define SPEC_FAIL_CODE_NOT_OPTIMIZED 8
470
471
472 #define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17
473 #define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18
474
475 /* Super */
476
477 #define SPEC_FAIL_SUPER_BAD_CLASS 9
478 #define SPEC_FAIL_SUPER_SHADOWED 10
479
480 /* Attributes */
481
482 #define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 9
483 #define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 10
484 #define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 11
485 #define SPEC_FAIL_ATTR_METHOD 12
486 #define SPEC_FAIL_ATTR_MUTABLE_CLASS 13
487 #define SPEC_FAIL_ATTR_PROPERTY 14
488 #define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 15
489 #define SPEC_FAIL_ATTR_READ_ONLY 16
490 #define SPEC_FAIL_ATTR_AUDITED_SLOT 17
491 #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18
492 #define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19
493 #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20
494 #define SPEC_FAIL_ATTR_SHADOWED 21
495 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22
496 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23
497 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24
498
499 #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26
500 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27
501 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28
502 #define SPEC_FAIL_ATTR_NOT_IN_KEYS 29
503 #define SPEC_FAIL_ATTR_NOT_IN_DICT 30
504 #define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE 31
505 #define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR 32
506 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33
507
508 /* Binary subscr and store subscr */
509
510 #define SPEC_FAIL_SUBSCR_ARRAY_INT 9
511 #define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10
512 #define SPEC_FAIL_SUBSCR_LIST_SLICE 11
513 #define SPEC_FAIL_SUBSCR_TUPLE_SLICE 12
514 #define SPEC_FAIL_SUBSCR_STRING_SLICE 14
515 #define SPEC_FAIL_SUBSCR_BUFFER_INT 15
516 #define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16
517 #define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17
518
519 /* Store subscr */
520 #define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18
521 #define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE 19
522 #define SPEC_FAIL_SUBSCR_PY_SIMPLE 20
523 #define SPEC_FAIL_SUBSCR_PY_OTHER 21
524 #define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE 22
525 #define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE 23
526
527 /* Binary op */
528
529 #define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 9
530 #define SPEC_FAIL_BINARY_OP_ADD_OTHER 10
531 #define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 11
532 #define SPEC_FAIL_BINARY_OP_AND_INT 12
533 #define SPEC_FAIL_BINARY_OP_AND_OTHER 13
534 #define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 14
535 #define SPEC_FAIL_BINARY_OP_LSHIFT 15
536 #define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 16
537 #define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 17
538 #define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 18
539 #define SPEC_FAIL_BINARY_OP_OR 19
540 #define SPEC_FAIL_BINARY_OP_POWER 20
541 #define SPEC_FAIL_BINARY_OP_REMAINDER 21
542 #define SPEC_FAIL_BINARY_OP_RSHIFT 22
543 #define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 23
544 #define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 24
545 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 25
546 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 26
547 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 27
548 #define SPEC_FAIL_BINARY_OP_XOR 28
549
550 /* Calls */
551
552 #define SPEC_FAIL_CALL_INSTANCE_METHOD 11
553 #define SPEC_FAIL_CALL_CMETHOD 12
554 #define SPEC_FAIL_CALL_CFUNC_VARARGS 13
555 #define SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS 14
556 #define SPEC_FAIL_CALL_CFUNC_NOARGS 15
557 #define SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS 16
558 #define SPEC_FAIL_CALL_METH_DESCR_VARARGS 17
559 #define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18
560 #define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19
561 #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20
562 #define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21
563 #define SPEC_FAIL_CALL_PEP_523 22
564 #define SPEC_FAIL_CALL_BOUND_METHOD 23
565 #define SPEC_FAIL_CALL_STR 24
566 #define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 25
567 #define SPEC_FAIL_CALL_CLASS_MUTABLE 26
568 #define SPEC_FAIL_CALL_METHOD_WRAPPER 28
569 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29
570 #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30
571 #define SPEC_FAIL_CALL_METACLASS 31
572 #define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32
573
574 /* COMPARE_OP */
575 #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
576 #define SPEC_FAIL_COMPARE_OP_STRING 13
577 #define SPEC_FAIL_COMPARE_OP_BIG_INT 14
578 #define SPEC_FAIL_COMPARE_OP_BYTES 15
579 #define SPEC_FAIL_COMPARE_OP_TUPLE 16
580 #define SPEC_FAIL_COMPARE_OP_LIST 17
581 #define SPEC_FAIL_COMPARE_OP_SET 18
582 #define SPEC_FAIL_COMPARE_OP_BOOL 19
583 #define SPEC_FAIL_COMPARE_OP_BASEOBJECT 20
584 #define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 21
585 #define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 22
586
587 /* FOR_ITER and SEND */
588 #define SPEC_FAIL_ITER_GENERATOR 10
589 #define SPEC_FAIL_ITER_COROUTINE 11
590 #define SPEC_FAIL_ITER_ASYNC_GENERATOR 12
591 #define SPEC_FAIL_ITER_LIST 13
592 #define SPEC_FAIL_ITER_TUPLE 14
593 #define SPEC_FAIL_ITER_SET 15
594 #define SPEC_FAIL_ITER_STRING 16
595 #define SPEC_FAIL_ITER_BYTES 17
596 #define SPEC_FAIL_ITER_RANGE 18
597 #define SPEC_FAIL_ITER_ITERTOOLS 19
598 #define SPEC_FAIL_ITER_DICT_KEYS 20
599 #define SPEC_FAIL_ITER_DICT_ITEMS 21
600 #define SPEC_FAIL_ITER_DICT_VALUES 22
601 #define SPEC_FAIL_ITER_ENUMERATE 23
602 #define SPEC_FAIL_ITER_MAP 24
603 #define SPEC_FAIL_ITER_ZIP 25
604 #define SPEC_FAIL_ITER_SEQ_ITER 26
605 #define SPEC_FAIL_ITER_REVERSED_LIST 27
606 #define SPEC_FAIL_ITER_CALLABLE 28
607 #define SPEC_FAIL_ITER_ASCII_STRING 29
608 #define SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND 30
609
610 // UNPACK_SEQUENCE
611
612 #define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 9
613 #define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 10
614
615 // TO_BOOL
616 #define SPEC_FAIL_TO_BOOL_BYTEARRAY 9
617 #define SPEC_FAIL_TO_BOOL_BYTES 10
618 #define SPEC_FAIL_TO_BOOL_DICT 11
619 #define SPEC_FAIL_TO_BOOL_FLOAT 12
620 #define SPEC_FAIL_TO_BOOL_MAPPING 13
621 #define SPEC_FAIL_TO_BOOL_MEMORY_VIEW 14
622 #define SPEC_FAIL_TO_BOOL_NUMBER 15
623 #define SPEC_FAIL_TO_BOOL_SEQUENCE 16
624 #define SPEC_FAIL_TO_BOOL_SET 17
625 #define SPEC_FAIL_TO_BOOL_TUPLE 18
626
627 // CONTAINS_OP
628 #define SPEC_FAIL_CONTAINS_OP_STR 9
629 #define SPEC_FAIL_CONTAINS_OP_TUPLE 10
630 #define SPEC_FAIL_CONTAINS_OP_LIST 11
631 #define SPEC_FAIL_CONTAINS_OP_USER_CLASS 12
632
633 static int function_kind(PyCodeObject *code);
634 static bool function_check_args(PyObject *o, int expected_argcount, int opcode);
635 static uint32_t function_get_version(PyObject *o, int opcode);
636 static uint32_t type_get_version(PyTypeObject *t, int opcode);
637
638 static int
specialize_module_load_attr(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name)639 specialize_module_load_attr(
640 PyObject *owner, _Py_CODEUNIT *instr, PyObject *name
641 ) {
642 _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
643 PyModuleObject *m = (PyModuleObject *)owner;
644 assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
645 PyDictObject *dict = (PyDictObject *)m->md_dict;
646 if (dict == NULL) {
647 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT);
648 return -1;
649 }
650 if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
651 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT);
652 return -1;
653 }
654 Py_ssize_t index = _PyDict_LookupIndex(dict, &_Py_ID(__getattr__));
655 assert(index != DKIX_ERROR);
656 if (index != DKIX_EMPTY) {
657 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND);
658 return -1;
659 }
660 index = _PyDict_LookupIndex(dict, name);
661 assert (index != DKIX_ERROR);
662 if (index != (uint16_t)index) {
663 SPECIALIZATION_FAIL(LOAD_ATTR,
664 index == DKIX_EMPTY ?
665 SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND :
666 SPEC_FAIL_OUT_OF_RANGE);
667 return -1;
668 }
669 uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
670 _PyInterpreterState_GET(), dict->ma_keys);
671 if (keys_version == 0) {
672 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
673 return -1;
674 }
675 write_u32(cache->version, keys_version);
676 cache->index = (uint16_t)index;
677 instr->op.code = LOAD_ATTR_MODULE;
678 return 0;
679 }
680
681
682
683 /* Attribute specialization */
684
685 void
_Py_Specialize_LoadSuperAttr(PyObject * global_super,PyObject * cls,_Py_CODEUNIT * instr,int load_method)686 _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, _Py_CODEUNIT *instr, int load_method) {
687 assert(ENABLE_SPECIALIZATION);
688 assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR);
689 _PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1);
690 if (global_super != (PyObject *)&PySuper_Type) {
691 SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED);
692 goto fail;
693 }
694 if (!PyType_Check(cls)) {
695 SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS);
696 goto fail;
697 }
698 instr->op.code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR;
699 goto success;
700
701 fail:
702 STAT_INC(LOAD_SUPER_ATTR, failure);
703 assert(!PyErr_Occurred());
704 instr->op.code = LOAD_SUPER_ATTR;
705 cache->counter = adaptive_counter_backoff(cache->counter);
706 return;
707 success:
708 STAT_INC(LOAD_SUPER_ATTR, success);
709 assert(!PyErr_Occurred());
710 cache->counter = adaptive_counter_cooldown();
711 }
712
713 typedef enum {
714 OVERRIDING, /* Is an overriding descriptor, and will remain so. */
715 METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */
716 PROPERTY, /* Is a property */
717 OBJECT_SLOT, /* Is an object slot descriptor */
718 OTHER_SLOT, /* Is a slot descriptor of another type */
719 NON_OVERRIDING, /* Is another non-overriding descriptor, and is an instance of an immutable class*/
720 BUILTIN_CLASSMETHOD, /* Builtin methods with METH_CLASS */
721 PYTHON_CLASSMETHOD, /* Python classmethod(func) object */
722 NON_DESCRIPTOR, /* Is not a descriptor, and is an instance of an immutable class */
723 MUTABLE, /* Instance of a mutable class; might, or might not, be a descriptor */
724 ABSENT, /* Attribute is not present on the class */
725 DUNDER_CLASS, /* __class__ attribute */
726 GETSET_OVERRIDDEN, /* __getattribute__ or __setattr__ has been overridden */
727 GETATTRIBUTE_IS_PYTHON_FUNCTION /* Descriptor requires calling a Python __getattribute__ */
728 } DescriptorClassification;
729
730
731 static DescriptorClassification
analyze_descriptor(PyTypeObject * type,PyObject * name,PyObject ** descr,int store)732 analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store)
733 {
734 bool has_getattr = false;
735 if (store) {
736 if (type->tp_setattro != PyObject_GenericSetAttr) {
737 *descr = NULL;
738 return GETSET_OVERRIDDEN;
739 }
740 }
741 else {
742 getattrofunc getattro_slot = type->tp_getattro;
743 if (getattro_slot == PyObject_GenericGetAttr) {
744 /* Normal attribute lookup; */
745 has_getattr = false;
746 }
747 else if (getattro_slot == _Py_slot_tp_getattr_hook ||
748 getattro_slot == _Py_slot_tp_getattro) {
749 /* One or both of __getattribute__ or __getattr__ may have been
750 overridden See typeobject.c for why these functions are special. */
751 PyObject *getattribute = _PyType_Lookup(type,
752 &_Py_ID(__getattribute__));
753 PyInterpreterState *interp = _PyInterpreterState_GET();
754 bool has_custom_getattribute = getattribute != NULL &&
755 getattribute != interp->callable_cache.object__getattribute__;
756 has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL;
757 if (has_custom_getattribute) {
758 if (getattro_slot == _Py_slot_tp_getattro &&
759 !has_getattr &&
760 Py_IS_TYPE(getattribute, &PyFunction_Type)) {
761 *descr = getattribute;
762 return GETATTRIBUTE_IS_PYTHON_FUNCTION;
763 }
764 /* Potentially both __getattr__ and __getattribute__ are set.
765 Too complicated */
766 *descr = NULL;
767 return GETSET_OVERRIDDEN;
768 }
769 /* Potentially has __getattr__ but no custom __getattribute__.
770 Fall through to usual descriptor analysis.
771 Usual attribute lookup should only be allowed at runtime
772 if we can guarantee that there is no way an exception can be
773 raised. This means some specializations, e.g. specializing
774 for property() isn't safe.
775 */
776 }
777 else {
778 *descr = NULL;
779 return GETSET_OVERRIDDEN;
780 }
781 }
782 PyObject *descriptor = _PyType_Lookup(type, name);
783 *descr = descriptor;
784 if (descriptor == NULL) {
785 return ABSENT;
786 }
787 PyTypeObject *desc_cls = Py_TYPE(descriptor);
788 if (!(desc_cls->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) {
789 return MUTABLE;
790 }
791 if (desc_cls->tp_descr_set) {
792 if (desc_cls == &PyMemberDescr_Type) {
793 PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor;
794 struct PyMemberDef *dmem = member->d_member;
795 if (dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT) {
796 return OBJECT_SLOT;
797 }
798 return OTHER_SLOT;
799 }
800 if (desc_cls == &PyProperty_Type) {
801 /* We can't detect at runtime whether an attribute exists
802 with property. So that means we may have to call
803 __getattr__. */
804 return has_getattr ? GETSET_OVERRIDDEN : PROPERTY;
805 }
806 if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) {
807 if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) {
808 return DUNDER_CLASS;
809 }
810 }
811 if (store) {
812 return OVERRIDING;
813 }
814 }
815 if (desc_cls->tp_descr_get) {
816 if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) {
817 return METHOD;
818 }
819 if (Py_IS_TYPE(descriptor, &PyClassMethodDescr_Type)) {
820 return BUILTIN_CLASSMETHOD;
821 }
822 if (Py_IS_TYPE(descriptor, &PyClassMethod_Type)) {
823 return PYTHON_CLASSMETHOD;
824 }
825 return NON_OVERRIDING;
826 }
827 return NON_DESCRIPTOR;
828 }
829
830 static int
specialize_dict_access(PyObject * owner,_Py_CODEUNIT * instr,PyTypeObject * type,DescriptorClassification kind,PyObject * name,int base_op,int values_op,int hint_op)831 specialize_dict_access(
832 PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
833 DescriptorClassification kind, PyObject *name,
834 int base_op, int values_op, int hint_op)
835 {
836 assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
837 kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD);
838 // No descriptor, or non overriding.
839 if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
840 SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
841 return 0;
842 }
843 _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
844 if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && _PyObject_InlineValues(owner)->valid) {
845 PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
846 assert(PyUnicode_CheckExact(name));
847 Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
848 assert (index != DKIX_ERROR);
849 if (index != (uint16_t)index) {
850 SPECIALIZATION_FAIL(base_op,
851 index == DKIX_EMPTY ?
852 SPEC_FAIL_ATTR_NOT_IN_KEYS :
853 SPEC_FAIL_OUT_OF_RANGE);
854 return 0;
855 }
856 write_u32(cache->version, type->tp_version_tag);
857 cache->index = (uint16_t)index;
858 instr->op.code = values_op;
859 }
860 else {
861 PyDictObject *dict = _PyObject_GetManagedDict(owner);
862 if (dict == NULL || !PyDict_CheckExact(dict)) {
863 SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
864 return 0;
865 }
866 // We found an instance with a __dict__.
867 if (dict->ma_values) {
868 SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT);
869 return 0;
870 }
871 Py_ssize_t index =
872 _PyDict_LookupIndex(dict, name);
873 if (index != (uint16_t)index) {
874 SPECIALIZATION_FAIL(base_op,
875 index == DKIX_EMPTY ?
876 SPEC_FAIL_ATTR_NOT_IN_DICT :
877 SPEC_FAIL_OUT_OF_RANGE);
878 return 0;
879 }
880 cache->index = (uint16_t)index;
881 write_u32(cache->version, type->tp_version_tag);
882 instr->op.code = hint_op;
883 }
884 return 1;
885 }
886
887 static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
888 PyObject* descr, DescriptorClassification kind, bool is_method);
889 static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);
890
891 void
_Py_Specialize_LoadAttr(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name)892 _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
893 {
894 assert(ENABLE_SPECIALIZATION);
895 assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
896 _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
897 PyTypeObject *type = Py_TYPE(owner);
898 if (!_PyType_IsReady(type)) {
899 // We *might* not really need this check, but we inherited it from
900 // PyObject_GenericGetAttr and friends... and this way we still do the
901 // right thing if someone forgets to call PyType_Ready(type):
902 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
903 goto fail;
904 }
905 if (PyModule_CheckExact(owner)) {
906 if (specialize_module_load_attr(owner, instr, name))
907 {
908 goto fail;
909 }
910 goto success;
911 }
912 if (PyType_Check(owner)) {
913 if (specialize_class_load_attr(owner, instr, name)) {
914 goto fail;
915 }
916 goto success;
917 }
918 PyObject *descr = NULL;
919 DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);
920 assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
921 if (type_get_version(type, LOAD_ATTR) == 0) {
922 goto fail;
923 }
924 switch(kind) {
925 case OVERRIDING:
926 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
927 goto fail;
928 case METHOD:
929 {
930 int oparg = instr->op.arg;
931 if (oparg & 1) {
932 if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, true)) {
933 goto success;
934 }
935 }
936 else {
937 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
938 }
939 goto fail;
940 }
941 case PROPERTY:
942 {
943 _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
944 assert(Py_TYPE(descr) == &PyProperty_Type);
945 PyObject *fget = ((_PyPropertyObject *)descr)->prop_get;
946 if (fget == NULL) {
947 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
948 goto fail;
949 }
950 if (!Py_IS_TYPE(fget, &PyFunction_Type)) {
951 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION);
952 goto fail;
953 }
954 if (!function_check_args(fget, 1, LOAD_ATTR)) {
955 goto fail;
956 }
957 if (instr->op.arg & 1) {
958 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
959 goto fail;
960 }
961 uint32_t version = function_get_version(fget, LOAD_ATTR);
962 if (version == 0) {
963 goto fail;
964 }
965 if (_PyInterpreterState_GET()->eval_frame) {
966 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
967 goto fail;
968 }
969 write_u32(lm_cache->keys_version, version);
970 assert(type->tp_version_tag != 0);
971 write_u32(lm_cache->type_version, type->tp_version_tag);
972 /* borrowed */
973 write_obj(lm_cache->descr, fget);
974 instr->op.code = LOAD_ATTR_PROPERTY;
975 goto success;
976 }
977 case OBJECT_SLOT:
978 {
979 PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
980 struct PyMemberDef *dmem = member->d_member;
981 Py_ssize_t offset = dmem->offset;
982 if (!PyObject_TypeCheck(owner, member->d_common.d_type)) {
983 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
984 goto fail;
985 }
986 if (dmem->flags & Py_AUDIT_READ) {
987 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT);
988 goto fail;
989 }
990 if (offset != (uint16_t)offset) {
991 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
992 goto fail;
993 }
994 assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
995 assert(offset > 0);
996 cache->index = (uint16_t)offset;
997 write_u32(cache->version, type->tp_version_tag);
998 instr->op.code = LOAD_ATTR_SLOT;
999 goto success;
1000 }
1001 case DUNDER_CLASS:
1002 {
1003 Py_ssize_t offset = offsetof(PyObject, ob_type);
1004 assert(offset == (uint16_t)offset);
1005 cache->index = (uint16_t)offset;
1006 write_u32(cache->version, type->tp_version_tag);
1007 instr->op.code = LOAD_ATTR_SLOT;
1008 goto success;
1009 }
1010 case OTHER_SLOT:
1011 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
1012 goto fail;
1013 case MUTABLE:
1014 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
1015 goto fail;
1016 case GETSET_OVERRIDDEN:
1017 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);
1018 goto fail;
1019 case GETATTRIBUTE_IS_PYTHON_FUNCTION:
1020 {
1021 assert(type->tp_getattro == _Py_slot_tp_getattro);
1022 assert(Py_IS_TYPE(descr, &PyFunction_Type));
1023 _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1);
1024 if (!function_check_args(descr, 2, LOAD_ATTR)) {
1025 goto fail;
1026 }
1027 if (instr->op.arg & 1) {
1028 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
1029 goto fail;
1030 }
1031 uint32_t version = function_get_version(descr, LOAD_ATTR);
1032 if (version == 0) {
1033 goto fail;
1034 }
1035 if (_PyInterpreterState_GET()->eval_frame) {
1036 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
1037 goto fail;
1038 }
1039 write_u32(lm_cache->keys_version, version);
1040 /* borrowed */
1041 write_obj(lm_cache->descr, descr);
1042 write_u32(lm_cache->type_version, type->tp_version_tag);
1043 instr->op.code = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN;
1044 goto success;
1045 }
1046 case BUILTIN_CLASSMETHOD:
1047 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ);
1048 goto fail;
1049 case PYTHON_CLASSMETHOD:
1050 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ);
1051 goto fail;
1052 case NON_OVERRIDING:
1053 SPECIALIZATION_FAIL(LOAD_ATTR,
1054 (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) ?
1055 SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR :
1056 SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
1057 goto fail;
1058 case NON_DESCRIPTOR:
1059 if ((instr->op.arg & 1) == 0) {
1060 if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, false)) {
1061 goto success;
1062 }
1063 }
1064 else {
1065 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
1066 }
1067 goto fail;
1068 case ABSENT:
1069 if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR,
1070 LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
1071 {
1072 goto success;
1073 }
1074 }
1075 fail:
1076 STAT_INC(LOAD_ATTR, failure);
1077 assert(!PyErr_Occurred());
1078 instr->op.code = LOAD_ATTR;
1079 cache->counter = adaptive_counter_backoff(cache->counter);
1080 return;
1081 success:
1082 STAT_INC(LOAD_ATTR, success);
1083 assert(!PyErr_Occurred());
1084 cache->counter = adaptive_counter_cooldown();
1085 }
1086
1087 void
_Py_Specialize_StoreAttr(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name)1088 _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
1089 {
1090 assert(ENABLE_SPECIALIZATION);
1091 assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
1092 _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
1093 PyTypeObject *type = Py_TYPE(owner);
1094 if (!_PyType_IsReady(type)) {
1095 // We *might* not really need this check, but we inherited it from
1096 // PyObject_GenericSetAttr and friends... and this way we still do the
1097 // right thing if someone forgets to call PyType_Ready(type):
1098 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER);
1099 goto fail;
1100 }
1101 if (PyModule_CheckExact(owner)) {
1102 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
1103 goto fail;
1104 }
1105 PyObject *descr;
1106 DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1);
1107 if (type_get_version(type, STORE_ATTR) == 0) {
1108 goto fail;
1109 }
1110 switch(kind) {
1111 case OVERRIDING:
1112 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
1113 goto fail;
1114 case METHOD:
1115 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_METHOD);
1116 goto fail;
1117 case PROPERTY:
1118 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_PROPERTY);
1119 goto fail;
1120 case OBJECT_SLOT:
1121 {
1122 PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
1123 struct PyMemberDef *dmem = member->d_member;
1124 Py_ssize_t offset = dmem->offset;
1125 if (!PyObject_TypeCheck(owner, member->d_common.d_type)) {
1126 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_EXPECTED_ERROR);
1127 goto fail;
1128 }
1129 if (dmem->flags & Py_READONLY) {
1130 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_READ_ONLY);
1131 goto fail;
1132 }
1133 if (offset != (uint16_t)offset) {
1134 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OUT_OF_RANGE);
1135 goto fail;
1136 }
1137 assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
1138 assert(offset > 0);
1139 cache->index = (uint16_t)offset;
1140 write_u32(cache->version, type->tp_version_tag);
1141 instr->op.code = STORE_ATTR_SLOT;
1142 goto success;
1143 }
1144 case DUNDER_CLASS:
1145 case OTHER_SLOT:
1146 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
1147 goto fail;
1148 case MUTABLE:
1149 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
1150 goto fail;
1151 case GETATTRIBUTE_IS_PYTHON_FUNCTION:
1152 case GETSET_OVERRIDDEN:
1153 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
1154 goto fail;
1155 case BUILTIN_CLASSMETHOD:
1156 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ);
1157 goto fail;
1158 case PYTHON_CLASSMETHOD:
1159 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ);
1160 goto fail;
1161 case NON_OVERRIDING:
1162 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR);
1163 goto fail;
1164 case NON_DESCRIPTOR:
1165 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
1166 goto fail;
1167 case ABSENT:
1168 if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR,
1169 STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT))
1170 {
1171 goto success;
1172 }
1173 }
1174 fail:
1175 STAT_INC(STORE_ATTR, failure);
1176 assert(!PyErr_Occurred());
1177 instr->op.code = STORE_ATTR;
1178 cache->counter = adaptive_counter_backoff(cache->counter);
1179 return;
1180 success:
1181 STAT_INC(STORE_ATTR, success);
1182 assert(!PyErr_Occurred());
1183 cache->counter = adaptive_counter_cooldown();
1184 }
1185
1186
1187 #ifdef Py_STATS
1188 static int
load_attr_fail_kind(DescriptorClassification kind)1189 load_attr_fail_kind(DescriptorClassification kind)
1190 {
1191 switch (kind) {
1192 case OVERRIDING:
1193 return SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR;
1194 case METHOD:
1195 return SPEC_FAIL_ATTR_METHOD;
1196 case PROPERTY:
1197 return SPEC_FAIL_ATTR_PROPERTY;
1198 case OBJECT_SLOT:
1199 return SPEC_FAIL_ATTR_OBJECT_SLOT;
1200 case OTHER_SLOT:
1201 return SPEC_FAIL_ATTR_NON_OBJECT_SLOT;
1202 case DUNDER_CLASS:
1203 return SPEC_FAIL_OTHER;
1204 case MUTABLE:
1205 return SPEC_FAIL_ATTR_MUTABLE_CLASS;
1206 case GETSET_OVERRIDDEN:
1207 case GETATTRIBUTE_IS_PYTHON_FUNCTION:
1208 return SPEC_FAIL_OVERRIDDEN;
1209 case BUILTIN_CLASSMETHOD:
1210 return SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD;
1211 case PYTHON_CLASSMETHOD:
1212 return SPEC_FAIL_ATTR_CLASS_METHOD_OBJ;
1213 case NON_OVERRIDING:
1214 return SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR;
1215 case NON_DESCRIPTOR:
1216 return SPEC_FAIL_ATTR_NOT_DESCRIPTOR;
1217 case ABSENT:
1218 return SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE;
1219 }
1220 Py_UNREACHABLE();
1221 }
1222 #endif // Py_STATS
1223
1224 static int
specialize_class_load_attr(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name)1225 specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
1226 PyObject *name)
1227 {
1228 _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
1229 if (!PyType_CheckExact(owner) || _PyType_Lookup(Py_TYPE(owner), name)) {
1230 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE);
1231 return -1;
1232 }
1233 PyObject *descr = NULL;
1234 DescriptorClassification kind = 0;
1235 kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0);
1236 if (type_get_version((PyTypeObject *)owner, LOAD_ATTR) == 0) {
1237 return -1;
1238 }
1239 switch (kind) {
1240 case METHOD:
1241 case NON_DESCRIPTOR:
1242 write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag);
1243 write_obj(cache->descr, descr);
1244 instr->op.code = LOAD_ATTR_CLASS;
1245 return 0;
1246 #ifdef Py_STATS
1247 case ABSENT:
1248 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
1249 return -1;
1250 #endif
1251 default:
1252 SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind));
1253 return -1;
1254 }
1255 }
1256
1257 // Please collect stats carefully before and after modifying. A subtle change
1258 // can cause a significant drop in cache hits. A possible test is
1259 // python.exe -m test_typing test_re test_dis test_zlib.
1260 static int
specialize_attr_loadclassattr(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name,PyObject * descr,DescriptorClassification kind,bool is_method)1261 specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
1262 PyObject *descr, DescriptorClassification kind, bool is_method)
1263 {
1264 _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
1265 PyTypeObject *owner_cls = Py_TYPE(owner);
1266
1267 assert(descr != NULL);
1268 assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR));
1269 if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
1270 PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
1271 Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
1272 if (index != DKIX_EMPTY) {
1273 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED);
1274 return 0;
1275 }
1276 uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
1277 _PyInterpreterState_GET(), keys);
1278 if (keys_version == 0) {
1279 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
1280 return 0;
1281 }
1282 write_u32(cache->keys_version, keys_version);
1283 instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
1284 }
1285 else {
1286 Py_ssize_t dictoffset;
1287 if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
1288 dictoffset = MANAGED_DICT_OFFSET;
1289 }
1290 else {
1291 dictoffset = owner_cls->tp_dictoffset;
1292 if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) {
1293 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
1294 return 0;
1295 }
1296 }
1297 if (dictoffset == 0) {
1298 instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
1299 }
1300 else if (is_method) {
1301 PyObject *dict = *(PyObject **) ((char *)owner + dictoffset);
1302 if (dict) {
1303 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
1304 return 0;
1305 }
1306 /* Cache entries must be unsigned values, so we offset the
1307 * dictoffset by MANAGED_DICT_OFFSET.
1308 * We do the reverse offset in LOAD_ATTR_METHOD_LAZY_DICT */
1309 dictoffset -= MANAGED_DICT_OFFSET;
1310 assert(((uint16_t)dictoffset) == dictoffset);
1311 cache->dict_offset = (uint16_t)dictoffset;
1312 instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT;
1313 }
1314 else {
1315 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
1316 return 0;
1317 }
1318 }
1319 /* `descr` is borrowed. This is safe for methods (even inherited ones from
1320 * super classes!) as long as tp_version_tag is validated for two main reasons:
1321 *
1322 * 1. The class will always hold a reference to the method so it will
1323 * usually not be GC-ed. Should it be deleted in Python, e.g.
1324 * `del obj.meth`, tp_version_tag will be invalidated, because of reason 2.
1325 *
1326 * 2. The pre-existing type method cache (MCACHE) uses the same principles
1327 * of caching a borrowed descriptor. The MCACHE infrastructure does all the
1328 * heavy lifting for us. E.g. it invalidates tp_version_tag on any MRO
1329 * modification, on any type object change along said MRO, etc. (see
1330 * PyType_Modified usages in typeobject.c). The MCACHE has been
1331 * working since Python 2.6 and it's battle-tested.
1332 */
1333 write_u32(cache->type_version, owner_cls->tp_version_tag);
1334 write_obj(cache->descr, descr);
1335 return 1;
1336 }
1337
1338 void
_Py_Specialize_LoadGlobal(PyObject * globals,PyObject * builtins,_Py_CODEUNIT * instr,PyObject * name)1339 _Py_Specialize_LoadGlobal(
1340 PyObject *globals, PyObject *builtins,
1341 _Py_CODEUNIT *instr, PyObject *name)
1342 {
1343 assert(ENABLE_SPECIALIZATION);
1344 assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
1345 /* Use inline cache */
1346 _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1);
1347 assert(PyUnicode_CheckExact(name));
1348 if (!PyDict_CheckExact(globals)) {
1349 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT);
1350 goto fail;
1351 }
1352 PyDictKeysObject * globals_keys = ((PyDictObject *)globals)->ma_keys;
1353 if (!DK_IS_UNICODE(globals_keys)) {
1354 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1355 goto fail;
1356 }
1357 Py_ssize_t index = _PyDictKeys_StringLookup(globals_keys, name);
1358 if (index == DKIX_ERROR) {
1359 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR);
1360 goto fail;
1361 }
1362 PyInterpreterState *interp = _PyInterpreterState_GET();
1363 if (index != DKIX_EMPTY) {
1364 if (index != (uint16_t)index) {
1365 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
1366 goto fail;
1367 }
1368 uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
1369 interp, globals_keys);
1370 if (keys_version == 0) {
1371 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
1372 goto fail;
1373 }
1374 if (keys_version != (uint16_t)keys_version) {
1375 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
1376 goto fail;
1377 }
1378 cache->index = (uint16_t)index;
1379 cache->module_keys_version = (uint16_t)keys_version;
1380 instr->op.code = LOAD_GLOBAL_MODULE;
1381 goto success;
1382 }
1383 if (!PyDict_CheckExact(builtins)) {
1384 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT);
1385 goto fail;
1386 }
1387 PyDictKeysObject * builtin_keys = ((PyDictObject *)builtins)->ma_keys;
1388 if (!DK_IS_UNICODE(builtin_keys)) {
1389 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1390 goto fail;
1391 }
1392 index = _PyDictKeys_StringLookup(builtin_keys, name);
1393 if (index == DKIX_ERROR) {
1394 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR);
1395 goto fail;
1396 }
1397 if (index != (uint16_t)index) {
1398 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
1399 goto fail;
1400 }
1401 uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(
1402 interp, globals_keys);
1403 if (globals_version == 0) {
1404 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
1405 goto fail;
1406 }
1407 if (globals_version != (uint16_t)globals_version) {
1408 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
1409 goto fail;
1410 }
1411 uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(
1412 interp, builtin_keys);
1413 if (builtins_version == 0) {
1414 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
1415 goto fail;
1416 }
1417 if (builtins_version > UINT16_MAX) {
1418 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
1419 goto fail;
1420 }
1421 cache->index = (uint16_t)index;
1422 cache->module_keys_version = (uint16_t)globals_version;
1423 cache->builtin_keys_version = (uint16_t)builtins_version;
1424 instr->op.code = LOAD_GLOBAL_BUILTIN;
1425 goto success;
1426 fail:
1427 STAT_INC(LOAD_GLOBAL, failure);
1428 assert(!PyErr_Occurred());
1429 instr->op.code = LOAD_GLOBAL;
1430 cache->counter = adaptive_counter_backoff(cache->counter);
1431 return;
1432 success:
1433 STAT_INC(LOAD_GLOBAL, success);
1434 assert(!PyErr_Occurred());
1435 cache->counter = adaptive_counter_cooldown();
1436 }
1437
1438 #ifdef Py_STATS
1439 static int
binary_subscr_fail_kind(PyTypeObject * container_type,PyObject * sub)1440 binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub)
1441 {
1442 if (strcmp(container_type->tp_name, "array.array") == 0) {
1443 if (PyLong_CheckExact(sub)) {
1444 return SPEC_FAIL_SUBSCR_ARRAY_INT;
1445 }
1446 if (PySlice_Check(sub)) {
1447 return SPEC_FAIL_SUBSCR_ARRAY_SLICE;
1448 }
1449 return SPEC_FAIL_OTHER;
1450 }
1451 else if (container_type->tp_as_buffer) {
1452 if (PyLong_CheckExact(sub)) {
1453 return SPEC_FAIL_SUBSCR_BUFFER_INT;
1454 }
1455 if (PySlice_Check(sub)) {
1456 return SPEC_FAIL_SUBSCR_BUFFER_SLICE;
1457 }
1458 return SPEC_FAIL_OTHER;
1459 }
1460 else if (container_type->tp_as_sequence) {
1461 if (PyLong_CheckExact(sub) && container_type->tp_as_sequence->sq_item) {
1462 return SPEC_FAIL_SUBSCR_SEQUENCE_INT;
1463 }
1464 }
1465 return SPEC_FAIL_OTHER;
1466 }
1467 #endif // Py_STATS
1468
1469 static int
function_kind(PyCodeObject * code)1470 function_kind(PyCodeObject *code) {
1471 int flags = code->co_flags;
1472 if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount) {
1473 return SPEC_FAIL_CODE_COMPLEX_PARAMETERS;
1474 }
1475 if ((flags & CO_OPTIMIZED) == 0) {
1476 return SPEC_FAIL_CODE_NOT_OPTIMIZED;
1477 }
1478 return SIMPLE_FUNCTION;
1479 }
1480
1481 /* Returning false indicates a failure. */
1482 static bool
function_check_args(PyObject * o,int expected_argcount,int opcode)1483 function_check_args(PyObject *o, int expected_argcount, int opcode)
1484 {
1485 assert(Py_IS_TYPE(o, &PyFunction_Type));
1486 PyFunctionObject *func = (PyFunctionObject *)o;
1487 PyCodeObject *fcode = (PyCodeObject *)func->func_code;
1488 int kind = function_kind(fcode);
1489 if (kind != SIMPLE_FUNCTION) {
1490 SPECIALIZATION_FAIL(opcode, kind);
1491 return false;
1492 }
1493 if (fcode->co_argcount != expected_argcount) {
1494 SPECIALIZATION_FAIL(opcode, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1495 return false;
1496 }
1497 return true;
1498 }
1499
1500 /* Returning 0 indicates a failure. */
1501 static uint32_t
function_get_version(PyObject * o,int opcode)1502 function_get_version(PyObject *o, int opcode)
1503 {
1504 assert(Py_IS_TYPE(o, &PyFunction_Type));
1505 PyFunctionObject *func = (PyFunctionObject *)o;
1506 uint32_t version = _PyFunction_GetVersionForCurrentState(func);
1507 if (version == 0) {
1508 SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
1509 return 0;
1510 }
1511 return version;
1512 }
1513
1514 /* Returning 0 indicates a failure. */
1515 static uint32_t
type_get_version(PyTypeObject * t,int opcode)1516 type_get_version(PyTypeObject *t, int opcode)
1517 {
1518 uint32_t version = t->tp_version_tag;
1519 if (version == 0) {
1520 SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
1521 return 0;
1522 }
1523 return version;
1524 }
1525
1526 void
_Py_Specialize_BinarySubscr(PyObject * container,PyObject * sub,_Py_CODEUNIT * instr)1527 _Py_Specialize_BinarySubscr(
1528 PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
1529 {
1530 assert(ENABLE_SPECIALIZATION);
1531 assert(_PyOpcode_Caches[BINARY_SUBSCR] ==
1532 INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
1533 _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1);
1534 PyTypeObject *container_type = Py_TYPE(container);
1535 if (container_type == &PyList_Type) {
1536 if (PyLong_CheckExact(sub)) {
1537 if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
1538 instr->op.code = BINARY_SUBSCR_LIST_INT;
1539 goto success;
1540 }
1541 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1542 goto fail;
1543 }
1544 SPECIALIZATION_FAIL(BINARY_SUBSCR,
1545 PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_LIST_SLICE : SPEC_FAIL_OTHER);
1546 goto fail;
1547 }
1548 if (container_type == &PyTuple_Type) {
1549 if (PyLong_CheckExact(sub)) {
1550 if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
1551 instr->op.code = BINARY_SUBSCR_TUPLE_INT;
1552 goto success;
1553 }
1554 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1555 goto fail;
1556 }
1557 SPECIALIZATION_FAIL(BINARY_SUBSCR,
1558 PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER);
1559 goto fail;
1560 }
1561 if (container_type == &PyUnicode_Type) {
1562 if (PyLong_CheckExact(sub)) {
1563 if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
1564 instr->op.code = BINARY_SUBSCR_STR_INT;
1565 goto success;
1566 }
1567 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1568 goto fail;
1569 }
1570 SPECIALIZATION_FAIL(BINARY_SUBSCR,
1571 PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_STRING_SLICE : SPEC_FAIL_OTHER);
1572 goto fail;
1573 }
1574 if (container_type == &PyDict_Type) {
1575 instr->op.code = BINARY_SUBSCR_DICT;
1576 goto success;
1577 }
1578 PyTypeObject *cls = Py_TYPE(container);
1579 PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__));
1580 if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
1581 if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
1582 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE);
1583 goto fail;
1584 }
1585 PyFunctionObject *func = (PyFunctionObject *)descriptor;
1586 PyCodeObject *fcode = (PyCodeObject *)func->func_code;
1587 int kind = function_kind(fcode);
1588 if (kind != SIMPLE_FUNCTION) {
1589 SPECIALIZATION_FAIL(BINARY_SUBSCR, kind);
1590 goto fail;
1591 }
1592 if (fcode->co_argcount != 2) {
1593 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1594 goto fail;
1595 }
1596 uint32_t version = _PyFunction_GetVersionForCurrentState(func);
1597 if (version == 0) {
1598 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS);
1599 goto fail;
1600 }
1601 if (_PyInterpreterState_GET()->eval_frame) {
1602 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OTHER);
1603 goto fail;
1604 }
1605 PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type;
1606 // This pointer is invalidated by PyType_Modified (see the comment on
1607 // struct _specialization_cache):
1608 ht->_spec_cache.getitem = descriptor;
1609 ht->_spec_cache.getitem_version = version;
1610 instr->op.code = BINARY_SUBSCR_GETITEM;
1611 goto success;
1612 }
1613 SPECIALIZATION_FAIL(BINARY_SUBSCR,
1614 binary_subscr_fail_kind(container_type, sub));
1615 fail:
1616 STAT_INC(BINARY_SUBSCR, failure);
1617 assert(!PyErr_Occurred());
1618 instr->op.code = BINARY_SUBSCR;
1619 cache->counter = adaptive_counter_backoff(cache->counter);
1620 return;
1621 success:
1622 STAT_INC(BINARY_SUBSCR, success);
1623 assert(!PyErr_Occurred());
1624 cache->counter = adaptive_counter_cooldown();
1625 }
1626
1627 void
_Py_Specialize_StoreSubscr(PyObject * container,PyObject * sub,_Py_CODEUNIT * instr)1628 _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
1629 {
1630 assert(ENABLE_SPECIALIZATION);
1631 _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1);
1632 PyTypeObject *container_type = Py_TYPE(container);
1633 if (container_type == &PyList_Type) {
1634 if (PyLong_CheckExact(sub)) {
1635 if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)
1636 && ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container))
1637 {
1638 instr->op.code = STORE_SUBSCR_LIST_INT;
1639 goto success;
1640 }
1641 else {
1642 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1643 goto fail;
1644 }
1645 }
1646 else if (PySlice_Check(sub)) {
1647 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE);
1648 goto fail;
1649 }
1650 else {
1651 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1652 goto fail;
1653 }
1654 }
1655 if (container_type == &PyDict_Type) {
1656 instr->op.code = STORE_SUBSCR_DICT;
1657 goto success;
1658 }
1659 #ifdef Py_STATS
1660 PyMappingMethods *as_mapping = container_type->tp_as_mapping;
1661 if (as_mapping && (as_mapping->mp_ass_subscript
1662 == PyDict_Type.tp_as_mapping->mp_ass_subscript)) {
1663 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE);
1664 goto fail;
1665 }
1666 if (PyObject_CheckBuffer(container)) {
1667 if (PyLong_CheckExact(sub) && (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub))) {
1668 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1669 }
1670 else if (strcmp(container_type->tp_name, "array.array") == 0) {
1671 if (PyLong_CheckExact(sub)) {
1672 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_INT);
1673 }
1674 else if (PySlice_Check(sub)) {
1675 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_SLICE);
1676 }
1677 else {
1678 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1679 }
1680 }
1681 else if (PyByteArray_CheckExact(container)) {
1682 if (PyLong_CheckExact(sub)) {
1683 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_INT);
1684 }
1685 else if (PySlice_Check(sub)) {
1686 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE);
1687 }
1688 else {
1689 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1690 }
1691 }
1692 else {
1693 if (PyLong_CheckExact(sub)) {
1694 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_INT);
1695 }
1696 else if (PySlice_Check(sub)) {
1697 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_SLICE);
1698 }
1699 else {
1700 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1701 }
1702 }
1703 goto fail;
1704 }
1705 PyObject *descriptor = _PyType_Lookup(container_type, &_Py_ID(__setitem__));
1706 if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
1707 PyFunctionObject *func = (PyFunctionObject *)descriptor;
1708 PyCodeObject *code = (PyCodeObject *)func->func_code;
1709 int kind = function_kind(code);
1710 if (kind == SIMPLE_FUNCTION) {
1711 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_SIMPLE);
1712 }
1713 else {
1714 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_OTHER);
1715 }
1716 goto fail;
1717 }
1718 #endif // Py_STATS
1719 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1720 fail:
1721 STAT_INC(STORE_SUBSCR, failure);
1722 assert(!PyErr_Occurred());
1723 instr->op.code = STORE_SUBSCR;
1724 cache->counter = adaptive_counter_backoff(cache->counter);
1725 return;
1726 success:
1727 STAT_INC(STORE_SUBSCR, success);
1728 assert(!PyErr_Occurred());
1729 cache->counter = adaptive_counter_cooldown();
1730 }
1731
1732 /* Returns a borrowed reference.
1733 * The reference is only valid if guarded by a type version check.
1734 */
1735 static PyFunctionObject *
get_init_for_simple_managed_python_class(PyTypeObject * tp)1736 get_init_for_simple_managed_python_class(PyTypeObject *tp)
1737 {
1738 assert(tp->tp_new == PyBaseObject_Type.tp_new);
1739 if (tp->tp_alloc != PyType_GenericAlloc) {
1740 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN);
1741 return NULL;
1742 }
1743 if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) {
1744 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES);
1745 return NULL;
1746 }
1747 if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
1748 /* Is this possible? */
1749 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_EXPECTED_ERROR);
1750 return NULL;
1751 }
1752 PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__));
1753 if (init == NULL || !PyFunction_Check(init)) {
1754 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_PYTHON);
1755 return NULL;
1756 }
1757 int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init));
1758 if (kind != SIMPLE_FUNCTION) {
1759 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_SIMPLE);
1760 return NULL;
1761 }
1762 ((PyHeapTypeObject *)tp)->_spec_cache.init = init;
1763 return (PyFunctionObject *)init;
1764 }
1765
1766 static int
specialize_class_call(PyObject * callable,_Py_CODEUNIT * instr,int nargs)1767 specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs)
1768 {
1769 assert(PyType_Check(callable));
1770 PyTypeObject *tp = _PyType_CAST(callable);
1771 if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
1772 int oparg = instr->op.arg;
1773 if (nargs == 1 && oparg == 1) {
1774 if (tp == &PyUnicode_Type) {
1775 instr->op.code = CALL_STR_1;
1776 return 0;
1777 }
1778 else if (tp == &PyType_Type) {
1779 instr->op.code = CALL_TYPE_1;
1780 return 0;
1781 }
1782 else if (tp == &PyTuple_Type) {
1783 instr->op.code = CALL_TUPLE_1;
1784 return 0;
1785 }
1786 }
1787 if (tp->tp_vectorcall != NULL) {
1788 instr->op.code = CALL_BUILTIN_CLASS;
1789 return 0;
1790 }
1791 SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ?
1792 SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL);
1793 return -1;
1794 }
1795 if (Py_TYPE(tp) != &PyType_Type) {
1796 goto generic;
1797 }
1798 if (tp->tp_new == PyBaseObject_Type.tp_new) {
1799 PyFunctionObject *init = get_init_for_simple_managed_python_class(tp);
1800 if (type_get_version(tp, CALL) == 0) {
1801 return -1;
1802 }
1803 if (init != NULL) {
1804 if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) {
1805 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1806 return -1;
1807 }
1808 _PyCallCache *cache = (_PyCallCache *)(instr + 1);
1809 write_u32(cache->func_version, tp->tp_version_tag);
1810 _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT);
1811 return 0;
1812 }
1813 }
1814 generic:
1815 instr->op.code = CALL_NON_PY_GENERAL;
1816 return 0;
1817 }
1818
1819 static int
specialize_method_descriptor(PyMethodDescrObject * descr,_Py_CODEUNIT * instr,int nargs)1820 specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
1821 int nargs)
1822 {
1823 switch (descr->d_method->ml_flags &
1824 (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
1825 METH_KEYWORDS | METH_METHOD)) {
1826 case METH_NOARGS: {
1827 if (nargs != 1) {
1828 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1829 return -1;
1830 }
1831 instr->op.code = CALL_METHOD_DESCRIPTOR_NOARGS;
1832 return 0;
1833 }
1834 case METH_O: {
1835 if (nargs != 2) {
1836 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1837 return -1;
1838 }
1839 PyInterpreterState *interp = _PyInterpreterState_GET();
1840 PyObject *list_append = interp->callable_cache.list_append;
1841 _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_CALL + 1];
1842 bool pop = (next.op.code == POP_TOP);
1843 int oparg = instr->op.arg;
1844 if ((PyObject *)descr == list_append && oparg == 1 && pop) {
1845 instr->op.code = CALL_LIST_APPEND;
1846 return 0;
1847 }
1848 instr->op.code = CALL_METHOD_DESCRIPTOR_O;
1849 return 0;
1850 }
1851 case METH_FASTCALL: {
1852 instr->op.code = CALL_METHOD_DESCRIPTOR_FAST;
1853 return 0;
1854 }
1855 case METH_FASTCALL | METH_KEYWORDS: {
1856 instr->op.code = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS;
1857 return 0;
1858 }
1859 }
1860 instr->op.code = CALL_NON_PY_GENERAL;
1861 return 0;
1862 }
1863
1864 static int
specialize_py_call(PyFunctionObject * func,_Py_CODEUNIT * instr,int nargs,bool bound_method)1865 specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs,
1866 bool bound_method)
1867 {
1868 _PyCallCache *cache = (_PyCallCache *)(instr + 1);
1869 PyCodeObject *code = (PyCodeObject *)func->func_code;
1870 int kind = function_kind(code);
1871 /* Don't specialize if PEP 523 is active */
1872 if (_PyInterpreterState_GET()->eval_frame) {
1873 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523);
1874 return -1;
1875 }
1876 int argcount = -1;
1877 if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) {
1878 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED);
1879 return -1;
1880 }
1881 if (kind == SIMPLE_FUNCTION) {
1882 argcount = code->co_argcount;
1883 }
1884 int version = _PyFunction_GetVersionForCurrentState(func);
1885 if (version == 0) {
1886 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
1887 return -1;
1888 }
1889 write_u32(cache->func_version, version);
1890 if (argcount == nargs + bound_method) {
1891 instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS;
1892 }
1893 else {
1894 instr->op.code = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL;
1895 }
1896 return 0;
1897 }
1898
1899 static int
specialize_c_call(PyObject * callable,_Py_CODEUNIT * instr,int nargs)1900 specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs)
1901 {
1902 if (PyCFunction_GET_FUNCTION(callable) == NULL) {
1903 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OTHER);
1904 return 1;
1905 }
1906 switch (PyCFunction_GET_FLAGS(callable) &
1907 (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
1908 METH_KEYWORDS | METH_METHOD)) {
1909 case METH_O: {
1910 if (nargs != 1) {
1911 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1912 return 1;
1913 }
1914 /* len(o) */
1915 PyInterpreterState *interp = _PyInterpreterState_GET();
1916 if (callable == interp->callable_cache.len) {
1917 instr->op.code = CALL_LEN;
1918 return 0;
1919 }
1920 instr->op.code = CALL_BUILTIN_O;
1921 return 0;
1922 }
1923 case METH_FASTCALL: {
1924 if (nargs == 2) {
1925 /* isinstance(o1, o2) */
1926 PyInterpreterState *interp = _PyInterpreterState_GET();
1927 if (callable == interp->callable_cache.isinstance) {
1928 instr->op.code = CALL_ISINSTANCE;
1929 return 0;
1930 }
1931 }
1932 instr->op.code = CALL_BUILTIN_FAST;
1933 return 0;
1934 }
1935 case METH_FASTCALL | METH_KEYWORDS: {
1936 instr->op.code = CALL_BUILTIN_FAST_WITH_KEYWORDS;
1937 return 0;
1938 }
1939 default:
1940 instr->op.code = CALL_NON_PY_GENERAL;
1941 return 0;
1942 }
1943 }
1944
1945 void
_Py_Specialize_Call(PyObject * callable,_Py_CODEUNIT * instr,int nargs)1946 _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs)
1947 {
1948 assert(ENABLE_SPECIALIZATION);
1949 assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL);
1950 assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL);
1951 _PyCallCache *cache = (_PyCallCache *)(instr + 1);
1952 int fail;
1953 if (PyCFunction_CheckExact(callable)) {
1954 fail = specialize_c_call(callable, instr, nargs);
1955 }
1956 else if (PyFunction_Check(callable)) {
1957 fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, false);
1958 }
1959 else if (PyType_Check(callable)) {
1960 fail = specialize_class_call(callable, instr, nargs);
1961 }
1962 else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
1963 fail = specialize_method_descriptor((PyMethodDescrObject *)callable, instr, nargs);
1964 }
1965 else if (PyMethod_Check(callable)) {
1966 PyObject *func = ((PyMethodObject *)callable)->im_func;
1967 if (PyFunction_Check(func)) {
1968 fail = specialize_py_call((PyFunctionObject *)func, instr, nargs, true);
1969 }
1970 else {
1971 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD);
1972 fail = -1;
1973 }
1974 }
1975 else {
1976 instr->op.code = CALL_NON_PY_GENERAL;
1977 fail = 0;
1978 }
1979 if (fail) {
1980 STAT_INC(CALL, failure);
1981 assert(!PyErr_Occurred());
1982 instr->op.code = CALL;
1983 cache->counter = adaptive_counter_backoff(cache->counter);
1984 }
1985 else {
1986 STAT_INC(CALL, success);
1987 assert(!PyErr_Occurred());
1988 cache->counter = adaptive_counter_cooldown();
1989 }
1990 }
1991
1992 #ifdef Py_STATS
1993 static int
binary_op_fail_kind(int oparg,PyObject * lhs,PyObject * rhs)1994 binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
1995 {
1996 switch (oparg) {
1997 case NB_ADD:
1998 case NB_INPLACE_ADD:
1999 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
2000 return SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES;
2001 }
2002 return SPEC_FAIL_BINARY_OP_ADD_OTHER;
2003 case NB_AND:
2004 case NB_INPLACE_AND:
2005 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
2006 return SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES;
2007 }
2008 if (PyLong_CheckExact(lhs)) {
2009 return SPEC_FAIL_BINARY_OP_AND_INT;
2010 }
2011 return SPEC_FAIL_BINARY_OP_AND_OTHER;
2012 case NB_FLOOR_DIVIDE:
2013 case NB_INPLACE_FLOOR_DIVIDE:
2014 return SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE;
2015 case NB_LSHIFT:
2016 case NB_INPLACE_LSHIFT:
2017 return SPEC_FAIL_BINARY_OP_LSHIFT;
2018 case NB_MATRIX_MULTIPLY:
2019 case NB_INPLACE_MATRIX_MULTIPLY:
2020 return SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY;
2021 case NB_MULTIPLY:
2022 case NB_INPLACE_MULTIPLY:
2023 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
2024 return SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES;
2025 }
2026 return SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER;
2027 case NB_OR:
2028 case NB_INPLACE_OR:
2029 return SPEC_FAIL_BINARY_OP_OR;
2030 case NB_POWER:
2031 case NB_INPLACE_POWER:
2032 return SPEC_FAIL_BINARY_OP_POWER;
2033 case NB_REMAINDER:
2034 case NB_INPLACE_REMAINDER:
2035 return SPEC_FAIL_BINARY_OP_REMAINDER;
2036 case NB_RSHIFT:
2037 case NB_INPLACE_RSHIFT:
2038 return SPEC_FAIL_BINARY_OP_RSHIFT;
2039 case NB_SUBTRACT:
2040 case NB_INPLACE_SUBTRACT:
2041 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
2042 return SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES;
2043 }
2044 return SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER;
2045 case NB_TRUE_DIVIDE:
2046 case NB_INPLACE_TRUE_DIVIDE:
2047 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
2048 return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES;
2049 }
2050 if (PyFloat_CheckExact(lhs)) {
2051 return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT;
2052 }
2053 return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER;
2054 case NB_XOR:
2055 case NB_INPLACE_XOR:
2056 return SPEC_FAIL_BINARY_OP_XOR;
2057 }
2058 Py_UNREACHABLE();
2059 }
2060 #endif // Py_STATS
2061
2062 void
_Py_Specialize_BinaryOp(PyObject * lhs,PyObject * rhs,_Py_CODEUNIT * instr,int oparg,PyObject ** locals)2063 _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
2064 int oparg, PyObject **locals)
2065 {
2066 assert(ENABLE_SPECIALIZATION);
2067 assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);
2068 _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
2069 switch (oparg) {
2070 case NB_ADD:
2071 case NB_INPLACE_ADD:
2072 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
2073 break;
2074 }
2075 if (PyUnicode_CheckExact(lhs)) {
2076 _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1];
2077 bool to_store = (next.op.code == STORE_FAST);
2078 if (to_store && locals[next.op.arg] == lhs) {
2079 instr->op.code = BINARY_OP_INPLACE_ADD_UNICODE;
2080 goto success;
2081 }
2082 instr->op.code = BINARY_OP_ADD_UNICODE;
2083 goto success;
2084 }
2085 if (PyLong_CheckExact(lhs)) {
2086 instr->op.code = BINARY_OP_ADD_INT;
2087 goto success;
2088 }
2089 if (PyFloat_CheckExact(lhs)) {
2090 instr->op.code = BINARY_OP_ADD_FLOAT;
2091 goto success;
2092 }
2093 break;
2094 case NB_MULTIPLY:
2095 case NB_INPLACE_MULTIPLY:
2096 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
2097 break;
2098 }
2099 if (PyLong_CheckExact(lhs)) {
2100 instr->op.code = BINARY_OP_MULTIPLY_INT;
2101 goto success;
2102 }
2103 if (PyFloat_CheckExact(lhs)) {
2104 instr->op.code = BINARY_OP_MULTIPLY_FLOAT;
2105 goto success;
2106 }
2107 break;
2108 case NB_SUBTRACT:
2109 case NB_INPLACE_SUBTRACT:
2110 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
2111 break;
2112 }
2113 if (PyLong_CheckExact(lhs)) {
2114 instr->op.code = BINARY_OP_SUBTRACT_INT;
2115 goto success;
2116 }
2117 if (PyFloat_CheckExact(lhs)) {
2118 instr->op.code = BINARY_OP_SUBTRACT_FLOAT;
2119 goto success;
2120 }
2121 break;
2122 }
2123 SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs));
2124 STAT_INC(BINARY_OP, failure);
2125 instr->op.code = BINARY_OP;
2126 cache->counter = adaptive_counter_backoff(cache->counter);
2127 return;
2128 success:
2129 STAT_INC(BINARY_OP, success);
2130 cache->counter = adaptive_counter_cooldown();
2131 }
2132
2133
2134 #ifdef Py_STATS
2135 static int
compare_op_fail_kind(PyObject * lhs,PyObject * rhs)2136 compare_op_fail_kind(PyObject *lhs, PyObject *rhs)
2137 {
2138 if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
2139 if (PyFloat_CheckExact(lhs) && PyLong_CheckExact(rhs)) {
2140 return SPEC_FAIL_COMPARE_OP_FLOAT_LONG;
2141 }
2142 if (PyLong_CheckExact(lhs) && PyFloat_CheckExact(rhs)) {
2143 return SPEC_FAIL_COMPARE_OP_LONG_FLOAT;
2144 }
2145 return SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES;
2146 }
2147 if (PyBytes_CheckExact(lhs)) {
2148 return SPEC_FAIL_COMPARE_OP_BYTES;
2149 }
2150 if (PyTuple_CheckExact(lhs)) {
2151 return SPEC_FAIL_COMPARE_OP_TUPLE;
2152 }
2153 if (PyList_CheckExact(lhs)) {
2154 return SPEC_FAIL_COMPARE_OP_LIST;
2155 }
2156 if (PySet_CheckExact(lhs) || PyFrozenSet_CheckExact(lhs)) {
2157 return SPEC_FAIL_COMPARE_OP_SET;
2158 }
2159 if (PyBool_Check(lhs)) {
2160 return SPEC_FAIL_COMPARE_OP_BOOL;
2161 }
2162 if (Py_TYPE(lhs)->tp_richcompare == PyBaseObject_Type.tp_richcompare) {
2163 return SPEC_FAIL_COMPARE_OP_BASEOBJECT;
2164 }
2165 return SPEC_FAIL_OTHER;
2166 }
2167 #endif // Py_STATS
2168
2169 void
_Py_Specialize_CompareOp(PyObject * lhs,PyObject * rhs,_Py_CODEUNIT * instr,int oparg)2170 _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
2171 int oparg)
2172 {
2173 assert(ENABLE_SPECIALIZATION);
2174 assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
2175 // All of these specializations compute boolean values, so they're all valid
2176 // regardless of the fifth-lowest oparg bit.
2177 _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
2178 if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
2179 SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
2180 goto failure;
2181 }
2182 if (PyFloat_CheckExact(lhs)) {
2183 instr->op.code = COMPARE_OP_FLOAT;
2184 goto success;
2185 }
2186 if (PyLong_CheckExact(lhs)) {
2187 if (_PyLong_IsCompact((PyLongObject *)lhs) && _PyLong_IsCompact((PyLongObject *)rhs)) {
2188 instr->op.code = COMPARE_OP_INT;
2189 goto success;
2190 }
2191 else {
2192 SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_BIG_INT);
2193 goto failure;
2194 }
2195 }
2196 if (PyUnicode_CheckExact(lhs)) {
2197 int cmp = oparg >> 5;
2198 if (cmp != Py_EQ && cmp != Py_NE) {
2199 SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_STRING);
2200 goto failure;
2201 }
2202 else {
2203 instr->op.code = COMPARE_OP_STR;
2204 goto success;
2205 }
2206 }
2207 SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
2208 failure:
2209 STAT_INC(COMPARE_OP, failure);
2210 instr->op.code = COMPARE_OP;
2211 cache->counter = adaptive_counter_backoff(cache->counter);
2212 return;
2213 success:
2214 STAT_INC(COMPARE_OP, success);
2215 cache->counter = adaptive_counter_cooldown();
2216 }
2217
2218 #ifdef Py_STATS
2219 static int
unpack_sequence_fail_kind(PyObject * seq)2220 unpack_sequence_fail_kind(PyObject *seq)
2221 {
2222 if (PySequence_Check(seq)) {
2223 return SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE;
2224 }
2225 if (PyIter_Check(seq)) {
2226 return SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR;
2227 }
2228 return SPEC_FAIL_OTHER;
2229 }
2230 #endif // Py_STATS
2231
2232 void
_Py_Specialize_UnpackSequence(PyObject * seq,_Py_CODEUNIT * instr,int oparg)2233 _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
2234 {
2235 assert(ENABLE_SPECIALIZATION);
2236 assert(_PyOpcode_Caches[UNPACK_SEQUENCE] ==
2237 INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
2238 _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1);
2239 if (PyTuple_CheckExact(seq)) {
2240 if (PyTuple_GET_SIZE(seq) != oparg) {
2241 SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR);
2242 goto failure;
2243 }
2244 if (PyTuple_GET_SIZE(seq) == 2) {
2245 instr->op.code = UNPACK_SEQUENCE_TWO_TUPLE;
2246 goto success;
2247 }
2248 instr->op.code = UNPACK_SEQUENCE_TUPLE;
2249 goto success;
2250 }
2251 if (PyList_CheckExact(seq)) {
2252 if (PyList_GET_SIZE(seq) != oparg) {
2253 SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR);
2254 goto failure;
2255 }
2256 instr->op.code = UNPACK_SEQUENCE_LIST;
2257 goto success;
2258 }
2259 SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq));
2260 failure:
2261 STAT_INC(UNPACK_SEQUENCE, failure);
2262 instr->op.code = UNPACK_SEQUENCE;
2263 cache->counter = adaptive_counter_backoff(cache->counter);
2264 return;
2265 success:
2266 STAT_INC(UNPACK_SEQUENCE, success);
2267 cache->counter = adaptive_counter_cooldown();
2268 }
2269
2270 #ifdef Py_STATS
2271 int
_PySpecialization_ClassifyIterator(PyObject * iter)2272 _PySpecialization_ClassifyIterator(PyObject *iter)
2273 {
2274 if (PyGen_CheckExact(iter)) {
2275 return SPEC_FAIL_ITER_GENERATOR;
2276 }
2277 if (PyCoro_CheckExact(iter)) {
2278 return SPEC_FAIL_ITER_COROUTINE;
2279 }
2280 if (PyAsyncGen_CheckExact(iter)) {
2281 return SPEC_FAIL_ITER_ASYNC_GENERATOR;
2282 }
2283 if (PyAsyncGenASend_CheckExact(iter)) {
2284 return SPEC_FAIL_ITER_ASYNC_GENERATOR_SEND;
2285 }
2286 PyTypeObject *t = Py_TYPE(iter);
2287 if (t == &PyListIter_Type) {
2288 return SPEC_FAIL_ITER_LIST;
2289 }
2290 if (t == &PyTupleIter_Type) {
2291 return SPEC_FAIL_ITER_TUPLE;
2292 }
2293 if (t == &PyDictIterKey_Type) {
2294 return SPEC_FAIL_ITER_DICT_KEYS;
2295 }
2296 if (t == &PyDictIterValue_Type) {
2297 return SPEC_FAIL_ITER_DICT_VALUES;
2298 }
2299 if (t == &PyDictIterItem_Type) {
2300 return SPEC_FAIL_ITER_DICT_ITEMS;
2301 }
2302 if (t == &PySetIter_Type) {
2303 return SPEC_FAIL_ITER_SET;
2304 }
2305 if (t == &PyUnicodeIter_Type) {
2306 return SPEC_FAIL_ITER_STRING;
2307 }
2308 if (t == &PyBytesIter_Type) {
2309 return SPEC_FAIL_ITER_BYTES;
2310 }
2311 if (t == &PyRangeIter_Type) {
2312 return SPEC_FAIL_ITER_RANGE;
2313 }
2314 if (t == &PyEnum_Type) {
2315 return SPEC_FAIL_ITER_ENUMERATE;
2316 }
2317 if (t == &PyMap_Type) {
2318 return SPEC_FAIL_ITER_MAP;
2319 }
2320 if (t == &PyZip_Type) {
2321 return SPEC_FAIL_ITER_ZIP;
2322 }
2323 if (t == &PySeqIter_Type) {
2324 return SPEC_FAIL_ITER_SEQ_ITER;
2325 }
2326 if (t == &PyListRevIter_Type) {
2327 return SPEC_FAIL_ITER_REVERSED_LIST;
2328 }
2329 if (t == &_PyUnicodeASCIIIter_Type) {
2330 return SPEC_FAIL_ITER_ASCII_STRING;
2331 }
2332 const char *name = t->tp_name;
2333 if (strncmp(name, "itertools", 9) == 0) {
2334 return SPEC_FAIL_ITER_ITERTOOLS;
2335 }
2336 if (strncmp(name, "callable_iterator", 17) == 0) {
2337 return SPEC_FAIL_ITER_CALLABLE;
2338 }
2339 return SPEC_FAIL_OTHER;
2340 }
2341 #endif // Py_STATS
2342
2343 void
_Py_Specialize_ForIter(PyObject * iter,_Py_CODEUNIT * instr,int oparg)2344 _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg)
2345 {
2346 assert(ENABLE_SPECIALIZATION);
2347 assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER);
2348 _PyForIterCache *cache = (_PyForIterCache *)(instr + 1);
2349 PyTypeObject *tp = Py_TYPE(iter);
2350 if (tp == &PyListIter_Type) {
2351 instr->op.code = FOR_ITER_LIST;
2352 goto success;
2353 }
2354 else if (tp == &PyTupleIter_Type) {
2355 instr->op.code = FOR_ITER_TUPLE;
2356 goto success;
2357 }
2358 else if (tp == &PyRangeIter_Type) {
2359 instr->op.code = FOR_ITER_RANGE;
2360 goto success;
2361 }
2362 else if (tp == &PyGen_Type && oparg <= SHRT_MAX) {
2363 assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR ||
2364 instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR
2365 );
2366 if (_PyInterpreterState_GET()->eval_frame) {
2367 SPECIALIZATION_FAIL(FOR_ITER, SPEC_FAIL_OTHER);
2368 goto failure;
2369 }
2370 instr->op.code = FOR_ITER_GEN;
2371 goto success;
2372 }
2373 SPECIALIZATION_FAIL(FOR_ITER,
2374 _PySpecialization_ClassifyIterator(iter));
2375 failure:
2376 STAT_INC(FOR_ITER, failure);
2377 instr->op.code = FOR_ITER;
2378 cache->counter = adaptive_counter_backoff(cache->counter);
2379 return;
2380 success:
2381 STAT_INC(FOR_ITER, success);
2382 cache->counter = adaptive_counter_cooldown();
2383 }
2384
2385 void
_Py_Specialize_Send(PyObject * receiver,_Py_CODEUNIT * instr)2386 _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr)
2387 {
2388 assert(ENABLE_SPECIALIZATION);
2389 assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND);
2390 _PySendCache *cache = (_PySendCache *)(instr + 1);
2391 PyTypeObject *tp = Py_TYPE(receiver);
2392 if (tp == &PyGen_Type || tp == &PyCoro_Type) {
2393 if (_PyInterpreterState_GET()->eval_frame) {
2394 SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER);
2395 goto failure;
2396 }
2397 instr->op.code = SEND_GEN;
2398 goto success;
2399 }
2400 SPECIALIZATION_FAIL(SEND,
2401 _PySpecialization_ClassifyIterator(receiver));
2402 failure:
2403 STAT_INC(SEND, failure);
2404 instr->op.code = SEND;
2405 cache->counter = adaptive_counter_backoff(cache->counter);
2406 return;
2407 success:
2408 STAT_INC(SEND, success);
2409 cache->counter = adaptive_counter_cooldown();
2410 }
2411
2412 void
_Py_Specialize_ToBool(PyObject * value,_Py_CODEUNIT * instr)2413 _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr)
2414 {
2415 assert(ENABLE_SPECIALIZATION);
2416 assert(_PyOpcode_Caches[TO_BOOL] == INLINE_CACHE_ENTRIES_TO_BOOL);
2417 _PyToBoolCache *cache = (_PyToBoolCache *)(instr + 1);
2418 if (PyBool_Check(value)) {
2419 instr->op.code = TO_BOOL_BOOL;
2420 goto success;
2421 }
2422 if (PyLong_CheckExact(value)) {
2423 instr->op.code = TO_BOOL_INT;
2424 goto success;
2425 }
2426 if (PyList_CheckExact(value)) {
2427 instr->op.code = TO_BOOL_LIST;
2428 goto success;
2429 }
2430 if (Py_IsNone(value)) {
2431 instr->op.code = TO_BOOL_NONE;
2432 goto success;
2433 }
2434 if (PyUnicode_CheckExact(value)) {
2435 instr->op.code = TO_BOOL_STR;
2436 goto success;
2437 }
2438 if (PyType_HasFeature(Py_TYPE(value), Py_TPFLAGS_HEAPTYPE)) {
2439 PyNumberMethods *nb = Py_TYPE(value)->tp_as_number;
2440 if (nb && nb->nb_bool) {
2441 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_NUMBER);
2442 goto failure;
2443 }
2444 PyMappingMethods *mp = Py_TYPE(value)->tp_as_mapping;
2445 if (mp && mp->mp_length) {
2446 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_MAPPING);
2447 goto failure;
2448 }
2449 PySequenceMethods *sq = Py_TYPE(value)->tp_as_sequence;
2450 if (sq && sq->sq_length) {
2451 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_SEQUENCE);
2452 goto failure;
2453 }
2454 if (!PyUnstable_Type_AssignVersionTag(Py_TYPE(value))) {
2455 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OUT_OF_VERSIONS);
2456 goto failure;
2457 }
2458 uint32_t version = type_get_version(Py_TYPE(value), TO_BOOL);
2459 if (version == 0) {
2460 goto failure;
2461 }
2462 instr->op.code = TO_BOOL_ALWAYS_TRUE;
2463 write_u32(cache->version, version);
2464 assert(version);
2465 goto success;
2466 }
2467 #ifdef Py_STATS
2468 if (PyByteArray_CheckExact(value)) {
2469 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_BYTEARRAY);
2470 goto failure;
2471 }
2472 if (PyBytes_CheckExact(value)) {
2473 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_BYTES);
2474 goto failure;
2475 }
2476 if (PyDict_CheckExact(value)) {
2477 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_DICT);
2478 goto failure;
2479 }
2480 if (PyFloat_CheckExact(value)) {
2481 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_FLOAT);
2482 goto failure;
2483 }
2484 if (PyMemoryView_Check(value)) {
2485 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_MEMORY_VIEW);
2486 goto failure;
2487 }
2488 if (PyAnySet_CheckExact(value)) {
2489 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_SET);
2490 goto failure;
2491 }
2492 if (PyTuple_CheckExact(value)) {
2493 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_TUPLE);
2494 goto failure;
2495 }
2496 SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OTHER);
2497 #endif // Py_STATS
2498 failure:
2499 STAT_INC(TO_BOOL, failure);
2500 instr->op.code = TO_BOOL;
2501 cache->counter = adaptive_counter_backoff(cache->counter);
2502 return;
2503 success:
2504 STAT_INC(TO_BOOL, success);
2505 cache->counter = adaptive_counter_cooldown();
2506 }
2507
2508 #ifdef Py_STATS
containsop_fail_kind(PyObject * value)2509 static int containsop_fail_kind(PyObject *value) {
2510 if (PyUnicode_CheckExact(value)) {
2511 return SPEC_FAIL_CONTAINS_OP_STR;
2512 }
2513 if (PyList_CheckExact(value)) {
2514 return SPEC_FAIL_CONTAINS_OP_LIST;
2515 }
2516 if (PyTuple_CheckExact(value)) {
2517 return SPEC_FAIL_CONTAINS_OP_TUPLE;
2518 }
2519 if (PyType_Check(value)) {
2520 return SPEC_FAIL_CONTAINS_OP_USER_CLASS;
2521 }
2522 return SPEC_FAIL_OTHER;
2523 }
2524 #endif // Py_STATS
2525
2526 void
_Py_Specialize_ContainsOp(PyObject * value,_Py_CODEUNIT * instr)2527 _Py_Specialize_ContainsOp(PyObject *value, _Py_CODEUNIT *instr)
2528 {
2529 assert(ENABLE_SPECIALIZATION);
2530 assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
2531 _PyContainsOpCache *cache = (_PyContainsOpCache *)(instr + 1);
2532 if (PyDict_CheckExact(value)) {
2533 instr->op.code = CONTAINS_OP_DICT;
2534 goto success;
2535 }
2536 if (PySet_CheckExact(value) || PyFrozenSet_CheckExact(value)) {
2537 instr->op.code = CONTAINS_OP_SET;
2538 goto success;
2539 }
2540
2541 SPECIALIZATION_FAIL(CONTAINS_OP, containsop_fail_kind(value));
2542 STAT_INC(CONTAINS_OP, failure);
2543 instr->op.code = CONTAINS_OP;
2544 cache->counter = adaptive_counter_backoff(cache->counter);
2545 return;
2546 success:
2547 STAT_INC(CONTAINS_OP, success);
2548 cache->counter = adaptive_counter_cooldown();
2549 }
2550
2551 /* Code init cleanup.
2552 * CALL_ALLOC_AND_ENTER_INIT will set up
2553 * the frame to execute the EXIT_INIT_CHECK
2554 * instruction.
2555 * Ends with a RESUME so that it is not traced.
2556 * This is used as a plain code object, not a function,
2557 * so must not access globals or builtins.
2558 */
2559
2560 #define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3)
2561
2562 static const PyBytesObject no_location = {
2563 PyVarObject_HEAD_INIT(&PyBytes_Type, 1)
2564 .ob_sval = { NO_LOC_4 }
2565 };
2566
2567 const struct _PyCode_DEF(8) _Py_InitCleanup = {
2568 _PyVarObject_HEAD_INIT(&PyCode_Type, 3),
2569 .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty),
2570 .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty),
2571 .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty),
2572 .co_flags = CO_OPTIMIZED | CO_NO_MONITORING_EVENTS,
2573 .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty),
2574 .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty),
2575 .co_filename = &_Py_ID(__init__),
2576 .co_name = &_Py_ID(__init__),
2577 .co_qualname = &_Py_ID(__init__),
2578 .co_linetable = (PyObject *)&no_location,
2579 ._co_firsttraceable = 4,
2580 .co_stacksize = 2,
2581 .co_framesize = 2 + FRAME_SPECIALS_SIZE,
2582 .co_code_adaptive = {
2583 EXIT_INIT_CHECK, 0,
2584 RETURN_VALUE, 0,
2585 RESUME, RESUME_AT_FUNC_START,
2586 }
2587 };
2588