1 #include "Python.h"
2
3 #include "pycore_backoff.h"
4 #include "pycore_call.h"
5 #include "pycore_ceval.h"
6 #include "pycore_cell.h"
7 #include "pycore_dict.h"
8 #include "pycore_emscripten_signal.h"
9 #include "pycore_intrinsics.h"
10 #include "pycore_jit.h"
11 #include "pycore_long.h"
12 #include "pycore_opcode_metadata.h"
13 #include "pycore_opcode_utils.h"
14 #include "pycore_optimizer.h"
15 #include "pycore_pyatomic_ft_wrappers.h"
16 #include "pycore_range.h"
17 #include "pycore_setobject.h"
18 #include "pycore_sliceobject.h"
19 #include "pycore_descrobject.h"
20
21 #include "ceval_macros.h"
22
23 #undef CURRENT_OPARG
24 #define CURRENT_OPARG() (_oparg)
25
26 #undef CURRENT_OPERAND
27 #define CURRENT_OPERAND() (_operand)
28
29 #undef DEOPT_IF
30 #define DEOPT_IF(COND, INSTNAME) \
31 do { \
32 if ((COND)) { \
33 goto deoptimize; \
34 } \
35 } while (0)
36
37 #undef ENABLE_SPECIALIZATION
38 #define ENABLE_SPECIALIZATION (0)
39
40 #undef GOTO_ERROR
41 #define GOTO_ERROR(LABEL) \
42 do { \
43 goto LABEL ## _tier_two; \
44 } while (0)
45
46 #undef GOTO_TIER_TWO
47 #define GOTO_TIER_TWO(EXECUTOR) \
48 do { \
49 OPT_STAT_INC(traces_executed); \
50 __attribute__((musttail)) \
51 return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \
52 } while (0)
53
54 #undef GOTO_TIER_ONE
55 #define GOTO_TIER_ONE(TARGET) \
56 do { \
57 _PyFrame_SetStackPointer(frame, stack_pointer); \
58 return TARGET; \
59 } while (0)
60
61 #undef LOAD_IP
62 #define LOAD_IP(UNUSED) \
63 do { \
64 } while (0)
65
66 #define PATCH_VALUE(TYPE, NAME, ALIAS) \
67 PyAPI_DATA(void) ALIAS; \
68 TYPE NAME = (TYPE)(uintptr_t)&ALIAS;
69
70 #define PATCH_JUMP(ALIAS) \
71 do { \
72 PyAPI_DATA(void) ALIAS; \
73 __attribute__((musttail)) \
74 return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \
75 } while (0)
76
77 #undef JUMP_TO_JUMP_TARGET
78 #define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET)
79
80 #undef JUMP_TO_ERROR
81 #define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET)
82
83 _Py_CODEUNIT *
_JIT_ENTRY(_PyInterpreterFrame * frame,PyObject ** stack_pointer,PyThreadState * tstate)84 _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate)
85 {
86 // Locals that the instruction implementations expect to exist:
87 PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR)
88 int oparg;
89 int uopcode = _JIT_OPCODE;
90 _Py_CODEUNIT *next_instr;
91 // Other stuff we need handy:
92 PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
93 #if SIZEOF_VOID_P == 8
94 PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND)
95 #else
96 assert(SIZEOF_VOID_P == 4);
97 PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI)
98 PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO)
99 uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo;
100 #endif
101 PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
102 PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX)
103
104 OPT_STAT_INC(uops_executed);
105 UOP_STAT_INC(uopcode, execution_count);
106
107 // The actual instruction definitions (only one will be used):
108 if (uopcode == _JUMP_TO_TOP) {
109 PATCH_JUMP(_JIT_TOP);
110 }
111 switch (uopcode) {
112 #include "executor_cases.c.h"
113 default:
114 Py_UNREACHABLE();
115 }
116 PATCH_JUMP(_JIT_CONTINUE);
117 // Labels that the instruction implementations expect to exist:
118
119 error_tier_two:
120 tstate->previous_executor = (PyObject *)current_executor;
121 GOTO_TIER_ONE(NULL);
122 exit_to_tier1:
123 tstate->previous_executor = (PyObject *)current_executor;
124 GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target);
125 exit_to_tier1_dynamic:
126 tstate->previous_executor = (PyObject *)current_executor;
127 GOTO_TIER_ONE(frame->instr_ptr);
128 exit_to_trace:
129 {
130 _PyExitData *exit = ¤t_executor->exits[_exit_index];
131 Py_INCREF(exit->executor);
132 tstate->previous_executor = (PyObject *)current_executor;
133 GOTO_TIER_TWO(exit->executor);
134 }
135 }
136