1 /*
2 * Implementation of the Global Interpreter Lock (GIL).
3 */
4
5 #include <stdlib.h>
6 #include <errno.h>
7
8
9 /* First some general settings */
10
11 #define INTERVAL (_PyRuntime.ceval.gil.interval >= 1 ? _PyRuntime.ceval.gil.interval : 1)
12
13
14 /*
15 Notes about the implementation:
16
17 - The GIL is just a boolean variable (locked) whose access is protected
18 by a mutex (gil_mutex), and whose changes are signalled by a condition
19 variable (gil_cond). gil_mutex is taken for short periods of time,
20 and therefore mostly uncontended.
21
22 - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be
23 able to release the GIL on demand by another thread. A volatile boolean
24 variable (gil_drop_request) is used for that purpose, which is checked
25 at every turn of the eval loop. That variable is set after a wait of
26 `interval` microseconds on `gil_cond` has timed out.
27
28 [Actually, another volatile boolean variable (eval_breaker) is used
29 which ORs several conditions into one. Volatile booleans are
30 sufficient as inter-thread signalling means since Python is run
31 on cache-coherent architectures only.]
32
33 - A thread wanting to take the GIL will first let pass a given amount of
34 time (`interval` microseconds) before setting gil_drop_request. This
35 encourages a defined switching period, but doesn't enforce it since
36 opcodes can take an arbitrary time to execute.
37
38 The `interval` value is available for the user to read and modify
39 using the Python API `sys.{get,set}switchinterval()`.
40
41 - When a thread releases the GIL and gil_drop_request is set, that thread
42 ensures that another GIL-awaiting thread gets scheduled.
43 It does so by waiting on a condition variable (switch_cond) until
44 the value of last_holder is changed to something else than its
45 own thread state pointer, indicating that another thread was able to
46 take the GIL.
47
48 This is meant to prohibit the latency-adverse behaviour on multi-core
49 machines where one thread would speculatively release the GIL, but still
50 run and end up being the first to re-acquire it, making the "timeslices"
51 much longer than expected.
52 (Note: this mechanism is enabled with FORCE_SWITCHING above)
53 */
54
55 #include "condvar.h"
56
57 #define MUTEX_INIT(mut) \
58 if (PyMUTEX_INIT(&(mut))) { \
59 Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); };
60 #define MUTEX_FINI(mut) \
61 if (PyMUTEX_FINI(&(mut))) { \
62 Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); };
63 #define MUTEX_LOCK(mut) \
64 if (PyMUTEX_LOCK(&(mut))) { \
65 Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); };
66 #define MUTEX_UNLOCK(mut) \
67 if (PyMUTEX_UNLOCK(&(mut))) { \
68 Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); };
69
70 #define COND_INIT(cond) \
71 if (PyCOND_INIT(&(cond))) { \
72 Py_FatalError("PyCOND_INIT(" #cond ") failed"); };
73 #define COND_FINI(cond) \
74 if (PyCOND_FINI(&(cond))) { \
75 Py_FatalError("PyCOND_FINI(" #cond ") failed"); };
76 #define COND_SIGNAL(cond) \
77 if (PyCOND_SIGNAL(&(cond))) { \
78 Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); };
79 #define COND_WAIT(cond, mut) \
80 if (PyCOND_WAIT(&(cond), &(mut))) { \
81 Py_FatalError("PyCOND_WAIT(" #cond ") failed"); };
82 #define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \
83 { \
84 int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \
85 if (r < 0) \
86 Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \
87 if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \
88 timeout_result = 1; \
89 else \
90 timeout_result = 0; \
91 } \
92
93
94 #define DEFAULT_INTERVAL 5000
95
_gil_initialize(struct _gil_runtime_state * state)96 static void _gil_initialize(struct _gil_runtime_state *state)
97 {
98 _Py_atomic_int uninitialized = {-1};
99 state->locked = uninitialized;
100 state->interval = DEFAULT_INTERVAL;
101 }
102
gil_created(void)103 static int gil_created(void)
104 {
105 return (_Py_atomic_load_explicit(&_PyRuntime.ceval.gil.locked,
106 _Py_memory_order_acquire)
107 ) >= 0;
108 }
109
create_gil(void)110 static void create_gil(void)
111 {
112 MUTEX_INIT(_PyRuntime.ceval.gil.mutex);
113 #ifdef FORCE_SWITCHING
114 MUTEX_INIT(_PyRuntime.ceval.gil.switch_mutex);
115 #endif
116 COND_INIT(_PyRuntime.ceval.gil.cond);
117 #ifdef FORCE_SWITCHING
118 COND_INIT(_PyRuntime.ceval.gil.switch_cond);
119 #endif
120 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder, 0);
121 _Py_ANNOTATE_RWLOCK_CREATE(&_PyRuntime.ceval.gil.locked);
122 _Py_atomic_store_explicit(&_PyRuntime.ceval.gil.locked, 0,
123 _Py_memory_order_release);
124 }
125
destroy_gil(void)126 static void destroy_gil(void)
127 {
128 /* some pthread-like implementations tie the mutex to the cond
129 * and must have the cond destroyed first.
130 */
131 COND_FINI(_PyRuntime.ceval.gil.cond);
132 MUTEX_FINI(_PyRuntime.ceval.gil.mutex);
133 #ifdef FORCE_SWITCHING
134 COND_FINI(_PyRuntime.ceval.gil.switch_cond);
135 MUTEX_FINI(_PyRuntime.ceval.gil.switch_mutex);
136 #endif
137 _Py_atomic_store_explicit(&_PyRuntime.ceval.gil.locked, -1,
138 _Py_memory_order_release);
139 _Py_ANNOTATE_RWLOCK_DESTROY(&_PyRuntime.ceval.gil.locked);
140 }
141
recreate_gil(void)142 static void recreate_gil(void)
143 {
144 _Py_ANNOTATE_RWLOCK_DESTROY(&_PyRuntime.ceval.gil.locked);
145 /* XXX should we destroy the old OS resources here? */
146 create_gil();
147 }
148
drop_gil(PyThreadState * tstate)149 static void drop_gil(PyThreadState *tstate)
150 {
151 if (!_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked))
152 Py_FatalError("drop_gil: GIL is not locked");
153 /* tstate is allowed to be NULL (early interpreter init) */
154 if (tstate != NULL) {
155 /* Sub-interpreter support: threads might have been switched
156 under our feet using PyThreadState_Swap(). Fix the GIL last
157 holder variable so that our heuristics work. */
158 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder,
159 (uintptr_t)tstate);
160 }
161
162 MUTEX_LOCK(_PyRuntime.ceval.gil.mutex);
163 _Py_ANNOTATE_RWLOCK_RELEASED(&_PyRuntime.ceval.gil.locked, /*is_write=*/1);
164 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.locked, 0);
165 COND_SIGNAL(_PyRuntime.ceval.gil.cond);
166 MUTEX_UNLOCK(_PyRuntime.ceval.gil.mutex);
167
168 #ifdef FORCE_SWITCHING
169 if (_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil_drop_request) &&
170 tstate != NULL)
171 {
172 MUTEX_LOCK(_PyRuntime.ceval.gil.switch_mutex);
173 /* Not switched yet => wait */
174 if (((PyThreadState*)_Py_atomic_load_relaxed(
175 &_PyRuntime.ceval.gil.last_holder)
176 ) == tstate)
177 {
178 RESET_GIL_DROP_REQUEST();
179 /* NOTE: if COND_WAIT does not atomically start waiting when
180 releasing the mutex, another thread can run through, take
181 the GIL and drop it again, and reset the condition
182 before we even had a chance to wait for it. */
183 COND_WAIT(_PyRuntime.ceval.gil.switch_cond,
184 _PyRuntime.ceval.gil.switch_mutex);
185 }
186 MUTEX_UNLOCK(_PyRuntime.ceval.gil.switch_mutex);
187 }
188 #endif
189 }
190
take_gil(PyThreadState * tstate)191 static void take_gil(PyThreadState *tstate)
192 {
193 int err;
194 if (tstate == NULL)
195 Py_FatalError("take_gil: NULL tstate");
196
197 err = errno;
198 MUTEX_LOCK(_PyRuntime.ceval.gil.mutex);
199
200 if (!_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked))
201 goto _ready;
202
203 while (_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked)) {
204 int timed_out = 0;
205 unsigned long saved_switchnum;
206
207 saved_switchnum = _PyRuntime.ceval.gil.switch_number;
208 COND_TIMED_WAIT(_PyRuntime.ceval.gil.cond, _PyRuntime.ceval.gil.mutex,
209 INTERVAL, timed_out);
210 /* If we timed out and no switch occurred in the meantime, it is time
211 to ask the GIL-holding thread to drop it. */
212 if (timed_out &&
213 _Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked) &&
214 _PyRuntime.ceval.gil.switch_number == saved_switchnum) {
215 SET_GIL_DROP_REQUEST();
216 }
217 }
218 _ready:
219 #ifdef FORCE_SWITCHING
220 /* This mutex must be taken before modifying
221 _PyRuntime.ceval.gil.last_holder (see drop_gil()). */
222 MUTEX_LOCK(_PyRuntime.ceval.gil.switch_mutex);
223 #endif
224 /* We now hold the GIL */
225 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.locked, 1);
226 _Py_ANNOTATE_RWLOCK_ACQUIRED(&_PyRuntime.ceval.gil.locked, /*is_write=*/1);
227
228 if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(
229 &_PyRuntime.ceval.gil.last_holder))
230 {
231 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder,
232 (uintptr_t)tstate);
233 ++_PyRuntime.ceval.gil.switch_number;
234 }
235
236 #ifdef FORCE_SWITCHING
237 COND_SIGNAL(_PyRuntime.ceval.gil.switch_cond);
238 MUTEX_UNLOCK(_PyRuntime.ceval.gil.switch_mutex);
239 #endif
240 if (_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil_drop_request)) {
241 RESET_GIL_DROP_REQUEST();
242 }
243 if (tstate->async_exc != NULL) {
244 _PyEval_SignalAsyncExc();
245 }
246
247 MUTEX_UNLOCK(_PyRuntime.ceval.gil.mutex);
248 errno = err;
249 }
250
_PyEval_SetSwitchInterval(unsigned long microseconds)251 void _PyEval_SetSwitchInterval(unsigned long microseconds)
252 {
253 _PyRuntime.ceval.gil.interval = microseconds;
254 }
255
_PyEval_GetSwitchInterval()256 unsigned long _PyEval_GetSwitchInterval()
257 {
258 return _PyRuntime.ceval.gil.interval;
259 }
260