1 /*
2 * Implementation of the Global Interpreter Lock (GIL).
3 */
4
5 #include <stdlib.h>
6 #include <errno.h>
7
8
9 /* First some general settings */
10
11 /* microseconds (the Python API uses seconds, though) */
12 #define DEFAULT_INTERVAL 5000
13 static unsigned long gil_interval = DEFAULT_INTERVAL;
14 #define INTERVAL (gil_interval >= 1 ? gil_interval : 1)
15
16 /* Enable if you want to force the switching of threads at least every `gil_interval` */
17 #undef FORCE_SWITCHING
18 #define FORCE_SWITCHING
19
20
21 /*
22 Notes about the implementation:
23
24 - The GIL is just a boolean variable (gil_locked) whose access is protected
25 by a mutex (gil_mutex), and whose changes are signalled by a condition
26 variable (gil_cond). gil_mutex is taken for short periods of time,
27 and therefore mostly uncontended.
28
29 - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be
30 able to release the GIL on demand by another thread. A volatile boolean
31 variable (gil_drop_request) is used for that purpose, which is checked
32 at every turn of the eval loop. That variable is set after a wait of
33 `interval` microseconds on `gil_cond` has timed out.
34
35 [Actually, another volatile boolean variable (eval_breaker) is used
36 which ORs several conditions into one. Volatile booleans are
37 sufficient as inter-thread signalling means since Python is run
38 on cache-coherent architectures only.]
39
40 - A thread wanting to take the GIL will first let pass a given amount of
41 time (`interval` microseconds) before setting gil_drop_request. This
42 encourages a defined switching period, but doesn't enforce it since
43 opcodes can take an arbitrary time to execute.
44
45 The `interval` value is available for the user to read and modify
46 using the Python API `sys.{get,set}switchinterval()`.
47
48 - When a thread releases the GIL and gil_drop_request is set, that thread
49 ensures that another GIL-awaiting thread gets scheduled.
50 It does so by waiting on a condition variable (switch_cond) until
51 the value of gil_last_holder is changed to something else than its
52 own thread state pointer, indicating that another thread was able to
53 take the GIL.
54
55 This is meant to prohibit the latency-adverse behaviour on multi-core
56 machines where one thread would speculatively release the GIL, but still
57 run and end up being the first to re-acquire it, making the "timeslices"
58 much longer than expected.
59 (Note: this mechanism is enabled with FORCE_SWITCHING above)
60 */
61
62 #include "condvar.h"
63 #ifndef Py_HAVE_CONDVAR
64 #error You need either a POSIX-compatible or a Windows system!
65 #endif
66
67 #define MUTEX_T PyMUTEX_T
68 #define MUTEX_INIT(mut) \
69 if (PyMUTEX_INIT(&(mut))) { \
70 Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); };
71 #define MUTEX_FINI(mut) \
72 if (PyMUTEX_FINI(&(mut))) { \
73 Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); };
74 #define MUTEX_LOCK(mut) \
75 if (PyMUTEX_LOCK(&(mut))) { \
76 Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); };
77 #define MUTEX_UNLOCK(mut) \
78 if (PyMUTEX_UNLOCK(&(mut))) { \
79 Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); };
80
81 #define COND_T PyCOND_T
82 #define COND_INIT(cond) \
83 if (PyCOND_INIT(&(cond))) { \
84 Py_FatalError("PyCOND_INIT(" #cond ") failed"); };
85 #define COND_FINI(cond) \
86 if (PyCOND_FINI(&(cond))) { \
87 Py_FatalError("PyCOND_FINI(" #cond ") failed"); };
88 #define COND_SIGNAL(cond) \
89 if (PyCOND_SIGNAL(&(cond))) { \
90 Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); };
91 #define COND_WAIT(cond, mut) \
92 if (PyCOND_WAIT(&(cond), &(mut))) { \
93 Py_FatalError("PyCOND_WAIT(" #cond ") failed"); };
94 #define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \
95 { \
96 int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \
97 if (r < 0) \
98 Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \
99 if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \
100 timeout_result = 1; \
101 else \
102 timeout_result = 0; \
103 } \
104
105
106
107 /* Whether the GIL is already taken (-1 if uninitialized). This is atomic
108 because it can be read without any lock taken in ceval.c. */
109 static _Py_atomic_int gil_locked = {-1};
110 /* Number of GIL switches since the beginning. */
111 static unsigned long gil_switch_number = 0;
112 /* Last PyThreadState holding / having held the GIL. This helps us know
113 whether anyone else was scheduled after we dropped the GIL. */
114 static _Py_atomic_address gil_last_holder = {0};
115
116 /* This condition variable allows one or several threads to wait until
117 the GIL is released. In addition, the mutex also protects the above
118 variables. */
119 static COND_T gil_cond;
120 static MUTEX_T gil_mutex;
121
122 #ifdef FORCE_SWITCHING
123 /* This condition variable helps the GIL-releasing thread wait for
124 a GIL-awaiting thread to be scheduled and take the GIL. */
125 static COND_T switch_cond;
126 static MUTEX_T switch_mutex;
127 #endif
128
129
gil_created(void)130 static int gil_created(void)
131 {
132 return _Py_atomic_load_explicit(&gil_locked, _Py_memory_order_acquire) >= 0;
133 }
134
create_gil(void)135 static void create_gil(void)
136 {
137 MUTEX_INIT(gil_mutex);
138 #ifdef FORCE_SWITCHING
139 MUTEX_INIT(switch_mutex);
140 #endif
141 COND_INIT(gil_cond);
142 #ifdef FORCE_SWITCHING
143 COND_INIT(switch_cond);
144 #endif
145 _Py_atomic_store_relaxed(&gil_last_holder, 0);
146 _Py_ANNOTATE_RWLOCK_CREATE(&gil_locked);
147 _Py_atomic_store_explicit(&gil_locked, 0, _Py_memory_order_release);
148 }
149
destroy_gil(void)150 static void destroy_gil(void)
151 {
152 /* some pthread-like implementations tie the mutex to the cond
153 * and must have the cond destroyed first.
154 */
155 COND_FINI(gil_cond);
156 MUTEX_FINI(gil_mutex);
157 #ifdef FORCE_SWITCHING
158 COND_FINI(switch_cond);
159 MUTEX_FINI(switch_mutex);
160 #endif
161 _Py_atomic_store_explicit(&gil_locked, -1, _Py_memory_order_release);
162 _Py_ANNOTATE_RWLOCK_DESTROY(&gil_locked);
163 }
164
recreate_gil(void)165 static void recreate_gil(void)
166 {
167 _Py_ANNOTATE_RWLOCK_DESTROY(&gil_locked);
168 /* XXX should we destroy the old OS resources here? */
169 create_gil();
170 }
171
drop_gil(PyThreadState * tstate)172 static void drop_gil(PyThreadState *tstate)
173 {
174 if (!_Py_atomic_load_relaxed(&gil_locked))
175 Py_FatalError("drop_gil: GIL is not locked");
176 /* tstate is allowed to be NULL (early interpreter init) */
177 if (tstate != NULL) {
178 /* Sub-interpreter support: threads might have been switched
179 under our feet using PyThreadState_Swap(). Fix the GIL last
180 holder variable so that our heuristics work. */
181 _Py_atomic_store_relaxed(&gil_last_holder, (uintptr_t)tstate);
182 }
183
184 MUTEX_LOCK(gil_mutex);
185 _Py_ANNOTATE_RWLOCK_RELEASED(&gil_locked, /*is_write=*/1);
186 _Py_atomic_store_relaxed(&gil_locked, 0);
187 COND_SIGNAL(gil_cond);
188 MUTEX_UNLOCK(gil_mutex);
189
190 #ifdef FORCE_SWITCHING
191 if (_Py_atomic_load_relaxed(&gil_drop_request) && tstate != NULL) {
192 MUTEX_LOCK(switch_mutex);
193 /* Not switched yet => wait */
194 if ((PyThreadState*)_Py_atomic_load_relaxed(&gil_last_holder) == tstate) {
195 RESET_GIL_DROP_REQUEST();
196 /* NOTE: if COND_WAIT does not atomically start waiting when
197 releasing the mutex, another thread can run through, take
198 the GIL and drop it again, and reset the condition
199 before we even had a chance to wait for it. */
200 COND_WAIT(switch_cond, switch_mutex);
201 }
202 MUTEX_UNLOCK(switch_mutex);
203 }
204 #endif
205 }
206
take_gil(PyThreadState * tstate)207 static void take_gil(PyThreadState *tstate)
208 {
209 int err;
210 if (tstate == NULL)
211 Py_FatalError("take_gil: NULL tstate");
212
213 err = errno;
214 MUTEX_LOCK(gil_mutex);
215
216 if (!_Py_atomic_load_relaxed(&gil_locked))
217 goto _ready;
218
219 while (_Py_atomic_load_relaxed(&gil_locked)) {
220 int timed_out = 0;
221 unsigned long saved_switchnum;
222
223 saved_switchnum = gil_switch_number;
224 COND_TIMED_WAIT(gil_cond, gil_mutex, INTERVAL, timed_out);
225 /* If we timed out and no switch occurred in the meantime, it is time
226 to ask the GIL-holding thread to drop it. */
227 if (timed_out &&
228 _Py_atomic_load_relaxed(&gil_locked) &&
229 gil_switch_number == saved_switchnum) {
230 SET_GIL_DROP_REQUEST();
231 }
232 }
233 _ready:
234 #ifdef FORCE_SWITCHING
235 /* This mutex must be taken before modifying gil_last_holder (see drop_gil()). */
236 MUTEX_LOCK(switch_mutex);
237 #endif
238 /* We now hold the GIL */
239 _Py_atomic_store_relaxed(&gil_locked, 1);
240 _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil_locked, /*is_write=*/1);
241
242 if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil_last_holder)) {
243 _Py_atomic_store_relaxed(&gil_last_holder, (uintptr_t)tstate);
244 ++gil_switch_number;
245 }
246
247 #ifdef FORCE_SWITCHING
248 COND_SIGNAL(switch_cond);
249 MUTEX_UNLOCK(switch_mutex);
250 #endif
251 if (_Py_atomic_load_relaxed(&gil_drop_request)) {
252 RESET_GIL_DROP_REQUEST();
253 }
254 if (tstate->async_exc != NULL) {
255 _PyEval_SignalAsyncExc();
256 }
257
258 MUTEX_UNLOCK(gil_mutex);
259 errno = err;
260 }
261
_PyEval_SetSwitchInterval(unsigned long microseconds)262 void _PyEval_SetSwitchInterval(unsigned long microseconds)
263 {
264 gil_interval = microseconds;
265 }
266
_PyEval_GetSwitchInterval()267 unsigned long _PyEval_GetSwitchInterval()
268 {
269 return gil_interval;
270 }
271