1 #include "Python.h"
2
3 #include "pycore_lock.h"
4 #include "pycore_critical_section.h"
5
6 #ifdef Py_GIL_DISABLED
7 static_assert(_Alignof(PyCriticalSection) >= 4,
8 "critical section must be aligned to at least 4 bytes");
9 #endif
10
11 void
_PyCriticalSection_BeginSlow(PyCriticalSection * c,PyMutex * m)12 _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m)
13 {
14 #ifdef Py_GIL_DISABLED
15 PyThreadState *tstate = _PyThreadState_GET();
16 c->_cs_mutex = NULL;
17 c->_cs_prev = (uintptr_t)tstate->critical_section;
18 tstate->critical_section = (uintptr_t)c;
19
20 PyMutex_Lock(m);
21 c->_cs_mutex = m;
22 #endif
23 }
24
25 void
_PyCriticalSection2_BeginSlow(PyCriticalSection2 * c,PyMutex * m1,PyMutex * m2,int is_m1_locked)26 _PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2,
27 int is_m1_locked)
28 {
29 #ifdef Py_GIL_DISABLED
30 PyThreadState *tstate = _PyThreadState_GET();
31 c->_cs_base._cs_mutex = NULL;
32 c->_cs_mutex2 = NULL;
33 c->_cs_base._cs_prev = tstate->critical_section;
34 tstate->critical_section = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES;
35
36 if (!is_m1_locked) {
37 PyMutex_Lock(m1);
38 }
39 PyMutex_Lock(m2);
40 c->_cs_base._cs_mutex = m1;
41 c->_cs_mutex2 = m2;
42 #endif
43 }
44
45 #ifdef Py_GIL_DISABLED
46 static PyCriticalSection *
untag_critical_section(uintptr_t tag)47 untag_critical_section(uintptr_t tag)
48 {
49 return (PyCriticalSection *)(tag & ~_Py_CRITICAL_SECTION_MASK);
50 }
51 #endif
52
53 // Release all locks held by critical sections. This is called by
54 // _PyThreadState_Detach.
55 void
_PyCriticalSection_SuspendAll(PyThreadState * tstate)56 _PyCriticalSection_SuspendAll(PyThreadState *tstate)
57 {
58 #ifdef Py_GIL_DISABLED
59 uintptr_t *tagptr = &tstate->critical_section;
60 while (_PyCriticalSection_IsActive(*tagptr)) {
61 PyCriticalSection *c = untag_critical_section(*tagptr);
62
63 if (c->_cs_mutex) {
64 PyMutex_Unlock(c->_cs_mutex);
65 if ((*tagptr & _Py_CRITICAL_SECTION_TWO_MUTEXES)) {
66 PyCriticalSection2 *c2 = (PyCriticalSection2 *)c;
67 if (c2->_cs_mutex2) {
68 PyMutex_Unlock(c2->_cs_mutex2);
69 }
70 }
71 }
72
73 *tagptr |= _Py_CRITICAL_SECTION_INACTIVE;
74 tagptr = &c->_cs_prev;
75 }
76 #endif
77 }
78
79 void
_PyCriticalSection_Resume(PyThreadState * tstate)80 _PyCriticalSection_Resume(PyThreadState *tstate)
81 {
82 #ifdef Py_GIL_DISABLED
83 uintptr_t p = tstate->critical_section;
84 PyCriticalSection *c = untag_critical_section(p);
85 assert(!_PyCriticalSection_IsActive(p));
86
87 PyMutex *m1 = c->_cs_mutex;
88 c->_cs_mutex = NULL;
89
90 PyMutex *m2 = NULL;
91 PyCriticalSection2 *c2 = NULL;
92 if ((p & _Py_CRITICAL_SECTION_TWO_MUTEXES)) {
93 c2 = (PyCriticalSection2 *)c;
94 m2 = c2->_cs_mutex2;
95 c2->_cs_mutex2 = NULL;
96 }
97
98 if (m1) {
99 PyMutex_Lock(m1);
100 }
101 if (m2) {
102 PyMutex_Lock(m2);
103 }
104
105 c->_cs_mutex = m1;
106 if (m2) {
107 c2->_cs_mutex2 = m2;
108 }
109
110 tstate->critical_section &= ~_Py_CRITICAL_SECTION_INACTIVE;
111 #endif
112 }
113
114 #undef PyCriticalSection_Begin
115 void
PyCriticalSection_Begin(PyCriticalSection * c,PyObject * op)116 PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op)
117 {
118 #ifdef Py_GIL_DISABLED
119 _PyCriticalSection_Begin(c, op);
120 #endif
121 }
122
123 #undef PyCriticalSection_End
124 void
PyCriticalSection_End(PyCriticalSection * c)125 PyCriticalSection_End(PyCriticalSection *c)
126 {
127 #ifdef Py_GIL_DISABLED
128 _PyCriticalSection_End(c);
129 #endif
130 }
131
132 #undef PyCriticalSection2_Begin
133 void
PyCriticalSection2_Begin(PyCriticalSection2 * c,PyObject * a,PyObject * b)134 PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b)
135 {
136 #ifdef Py_GIL_DISABLED
137 _PyCriticalSection2_Begin(c, a, b);
138 #endif
139 }
140
141 #undef PyCriticalSection2_End
142 void
PyCriticalSection2_End(PyCriticalSection2 * c)143 PyCriticalSection2_End(PyCriticalSection2 *c)
144 {
145 #ifdef Py_GIL_DISABLED
146 _PyCriticalSection2_End(c);
147 #endif
148 }
149