1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
3
4 typedef struct witness_s witness_t;
5 typedef unsigned witness_rank_t;
6 typedef ql_head(witness_t) witness_list_t;
7 typedef int witness_comp_t (const witness_t *, const witness_t *);
8
9 /*
10 * Lock ranks. Witnesses with rank WITNESS_RANK_OMIT are completely ignored by
11 * the witness machinery.
12 */
13 #define WITNESS_RANK_OMIT 0U
14
15 #define WITNESS_RANK_INIT 1U
16 #define WITNESS_RANK_CTL 1U
17 #define WITNESS_RANK_TCACHES 2U
18 #define WITNESS_RANK_ARENAS 3U
19
20 #define WITNESS_RANK_PROF_DUMP 4U
21 #define WITNESS_RANK_PROF_BT2GCTX 5U
22 #define WITNESS_RANK_PROF_TDATAS 6U
23 #define WITNESS_RANK_PROF_TDATA 7U
24 #define WITNESS_RANK_PROF_GCTX 8U
25
26 #define WITNESS_RANK_ARENA 9U
27 #define WITNESS_RANK_ARENA_CHUNKS 10U
28 #define WITNESS_RANK_ARENA_NODE_CACHE 11U
29
30 #define WITNESS_RANK_BASE 12U
31
32 #define WITNESS_RANK_LEAF 0xffffffffU
33 #define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF
34 #define WITNESS_RANK_ARENA_HUGE WITNESS_RANK_LEAF
35 #define WITNESS_RANK_DSS WITNESS_RANK_LEAF
36 #define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF
37 #define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF
38 #define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF
39 #define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF
40 #define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF
41
42 #define WITNESS_INITIALIZER(rank) {"initializer", rank, NULL, {NULL, NULL}}
43
44 #endif /* JEMALLOC_H_TYPES */
45 /******************************************************************************/
46 #ifdef JEMALLOC_H_STRUCTS
47
48 struct witness_s {
49 /* Name, used for printing lock order reversal messages. */
50 const char *name;
51
52 /*
53 * Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses
54 * must be acquired in order of increasing rank.
55 */
56 witness_rank_t rank;
57
58 /*
59 * If two witnesses are of equal rank and they have the samp comp
60 * function pointer, it is called as a last attempt to differentiate
61 * between witnesses of equal rank.
62 */
63 witness_comp_t *comp;
64
65 /* Linkage for thread's currently owned locks. */
66 ql_elm(witness_t) link;
67 };
68
69 #endif /* JEMALLOC_H_STRUCTS */
70 /******************************************************************************/
71 #ifdef JEMALLOC_H_EXTERNS
72
73 void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
74 witness_comp_t *comp);
75 #ifdef JEMALLOC_JET
76 typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
77 extern witness_lock_error_t *witness_lock_error;
78 #else
79 void witness_lock_error(const witness_list_t *witnesses,
80 const witness_t *witness);
81 #endif
82 #ifdef JEMALLOC_JET
83 typedef void (witness_owner_error_t)(const witness_t *);
84 extern witness_owner_error_t *witness_owner_error;
85 #else
86 void witness_owner_error(const witness_t *witness);
87 #endif
88 #ifdef JEMALLOC_JET
89 typedef void (witness_not_owner_error_t)(const witness_t *);
90 extern witness_not_owner_error_t *witness_not_owner_error;
91 #else
92 void witness_not_owner_error(const witness_t *witness);
93 #endif
94 #ifdef JEMALLOC_JET
95 typedef void (witness_lockless_error_t)(const witness_list_t *);
96 extern witness_lockless_error_t *witness_lockless_error;
97 #else
98 void witness_lockless_error(const witness_list_t *witnesses);
99 #endif
100
101 void witnesses_cleanup(tsd_t *tsd);
102 void witness_fork_cleanup(tsd_t *tsd);
103 void witness_prefork(tsd_t *tsd);
104 void witness_postfork_parent(tsd_t *tsd);
105 void witness_postfork_child(tsd_t *tsd);
106
107 #endif /* JEMALLOC_H_EXTERNS */
108 /******************************************************************************/
109 #ifdef JEMALLOC_H_INLINES
110
111 #ifndef JEMALLOC_ENABLE_INLINE
112 bool witness_owner(tsd_t *tsd, const witness_t *witness);
113 void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
114 void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
115 void witness_assert_lockless(tsdn_t *tsdn);
116 void witness_lock(tsdn_t *tsdn, witness_t *witness);
117 void witness_unlock(tsdn_t *tsdn, witness_t *witness);
118 #endif
119
120 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
121 JEMALLOC_INLINE bool
witness_owner(tsd_t * tsd,const witness_t * witness)122 witness_owner(tsd_t *tsd, const witness_t *witness)
123 {
124 witness_list_t *witnesses;
125 witness_t *w;
126
127 witnesses = tsd_witnessesp_get(tsd);
128 ql_foreach(w, witnesses, link) {
129 if (w == witness)
130 return (true);
131 }
132
133 return (false);
134 }
135
136 JEMALLOC_INLINE void
witness_assert_owner(tsdn_t * tsdn,const witness_t * witness)137 witness_assert_owner(tsdn_t *tsdn, const witness_t *witness)
138 {
139 tsd_t *tsd;
140
141 if (!config_debug)
142 return;
143
144 if (tsdn_null(tsdn))
145 return;
146 tsd = tsdn_tsd(tsdn);
147 if (witness->rank == WITNESS_RANK_OMIT)
148 return;
149
150 if (witness_owner(tsd, witness))
151 return;
152 witness_owner_error(witness);
153 }
154
155 JEMALLOC_INLINE void
witness_assert_not_owner(tsdn_t * tsdn,const witness_t * witness)156 witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
157 {
158 tsd_t *tsd;
159 witness_list_t *witnesses;
160 witness_t *w;
161
162 if (!config_debug)
163 return;
164
165 if (tsdn_null(tsdn))
166 return;
167 tsd = tsdn_tsd(tsdn);
168 if (witness->rank == WITNESS_RANK_OMIT)
169 return;
170
171 witnesses = tsd_witnessesp_get(tsd);
172 ql_foreach(w, witnesses, link) {
173 if (w == witness)
174 witness_not_owner_error(witness);
175 }
176 }
177
178 JEMALLOC_INLINE void
witness_assert_lockless(tsdn_t * tsdn)179 witness_assert_lockless(tsdn_t *tsdn)
180 {
181 tsd_t *tsd;
182 witness_list_t *witnesses;
183 witness_t *w;
184
185 if (!config_debug)
186 return;
187
188 if (tsdn_null(tsdn))
189 return;
190 tsd = tsdn_tsd(tsdn);
191
192 witnesses = tsd_witnessesp_get(tsd);
193 w = ql_last(witnesses, link);
194 if (w != NULL)
195 witness_lockless_error(witnesses);
196 }
197
198 JEMALLOC_INLINE void
witness_lock(tsdn_t * tsdn,witness_t * witness)199 witness_lock(tsdn_t *tsdn, witness_t *witness)
200 {
201 tsd_t *tsd;
202 witness_list_t *witnesses;
203 witness_t *w;
204
205 if (!config_debug)
206 return;
207
208 if (tsdn_null(tsdn))
209 return;
210 tsd = tsdn_tsd(tsdn);
211 if (witness->rank == WITNESS_RANK_OMIT)
212 return;
213
214 witness_assert_not_owner(tsdn, witness);
215
216 witnesses = tsd_witnessesp_get(tsd);
217 w = ql_last(witnesses, link);
218 if (w == NULL) {
219 /* No other locks; do nothing. */
220 } else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) {
221 /* Forking, and relaxed ranking satisfied. */
222 } else if (w->rank > witness->rank) {
223 /* Not forking, rank order reversal. */
224 witness_lock_error(witnesses, witness);
225 } else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
226 witness->comp || w->comp(w, witness) > 0)) {
227 /*
228 * Missing/incompatible comparison function, or comparison
229 * function indicates rank order reversal.
230 */
231 witness_lock_error(witnesses, witness);
232 }
233
234 ql_elm_new(witness, link);
235 ql_tail_insert(witnesses, witness, link);
236 }
237
238 JEMALLOC_INLINE void
witness_unlock(tsdn_t * tsdn,witness_t * witness)239 witness_unlock(tsdn_t *tsdn, witness_t *witness)
240 {
241 tsd_t *tsd;
242 witness_list_t *witnesses;
243
244 if (!config_debug)
245 return;
246
247 if (tsdn_null(tsdn))
248 return;
249 tsd = tsdn_tsd(tsdn);
250 if (witness->rank == WITNESS_RANK_OMIT)
251 return;
252
253 /*
254 * Check whether owner before removal, rather than relying on
255 * witness_assert_owner() to abort, so that unit tests can test this
256 * function's failure mode without causing undefined behavior.
257 */
258 if (witness_owner(tsd, witness)) {
259 witnesses = tsd_witnessesp_get(tsd);
260 ql_remove(witnesses, witness, link);
261 } else
262 witness_assert_owner(tsdn, witness);
263 }
264 #endif
265
266 #endif /* JEMALLOC_H_INLINES */
267 /******************************************************************************/
268