• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
11 //
12 // Mac-specific libdispatch (GCD) support.
13 //===----------------------------------------------------------------------===//
14 
15 #include "sanitizer_common/sanitizer_platform.h"
16 #if SANITIZER_MAC
17 
18 #include "sanitizer_common/sanitizer_common.h"
19 #include "interception/interception.h"
20 #include "tsan_interceptors.h"
21 #include "tsan_platform.h"
22 #include "tsan_rtl.h"
23 
24 #include <Block.h>
25 #include <dispatch/dispatch.h>
26 #include <pthread.h>
27 
28 typedef long long_t;  // NOLINT
29 
30 namespace __tsan {
31 
32 typedef struct {
33   dispatch_queue_t queue;
34   void *orig_context;
35   dispatch_function_t orig_work;
36   bool free_context_in_callback;
37   bool submitted_synchronously;
38   bool is_barrier_block;
39   uptr non_queue_sync_object;
40 } tsan_block_context_t;
41 
42 // The offsets of different fields of the dispatch_queue_t structure, exported
43 // by libdispatch.dylib.
44 extern "C" struct dispatch_queue_offsets_s {
45   const uint16_t dqo_version;
46   const uint16_t dqo_label;
47   const uint16_t dqo_label_size;
48   const uint16_t dqo_flags;
49   const uint16_t dqo_flags_size;
50   const uint16_t dqo_serialnum;
51   const uint16_t dqo_serialnum_size;
52   const uint16_t dqo_width;
53   const uint16_t dqo_width_size;
54   const uint16_t dqo_running;
55   const uint16_t dqo_running_size;
56   const uint16_t dqo_suspend_cnt;
57   const uint16_t dqo_suspend_cnt_size;
58   const uint16_t dqo_target_queue;
59   const uint16_t dqo_target_queue_size;
60   const uint16_t dqo_priority;
61   const uint16_t dqo_priority_size;
62 } dispatch_queue_offsets;
63 
IsQueueSerial(dispatch_queue_t q)64 static bool IsQueueSerial(dispatch_queue_t q) {
65   CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
66   uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
67   CHECK_NE(width, 0);
68   return width == 1;
69 }
70 
GetTargetQueueFromSource(dispatch_source_t source)71 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
72   CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
73   dispatch_queue_t target_queue =
74       *(dispatch_queue_t *)(((uptr)source) +
75                             dispatch_queue_offsets.dqo_target_queue);
76   CHECK_NE(target_queue, 0);
77   return target_queue;
78 }
79 
AllocContext(ThreadState * thr,uptr pc,dispatch_queue_t queue,void * orig_context,dispatch_function_t orig_work)80 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
81                                           dispatch_queue_t queue,
82                                           void *orig_context,
83                                           dispatch_function_t orig_work) {
84   tsan_block_context_t *new_context =
85       (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t));
86   new_context->queue = queue;
87   new_context->orig_context = orig_context;
88   new_context->orig_work = orig_work;
89   new_context->free_context_in_callback = true;
90   new_context->submitted_synchronously = false;
91   new_context->is_barrier_block = false;
92   return new_context;
93 }
94 
dispatch_callback_wrap(void * param)95 static void dispatch_callback_wrap(void *param) {
96   SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
97   tsan_block_context_t *context = (tsan_block_context_t *)param;
98   bool is_queue_serial = context->queue && IsQueueSerial(context->queue);
99   uptr sync_ptr = (uptr)context->queue ?: context->non_queue_sync_object;
100 
101   uptr serial_sync = (uptr)sync_ptr;
102   uptr concurrent_sync = ((uptr)sync_ptr) + sizeof(uptr);
103   uptr submit_sync = (uptr)context;
104   bool serial_task = context->is_barrier_block || is_queue_serial;
105 
106   Acquire(thr, pc, submit_sync);
107   Acquire(thr, pc, serial_sync);
108   if (serial_task) Acquire(thr, pc, concurrent_sync);
109 
110   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
111   context->orig_work(context->orig_context);
112   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
113 
114   Release(thr, pc, serial_task ? serial_sync : concurrent_sync);
115   if (context->submitted_synchronously) Release(thr, pc, submit_sync);
116 
117   if (context->free_context_in_callback) user_free(thr, pc, context);
118 }
119 
invoke_block(void * param)120 static void invoke_block(void *param) {
121   dispatch_block_t block = (dispatch_block_t)param;
122   block();
123 }
124 
invoke_and_release_block(void * param)125 static void invoke_and_release_block(void *param) {
126   dispatch_block_t block = (dispatch_block_t)param;
127   block();
128   Block_release(block);
129 }
130 
131 #define DISPATCH_INTERCEPT_B(name, barrier)                                  \
132   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
133     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
134     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
135     dispatch_block_t heap_block = Block_copy(block);                         \
136     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
137     tsan_block_context_t *new_context =                                      \
138         AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);     \
139     new_context->is_barrier_block = barrier;                                 \
140     Release(thr, pc, (uptr)new_context);                                     \
141     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
142     REAL(name##_f)(q, new_context, dispatch_callback_wrap);                  \
143     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
144   }
145 
146 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier)                             \
147   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
148     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
149     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
150     dispatch_block_t heap_block = Block_copy(block);                         \
151     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
152     tsan_block_context_t new_context = {                                     \
153         q, heap_block, &invoke_and_release_block, false, true, barrier, 0};  \
154     Release(thr, pc, (uptr)&new_context);                                    \
155     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
156     REAL(name##_f)(q, &new_context, dispatch_callback_wrap);                 \
157     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
158     Acquire(thr, pc, (uptr)&new_context);                                    \
159   }
160 
161 #define DISPATCH_INTERCEPT_F(name, barrier)                       \
162   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
163                    dispatch_function_t work) {                    \
164     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);              \
165     tsan_block_context_t *new_context =                           \
166         AllocContext(thr, pc, q, context, work);                  \
167     new_context->is_barrier_block = barrier;                      \
168     Release(thr, pc, (uptr)new_context);                          \
169     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                \
170     REAL(name)(q, new_context, dispatch_callback_wrap);           \
171     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                  \
172   }
173 
174 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier)                              \
175   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context,             \
176                    dispatch_function_t work) {                                \
177     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);                          \
178     tsan_block_context_t new_context = {                                      \
179         q, context, work, false, true, barrier, 0};                           \
180     Release(thr, pc, (uptr)&new_context);                                     \
181     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                            \
182     REAL(name)(q, &new_context, dispatch_callback_wrap);                      \
183     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                              \
184     Acquire(thr, pc, (uptr)&new_context);                                     \
185   }
186 
187 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
188 // context, which is used to synchronize (we release the context before
189 // submitting, and the callback acquires it before executing the original
190 // callback).
DISPATCH_INTERCEPT_B(dispatch_async,false)191 DISPATCH_INTERCEPT_B(dispatch_async, false)
192 DISPATCH_INTERCEPT_B(dispatch_barrier_async, true)
193 DISPATCH_INTERCEPT_F(dispatch_async_f, false)
194 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true)
195 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false)
196 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true)
197 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false)
198 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true)
199 
200 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
201                  dispatch_queue_t queue, dispatch_block_t block) {
202   SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
203   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
204   dispatch_block_t heap_block = Block_copy(block);
205   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
206   tsan_block_context_t *new_context =
207       AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
208   Release(thr, pc, (uptr)new_context);
209   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
210   REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
211   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
212 }
213 
TSAN_INTERCEPTOR(void,dispatch_after_f,dispatch_time_t when,dispatch_queue_t queue,void * context,dispatch_function_t work)214 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
215                  dispatch_queue_t queue, void *context,
216                  dispatch_function_t work) {
217   SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
218   WRAP(dispatch_after)(when, queue, ^(void) {
219     work(context);
220   });
221 }
222 
223 // GCD's dispatch_once implementation has a fast path that contains a racy read
224 // and it's inlined into user's code. Furthermore, this fast path doesn't
225 // establish a proper happens-before relations between the initialization and
226 // code following the call to dispatch_once. We could deal with this in
227 // instrumented code, but there's not much we can do about it in system
228 // libraries. Let's disable the fast path (by never storing the value ~0 to
229 // predicate), so the interceptor is always called, and let's add proper release
230 // and acquire semantics. Since TSan does not see its own atomic stores, the
231 // race on predicate won't be reported - the only accesses to it that TSan sees
232 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
233 // both a macro and a real function, we want to intercept the function, so we
234 // need to undefine the macro.
235 #undef dispatch_once
TSAN_INTERCEPTOR(void,dispatch_once,dispatch_once_t * predicate,dispatch_block_t block)236 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
237                  dispatch_block_t block) {
238   SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
239   atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
240   u32 v = atomic_load(a, memory_order_acquire);
241   if (v == 0 &&
242       atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
243     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
244     block();
245     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
246     Release(thr, pc, (uptr)a);
247     atomic_store(a, 2, memory_order_release);
248   } else {
249     while (v != 2) {
250       internal_sched_yield();
251       v = atomic_load(a, memory_order_acquire);
252     }
253     Acquire(thr, pc, (uptr)a);
254   }
255 }
256 
257 #undef dispatch_once_f
TSAN_INTERCEPTOR(void,dispatch_once_f,dispatch_once_t * predicate,void * context,dispatch_function_t function)258 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
259                  void *context, dispatch_function_t function) {
260   SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
261   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
262   WRAP(dispatch_once)(predicate, ^(void) {
263     function(context);
264   });
265   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
266 }
267 
TSAN_INTERCEPTOR(long_t,dispatch_semaphore_signal,dispatch_semaphore_t dsema)268 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
269                  dispatch_semaphore_t dsema) {
270   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
271   Release(thr, pc, (uptr)dsema);
272   return REAL(dispatch_semaphore_signal)(dsema);
273 }
274 
TSAN_INTERCEPTOR(long_t,dispatch_semaphore_wait,dispatch_semaphore_t dsema,dispatch_time_t timeout)275 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
276                  dispatch_time_t timeout) {
277   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
278   long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
279   if (result == 0) Acquire(thr, pc, (uptr)dsema);
280   return result;
281 }
282 
TSAN_INTERCEPTOR(long_t,dispatch_group_wait,dispatch_group_t group,dispatch_time_t timeout)283 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
284                  dispatch_time_t timeout) {
285   SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
286   long_t result = REAL(dispatch_group_wait)(group, timeout);
287   if (result == 0) Acquire(thr, pc, (uptr)group);
288   return result;
289 }
290 
TSAN_INTERCEPTOR(void,dispatch_group_leave,dispatch_group_t group)291 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
292   SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
293   // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
294   Release(thr, pc, (uptr)group);
295   REAL(dispatch_group_leave)(group);
296 }
297 
TSAN_INTERCEPTOR(void,dispatch_group_async,dispatch_group_t group,dispatch_queue_t queue,dispatch_block_t block)298 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
299                  dispatch_queue_t queue, dispatch_block_t block) {
300   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
301   dispatch_retain(group);
302   dispatch_group_enter(group);
303   __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block);
304   WRAP(dispatch_async)(queue, ^(void) {
305     block_copy();
306     _Block_release(block_copy);
307     WRAP(dispatch_group_leave)(group);
308     dispatch_release(group);
309   });
310 }
311 
TSAN_INTERCEPTOR(void,dispatch_group_async_f,dispatch_group_t group,dispatch_queue_t queue,void * context,dispatch_function_t work)312 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
313                  dispatch_queue_t queue, void *context,
314                  dispatch_function_t work) {
315   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
316   dispatch_retain(group);
317   dispatch_group_enter(group);
318   WRAP(dispatch_async)(queue, ^(void) {
319     work(context);
320     WRAP(dispatch_group_leave)(group);
321     dispatch_release(group);
322   });
323 }
324 
TSAN_INTERCEPTOR(void,dispatch_group_notify,dispatch_group_t group,dispatch_queue_t q,dispatch_block_t block)325 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
326                  dispatch_queue_t q, dispatch_block_t block) {
327   SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
328 
329   // To make sure the group is still available in the callback (otherwise
330   // it can be already destroyed).  Will be released in the callback.
331   dispatch_retain(group);
332 
333   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
334   dispatch_block_t heap_block = Block_copy(^(void) {
335     {
336       SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
337       // Released when leaving the group (dispatch_group_leave).
338       Acquire(thr, pc, (uptr)group);
339     }
340     dispatch_release(group);
341     block();
342   });
343   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
344   tsan_block_context_t *new_context =
345       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
346   new_context->is_barrier_block = true;
347   Release(thr, pc, (uptr)new_context);
348   REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
349 }
350 
TSAN_INTERCEPTOR(void,dispatch_group_notify_f,dispatch_group_t group,dispatch_queue_t q,void * context,dispatch_function_t work)351 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
352                  dispatch_queue_t q, void *context, dispatch_function_t work) {
353   WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
354 }
355 
TSAN_INTERCEPTOR(void,dispatch_source_set_event_handler,dispatch_source_t source,dispatch_block_t handler)356 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
357                  dispatch_source_t source, dispatch_block_t handler) {
358   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
359   if (handler == nullptr)
360     return REAL(dispatch_source_set_event_handler)(source, nullptr);
361   dispatch_queue_t q = GetTargetQueueFromSource(source);
362   __block tsan_block_context_t new_context = {
363       q, handler, &invoke_block, false, false, false, 0 };
364   dispatch_block_t new_handler = Block_copy(^(void) {
365     new_context.orig_context = handler;  // To explicitly capture "handler".
366     dispatch_callback_wrap(&new_context);
367   });
368   uptr submit_sync = (uptr)&new_context;
369   Release(thr, pc, submit_sync);
370   REAL(dispatch_source_set_event_handler)(source, new_handler);
371   Block_release(new_handler);
372 }
373 
TSAN_INTERCEPTOR(void,dispatch_source_set_event_handler_f,dispatch_source_t source,dispatch_function_t handler)374 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
375                  dispatch_source_t source, dispatch_function_t handler) {
376   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
377   if (handler == nullptr)
378     return REAL(dispatch_source_set_event_handler)(source, nullptr);
379   dispatch_block_t block = ^(void) {
380     handler(dispatch_get_context(source));
381   };
382   WRAP(dispatch_source_set_event_handler)(source, block);
383 }
384 
TSAN_INTERCEPTOR(void,dispatch_source_set_cancel_handler,dispatch_source_t source,dispatch_block_t handler)385 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
386                  dispatch_source_t source, dispatch_block_t handler) {
387   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
388   if (handler == nullptr)
389     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
390   dispatch_queue_t q = GetTargetQueueFromSource(source);
391   __block tsan_block_context_t new_context = {
392       q, handler, &invoke_block, false, false, false, 0};
393   dispatch_block_t new_handler = Block_copy(^(void) {
394     new_context.orig_context = handler;  // To explicitly capture "handler".
395     dispatch_callback_wrap(&new_context);
396   });
397   uptr submit_sync = (uptr)&new_context;
398   Release(thr, pc, submit_sync);
399   REAL(dispatch_source_set_cancel_handler)(source, new_handler);
400   Block_release(new_handler);
401 }
402 
TSAN_INTERCEPTOR(void,dispatch_source_set_cancel_handler_f,dispatch_source_t source,dispatch_function_t handler)403 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
404                  dispatch_source_t source, dispatch_function_t handler) {
405   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
406                           handler);
407   if (handler == nullptr)
408     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
409   dispatch_block_t block = ^(void) {
410     handler(dispatch_get_context(source));
411   };
412   WRAP(dispatch_source_set_cancel_handler)(source, block);
413 }
414 
TSAN_INTERCEPTOR(void,dispatch_source_set_registration_handler,dispatch_source_t source,dispatch_block_t handler)415 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
416                  dispatch_source_t source, dispatch_block_t handler) {
417   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
418                           handler);
419   if (handler == nullptr)
420     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
421   dispatch_queue_t q = GetTargetQueueFromSource(source);
422   __block tsan_block_context_t new_context = {
423       q, handler, &invoke_block, false, false, false, 0};
424   dispatch_block_t new_handler = Block_copy(^(void) {
425     new_context.orig_context = handler;  // To explicitly capture "handler".
426     dispatch_callback_wrap(&new_context);
427   });
428   uptr submit_sync = (uptr)&new_context;
429   Release(thr, pc, submit_sync);
430   REAL(dispatch_source_set_registration_handler)(source, new_handler);
431   Block_release(new_handler);
432 }
433 
TSAN_INTERCEPTOR(void,dispatch_source_set_registration_handler_f,dispatch_source_t source,dispatch_function_t handler)434 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
435                  dispatch_source_t source, dispatch_function_t handler) {
436   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
437                           handler);
438   if (handler == nullptr)
439     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
440   dispatch_block_t block = ^(void) {
441     handler(dispatch_get_context(source));
442   };
443   WRAP(dispatch_source_set_registration_handler)(source, block);
444 }
445 
446 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
447                  dispatch_queue_t queue, void (^block)(size_t)) {
448   SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
449 
450   void *parent_to_child_sync = nullptr;
451   uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync;
452   void *child_to_parent_sync = nullptr;
453   uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync;
454 
455   Release(thr, pc, parent_to_child_sync_uptr);
456   void (^new_block)(size_t) = ^(size_t iteration) {
457     SCOPED_INTERCEPTOR_RAW(dispatch_apply);
458     Acquire(thr, pc, parent_to_child_sync_uptr);
459     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
460     block(iteration);
461     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
462     Release(thr, pc, child_to_parent_sync_uptr);
463   };
464   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
465   REAL(dispatch_apply)(iterations, queue, new_block);
466   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
467   Acquire(thr, pc, child_to_parent_sync_uptr);
468 }
469 
TSAN_INTERCEPTOR(void,dispatch_apply_f,size_t iterations,dispatch_queue_t queue,void * context,void (* work)(void *,size_t))470 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
471                  dispatch_queue_t queue, void *context,
472                  void (*work)(void *, size_t)) {
473   SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
474   void (^new_block)(size_t) = ^(size_t iteration) {
475     work(context, iteration);
476   };
477   WRAP(dispatch_apply)(iterations, queue, new_block);
478 }
479 
DECLARE_REAL_AND_INTERCEPTOR(void,free,void * ptr)480 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
481 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
482 
483 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
484                  size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
485   SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
486   if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
487     return REAL(dispatch_data_create)(buffer, size, q, destructor);
488 
489   if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
490     destructor = ^(void) { WRAP(free)((void *)buffer); };
491   else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
492     destructor = ^(void) { WRAP(munmap)((void *)buffer, size); };
493 
494   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
495   dispatch_block_t heap_block = Block_copy(destructor);
496   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
497   tsan_block_context_t *new_context =
498       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
499   uptr submit_sync = (uptr)new_context;
500   Release(thr, pc, submit_sync);
501   return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
502     dispatch_callback_wrap(new_context);
503   });
504 }
505 
506 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
507 typedef void (^cleanup_handler_t)(int error);
508 
TSAN_INTERCEPTOR(void,dispatch_read,dispatch_fd_t fd,size_t length,dispatch_queue_t q,fd_handler_t h)509 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
510                  dispatch_queue_t q, fd_handler_t h) {
511   SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
512   __block tsan_block_context_t new_context = {
513       q, nullptr, &invoke_block, false, false, false, 0};
514   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
515     new_context.orig_context = ^(void) {
516       h(data, error);
517     };
518     dispatch_callback_wrap(&new_context);
519   });
520   uptr submit_sync = (uptr)&new_context;
521   Release(thr, pc, submit_sync);
522   REAL(dispatch_read)(fd, length, q, new_h);
523   Block_release(new_h);
524 }
525 
TSAN_INTERCEPTOR(void,dispatch_write,dispatch_fd_t fd,dispatch_data_t data,dispatch_queue_t q,fd_handler_t h)526 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
527                  dispatch_queue_t q, fd_handler_t h) {
528   SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
529   __block tsan_block_context_t new_context = {
530       q, nullptr, &invoke_block, false, false, false, 0};
531   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
532     new_context.orig_context = ^(void) {
533       h(data, error);
534     };
535     dispatch_callback_wrap(&new_context);
536   });
537   uptr submit_sync = (uptr)&new_context;
538   Release(thr, pc, submit_sync);
539   REAL(dispatch_write)(fd, data, q, new_h);
540   Block_release(new_h);
541 }
542 
TSAN_INTERCEPTOR(void,dispatch_io_read,dispatch_io_t channel,off_t offset,size_t length,dispatch_queue_t q,dispatch_io_handler_t h)543 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
544                  size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
545   SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
546   __block tsan_block_context_t new_context = {
547       q, nullptr, &invoke_block, false, false, false, 0};
548   dispatch_io_handler_t new_h =
549       Block_copy(^(bool done, dispatch_data_t data, int error) {
550         new_context.orig_context = ^(void) {
551           h(done, data, error);
552         };
553         dispatch_callback_wrap(&new_context);
554       });
555   uptr submit_sync = (uptr)&new_context;
556   Release(thr, pc, submit_sync);
557   REAL(dispatch_io_read)(channel, offset, length, q, new_h);
558   Block_release(new_h);
559 }
560 
TSAN_INTERCEPTOR(void,dispatch_io_write,dispatch_io_t channel,off_t offset,dispatch_data_t data,dispatch_queue_t q,dispatch_io_handler_t h)561 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
562                  dispatch_data_t data, dispatch_queue_t q,
563                  dispatch_io_handler_t h) {
564   SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
565   __block tsan_block_context_t new_context = {
566       q, nullptr, &invoke_block, false, false, false, 0};
567   dispatch_io_handler_t new_h =
568       Block_copy(^(bool done, dispatch_data_t data, int error) {
569         new_context.orig_context = ^(void) {
570           h(done, data, error);
571         };
572         dispatch_callback_wrap(&new_context);
573       });
574   uptr submit_sync = (uptr)&new_context;
575   Release(thr, pc, submit_sync);
576   REAL(dispatch_io_write)(channel, offset, data, q, new_h);
577   Block_release(new_h);
578 }
579 
TSAN_INTERCEPTOR(void,dispatch_io_barrier,dispatch_io_t channel,dispatch_block_t barrier)580 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
581                  dispatch_block_t barrier) {
582   SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
583   __block tsan_block_context_t new_context = {
584       nullptr, nullptr, &invoke_block, false, false, false, 0};
585   new_context.non_queue_sync_object = (uptr)channel;
586   new_context.is_barrier_block = true;
587   dispatch_block_t new_block = Block_copy(^(void) {
588     new_context.orig_context = ^(void) {
589       barrier();
590     };
591     dispatch_callback_wrap(&new_context);
592   });
593   uptr submit_sync = (uptr)&new_context;
594   Release(thr, pc, submit_sync);
595   REAL(dispatch_io_barrier)(channel, new_block);
596   Block_release(new_block);
597 }
598 
TSAN_INTERCEPTOR(dispatch_io_t,dispatch_io_create,dispatch_io_type_t type,dispatch_fd_t fd,dispatch_queue_t q,cleanup_handler_t h)599 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
600                  dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
601   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
602   __block dispatch_io_t new_channel = nullptr;
603   __block tsan_block_context_t new_context = {
604       q, nullptr, &invoke_block, false, false, false, 0};
605   cleanup_handler_t new_h = Block_copy(^(int error) {
606     {
607       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
608       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
609     }
610     new_context.orig_context = ^(void) {
611       h(error);
612     };
613     dispatch_callback_wrap(&new_context);
614   });
615   uptr submit_sync = (uptr)&new_context;
616   Release(thr, pc, submit_sync);
617   new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
618   Block_release(new_h);
619   return new_channel;
620 }
621 
TSAN_INTERCEPTOR(dispatch_io_t,dispatch_io_create_with_path,dispatch_io_type_t type,const char * path,int oflag,mode_t mode,dispatch_queue_t q,cleanup_handler_t h)622 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
623                  dispatch_io_type_t type, const char *path, int oflag,
624                  mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
625   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
626                           q, h);
627   __block dispatch_io_t new_channel = nullptr;
628   __block tsan_block_context_t new_context = {
629       q, nullptr, &invoke_block, false, false, false, 0};
630   cleanup_handler_t new_h = Block_copy(^(int error) {
631     {
632       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
633       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
634     }
635     new_context.orig_context = ^(void) {
636       h(error);
637     };
638     dispatch_callback_wrap(&new_context);
639   });
640   uptr submit_sync = (uptr)&new_context;
641   Release(thr, pc, submit_sync);
642   new_channel =
643       REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
644   Block_release(new_h);
645   return new_channel;
646 }
647 
TSAN_INTERCEPTOR(dispatch_io_t,dispatch_io_create_with_io,dispatch_io_type_t type,dispatch_io_t io,dispatch_queue_t q,cleanup_handler_t h)648 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
649                  dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
650                  cleanup_handler_t h) {
651   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
652   __block dispatch_io_t new_channel = nullptr;
653   __block tsan_block_context_t new_context = {
654       q, nullptr, &invoke_block, false, false, false, 0};
655   cleanup_handler_t new_h = Block_copy(^(int error) {
656     {
657       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
658       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
659     }
660     new_context.orig_context = ^(void) {
661       h(error);
662     };
663     dispatch_callback_wrap(&new_context);
664   });
665   uptr submit_sync = (uptr)&new_context;
666   Release(thr, pc, submit_sync);
667   new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
668   Block_release(new_h);
669   return new_channel;
670 }
671 
TSAN_INTERCEPTOR(void,dispatch_io_close,dispatch_io_t channel,dispatch_io_close_flags_t flags)672 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
673                  dispatch_io_close_flags_t flags) {
674   SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
675   Release(thr, pc, (uptr)channel);  // Acquire() in dispatch_io_create[_*].
676   return REAL(dispatch_io_close)(channel, flags);
677 }
678 
679 }  // namespace __tsan
680 
681 #endif  // SANITIZER_MAC
682