1 #include "pthread_impl.h"
2
3 /* This lock primitive combines a flag (in the sign bit) and a
4 * congestion count (= threads inside the critical section, CS) in a
5 * single int that is accessed through atomic operations. The states
6 * of the int for value x are:
7 *
8 * x == 0: unlocked and no thread inside the critical section
9 *
10 * x < 0: locked with a congestion of x-INT_MIN, including the thread
11 * that holds the lock
12 *
13 * x > 0: unlocked with a congestion of x
14 *
15 * or in an equivalent formulation x is the congestion count or'ed
16 * with INT_MIN as a lock flag.
17 */
18
19 #ifdef ENABLE_HWASAN
20 __attribute__((no_sanitize("hwaddress")))
21 #endif
__lock(volatile int * l)22 void __lock(volatile int *l)
23 {
24 int need_locks = libc.need_locks;
25 if (!need_locks) return;
26 /* fast path: INT_MIN for the lock, +1 for the congestion */
27 int current = a_cas(l, 0, INT_MIN + 1);
28 if (need_locks < 0) libc.need_locks = 0;
29 if (!current) return;
30 /* A first spin loop, for medium congestion. */
31 for (unsigned i = 0; i < 10; ++i) {
32 if (current < 0) current -= INT_MIN + 1;
33 // assertion: current >= 0
34 int val = a_cas(l, current, INT_MIN + (current + 1));
35 if (val == current) return;
36 current = val;
37 }
38 // Spinning failed, so mark ourselves as being inside the CS.
39 current = a_fetch_add(l, 1) + 1;
40 /* The main lock acquisition loop for heavy congestion. The only
41 * change to the value performed inside that loop is a successful
42 * lock via the CAS that acquires the lock. */
43 for (;;) {
44 /* We can only go into wait, if we know that somebody holds the
45 * lock and will eventually wake us up, again. */
46 if (current < 0) {
47 __futexwait(l, current, 1);
48 current -= INT_MIN + 1;
49 }
50 /* assertion: current > 0, the count includes us already. */
51 int val = a_cas(l, current, INT_MIN + current);
52 if (val == current) return;
53 current = val;
54 }
55 }
56
57 #ifdef ENABLE_HWASAN
58 __attribute__((no_sanitize("hwaddress")))
59 #endif
__unlock(volatile int * l)60 void __unlock(volatile int *l)
61 {
62 /* Check l[0] to see if we are multi-threaded. */
63 if (l[0] < 0) {
64 if (a_fetch_add(l, -(INT_MIN + 1)) != (INT_MIN + 1)) {
65 __wake(l, 1, 1);
66 }
67 }
68 }
69