1 /*
2 * Copyright © 2015 Intel
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef _SIMPLE_MTX_H
25 #define _SIMPLE_MTX_H
26
27 #include "util/futex.h"
28 #include "util/macros.h"
29
30 #include "c11/threads.h"
31
32 #if UTIL_FUTEX_SUPPORTED
33
34 /* mtx_t - Fast, simple mutex
35 *
36 * While modern pthread mutexes are very fast (implemented using futex), they
37 * still incur a call to an external DSO and overhead of the generality and
38 * features of pthread mutexes. Most mutexes in mesa only needs lock/unlock,
39 * and the idea here is that we can inline the atomic operation and make the
40 * fast case just two intructions. Mutexes are subtle and finicky to
41 * implement, so we carefully copy the implementation from Ulrich Dreppers
42 * well-written and well-reviewed paper:
43 *
44 * "Futexes Are Tricky"
45 * http://www.akkadia.org/drepper/futex.pdf
46 *
47 * We implement "mutex3", which gives us a mutex that has no syscalls on
48 * uncontended lock or unlock. Further, the uncontended case boils down to a
49 * locked cmpxchg and an untaken branch, the uncontended unlock is just a
50 * locked decr and an untaken branch. We use __builtin_expect() to indicate
51 * that contention is unlikely so that gcc will put the contention code out of
52 * the main code flow.
53 *
54 * A fast mutex only supports lock/unlock, can't be recursive or used with
55 * condition variables.
56 */
57
58 typedef struct {
59 uint32_t val;
60 } simple_mtx_t;
61
62 #define _SIMPLE_MTX_INITIALIZER_NP { 0 }
63
64 #define _SIMPLE_MTX_INVALID_VALUE 0xd0d0d0d0
65
66 static inline void
simple_mtx_init(simple_mtx_t * mtx,ASSERTED int type)67 simple_mtx_init(simple_mtx_t *mtx, ASSERTED int type)
68 {
69 assert(type == mtx_plain);
70
71 mtx->val = 0;
72 }
73
74 static inline void
simple_mtx_destroy(ASSERTED simple_mtx_t * mtx)75 simple_mtx_destroy(ASSERTED simple_mtx_t *mtx)
76 {
77 #ifndef NDEBUG
78 mtx->val = _SIMPLE_MTX_INVALID_VALUE;
79 #endif
80 }
81
82 static inline void
simple_mtx_lock(simple_mtx_t * mtx)83 simple_mtx_lock(simple_mtx_t *mtx)
84 {
85 uint32_t c;
86
87 c = __sync_val_compare_and_swap(&mtx->val, 0, 1);
88
89 assert(c != _SIMPLE_MTX_INVALID_VALUE);
90
91 if (__builtin_expect(c != 0, 0)) {
92 if (c != 2)
93 c = __sync_lock_test_and_set(&mtx->val, 2);
94 while (c != 0) {
95 futex_wait(&mtx->val, 2, NULL);
96 c = __sync_lock_test_and_set(&mtx->val, 2);
97 }
98 }
99 }
100
101 static inline void
simple_mtx_unlock(simple_mtx_t * mtx)102 simple_mtx_unlock(simple_mtx_t *mtx)
103 {
104 uint32_t c;
105
106 c = __sync_fetch_and_sub(&mtx->val, 1);
107
108 assert(c != _SIMPLE_MTX_INVALID_VALUE);
109
110 if (__builtin_expect(c != 1, 0)) {
111 mtx->val = 0;
112 futex_wake(&mtx->val, 1);
113 }
114 }
115
116 static inline void
simple_mtx_assert_locked(simple_mtx_t * mtx)117 simple_mtx_assert_locked(simple_mtx_t *mtx)
118 {
119 assert(mtx->val);
120 }
121
122 #else
123
124 typedef mtx_t simple_mtx_t;
125
126 #define _SIMPLE_MTX_INITIALIZER_NP _MTX_INITIALIZER_NP
127
128 static inline void
simple_mtx_init(simple_mtx_t * mtx,int type)129 simple_mtx_init(simple_mtx_t *mtx, int type)
130 {
131 mtx_init(mtx, type);
132 }
133
134 static inline void
simple_mtx_destroy(simple_mtx_t * mtx)135 simple_mtx_destroy(simple_mtx_t *mtx)
136 {
137 mtx_destroy(mtx);
138 }
139
140 static inline void
simple_mtx_lock(simple_mtx_t * mtx)141 simple_mtx_lock(simple_mtx_t *mtx)
142 {
143 mtx_lock(mtx);
144 }
145
146 static inline void
simple_mtx_unlock(simple_mtx_t * mtx)147 simple_mtx_unlock(simple_mtx_t *mtx)
148 {
149 mtx_unlock(mtx);
150 }
151
152 static inline void
simple_mtx_assert_locked(simple_mtx_t * mtx)153 simple_mtx_assert_locked(simple_mtx_t *mtx)
154 {
155 #ifdef DEBUG
156 /* NOTE: this would not work for recursive mutexes, but
157 * mtx_t doesn't support those
158 */
159 int ret = mtx_trylock(mtx);
160 assert(ret == thrd_busy);
161 if (ret == thrd_success)
162 mtx_unlock(mtx);
163 #else
164 (void)mtx;
165 #endif
166 }
167
168 #endif
169
170 #endif
171