1 /*
2 * Copyright © 2016 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
14 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
15 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
16 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
17 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * The above copyright notice and this permission notice (including the
23 * next paragraph) shall be included in all copies or substantial portions
24 * of the Software.
25 */
26
27 /* Job queue with execution in a separate thread.
28 *
29 * Jobs can be added from any thread. After that, the wait call can be used
30 * to wait for completion of the job.
31 */
32
33 #ifndef U_QUEUE_H
34 #define U_QUEUE_H
35
36 #include <string.h>
37
38 #include "simple_mtx.h"
39 #include "util/futex.h"
40 #include "util/list.h"
41 #include "util/macros.h"
42 #include "util/os_time.h"
43 #include "util/u_atomic.h"
44 #include "util/u_thread.h"
45
46 #ifdef __cplusplus
47 extern "C" {
48 #endif
49
50 #define UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY (1 << 0)
51 #define UTIL_QUEUE_INIT_RESIZE_IF_FULL (1 << 1)
52 #define UTIL_QUEUE_INIT_SET_FULL_THREAD_AFFINITY (1 << 2)
53 #define UTIL_QUEUE_INIT_SCALE_THREADS (1 << 3)
54
55 #if UTIL_FUTEX_SUPPORTED
56 #define UTIL_QUEUE_FENCE_FUTEX
57 #else
58 #define UTIL_QUEUE_FENCE_STANDARD
59 #endif
60
61 #ifdef UTIL_QUEUE_FENCE_FUTEX
62 /* Job completion fence.
63 * Put this into your job structure.
64 */
65 struct util_queue_fence {
66 /* The fence can be in one of three states:
67 * 0 - signaled
68 * 1 - unsignaled
69 * 2 - unsignaled, may have waiters
70 */
71 uint32_t val;
72 };
73
74 static inline void
util_queue_fence_init(struct util_queue_fence * fence)75 util_queue_fence_init(struct util_queue_fence *fence)
76 {
77 fence->val = 0;
78 }
79
80 static inline void
util_queue_fence_destroy(struct util_queue_fence * fence)81 util_queue_fence_destroy(struct util_queue_fence *fence)
82 {
83 assert(p_atomic_read_relaxed(&fence->val) == 0);
84 /* no-op */
85 }
86
87 static inline void
util_queue_fence_signal(struct util_queue_fence * fence)88 util_queue_fence_signal(struct util_queue_fence *fence)
89 {
90 uint32_t val = p_atomic_xchg(&fence->val, 0);
91
92 assert(val != 0);
93
94 if (val == 2)
95 futex_wake(&fence->val, INT_MAX);
96 }
97
98 /**
99 * Move \p fence back into unsignalled state.
100 *
101 * \warning The caller must ensure that no other thread may currently be
102 * waiting (or about to wait) on the fence.
103 */
104 static inline void
util_queue_fence_reset(struct util_queue_fence * fence)105 util_queue_fence_reset(struct util_queue_fence *fence)
106 {
107 #ifdef NDEBUG
108 fence->val = 1;
109 #else
110 uint32_t v = p_atomic_xchg(&fence->val, 1);
111 assert(v == 0);
112 #endif
113 }
114
115 static inline bool
util_queue_fence_is_signalled(struct util_queue_fence * fence)116 util_queue_fence_is_signalled(struct util_queue_fence *fence)
117 {
118 return p_atomic_read_relaxed(&fence->val) == 0;
119 }
120 #endif
121
122 #ifdef UTIL_QUEUE_FENCE_STANDARD
123 /* Job completion fence.
124 * Put this into your job structure.
125 */
126 struct util_queue_fence {
127 mtx_t mutex;
128 cnd_t cond;
129 int signalled;
130 };
131
132 void util_queue_fence_init(struct util_queue_fence *fence);
133 void util_queue_fence_destroy(struct util_queue_fence *fence);
134 void util_queue_fence_signal(struct util_queue_fence *fence);
135
136 /**
137 * Move \p fence back into unsignalled state.
138 *
139 * \warning The caller must ensure that no other thread may currently be
140 * waiting (or about to wait) on the fence.
141 */
142 static inline void
util_queue_fence_reset(struct util_queue_fence * fence)143 util_queue_fence_reset(struct util_queue_fence *fence)
144 {
145 assert(fence->signalled);
146 fence->signalled = 0;
147 }
148
149 static inline bool
util_queue_fence_is_signalled(struct util_queue_fence * fence)150 util_queue_fence_is_signalled(struct util_queue_fence *fence)
151 {
152 return fence->signalled != 0;
153 }
154 #endif
155
156 void
157 _util_queue_fence_wait(struct util_queue_fence *fence);
158
159 static inline void
util_queue_fence_wait(struct util_queue_fence * fence)160 util_queue_fence_wait(struct util_queue_fence *fence)
161 {
162 if (unlikely(!util_queue_fence_is_signalled(fence)))
163 _util_queue_fence_wait(fence);
164 }
165
166 bool
167 _util_queue_fence_wait_timeout(struct util_queue_fence *fence,
168 int64_t abs_timeout);
169
170 /**
171 * Wait for the fence to be signaled with a timeout.
172 *
173 * \param fence the fence
174 * \param abs_timeout the absolute timeout in nanoseconds, relative to the
175 * clock provided by os_time_get_nano.
176 *
177 * \return true if the fence was signaled, false if the timeout occurred.
178 */
179 static inline bool
util_queue_fence_wait_timeout(struct util_queue_fence * fence,int64_t abs_timeout)180 util_queue_fence_wait_timeout(struct util_queue_fence *fence,
181 int64_t abs_timeout)
182 {
183 if (util_queue_fence_is_signalled(fence))
184 return true;
185
186 if (abs_timeout == (int64_t)OS_TIMEOUT_INFINITE) {
187 _util_queue_fence_wait(fence);
188 return true;
189 }
190
191 return _util_queue_fence_wait_timeout(fence, abs_timeout);
192 }
193
194 typedef void (*util_queue_execute_func)(void *job, void *gdata, int thread_index);
195
196 struct util_queue_job {
197 void *job;
198 void *global_data;
199 size_t job_size;
200 struct util_queue_fence *fence;
201 util_queue_execute_func execute;
202 util_queue_execute_func cleanup;
203 };
204
205 /* Put this into your context. */
206 struct util_queue {
207 char name[14]; /* 13 characters = the thread name without the index */
208 simple_mtx_t finish_lock; /* for util_queue_finish and protects threads/num_threads */
209 mtx_t lock;
210 cnd_t has_queued_cond;
211 cnd_t has_space_cond;
212 thrd_t *threads;
213 unsigned flags;
214 int num_queued;
215 unsigned max_threads;
216 unsigned num_threads; /* decreasing this number will terminate threads */
217 int max_jobs;
218 int write_idx, read_idx; /* ring buffer pointers */
219 size_t total_jobs_size; /* memory use of all jobs in the queue */
220 struct util_queue_job *jobs;
221 void *global_data;
222
223 /* for cleanup at exit(), protected by exit_mutex */
224 struct list_head head;
225 };
226
227 bool util_queue_init(struct util_queue *queue,
228 const char *name,
229 unsigned max_jobs,
230 unsigned num_threads,
231 unsigned flags,
232 void *global_data);
233 void util_queue_destroy(struct util_queue *queue);
234
235 /* optional cleanup callback is called after fence is signaled: */
236 void util_queue_add_job(struct util_queue *queue,
237 void *job,
238 struct util_queue_fence *fence,
239 util_queue_execute_func execute,
240 util_queue_execute_func cleanup,
241 const size_t job_size);
242 void util_queue_drop_job(struct util_queue *queue,
243 struct util_queue_fence *fence);
244
245 void util_queue_finish(struct util_queue *queue);
246
247 /* Adjust the number of active threads. The new number of threads can't be
248 * greater than the initial number of threads at the creation of the queue,
249 * and it can't be less than 1.
250 */
251 void
252 util_queue_adjust_num_threads(struct util_queue *queue, unsigned num_threads);
253
254 int64_t util_queue_get_thread_time_nano(struct util_queue *queue,
255 unsigned thread_index);
256
257 /* util_queue needs to be cleared to zeroes for this to work */
258 static inline bool
util_queue_is_initialized(struct util_queue * queue)259 util_queue_is_initialized(struct util_queue *queue)
260 {
261 return queue->threads != NULL;
262 }
263
264 /* Convenient structure for monitoring the queue externally and passing
265 * the structure between Mesa components. The queue doesn't use it directly.
266 */
267 struct util_queue_monitoring
268 {
269 /* For querying the thread busyness. */
270 struct util_queue *queue;
271
272 /* Counters updated by the user of the queue. */
273 unsigned num_offloaded_items;
274 unsigned num_direct_items;
275 unsigned num_syncs;
276 };
277
278 #ifdef __cplusplus
279 }
280 #endif
281
282 #endif
283