1 /*
2 * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 *
23 * Authors:
24 * Rob Clark <robclark@freedesktop.org>
25 */
26
27 #ifndef FREEDRENO_PRIV_H_
28 #define FREEDRENO_PRIV_H_
29
30 #include <errno.h>
31 #include <fcntl.h>
32 #include <stdio.h>
33 #include <stdlib.h>
34 #include <string.h>
35 #include <unistd.h>
36 #include <sys/ioctl.h>
37 #include <sys/mman.h>
38
39 #include <xf86drm.h>
40
41 #include "util/hash_table.h"
42 #include "util/list.h"
43 #include "util/log.h"
44 #include "util/simple_mtx.h"
45 #include "util/slab.h"
46 #include "util/u_atomic.h"
47 #include "util/u_debug.h"
48 #include "util/u_math.h"
49
50 #include "freedreno_dev_info.h"
51 #include "freedreno_drmif.h"
52 #include "freedreno_ringbuffer.h"
53
54 extern simple_mtx_t table_lock;
55
56 /*
57 * Stupid/simple growable array implementation:
58 */
59
60 #define MAX_ARRAY_SIZE ((unsigned short)~0)
61
62 static inline void
grow(void ** ptr,uint16_t nr,uint16_t * max,uint16_t sz)63 grow(void **ptr, uint16_t nr, uint16_t *max, uint16_t sz)
64 {
65 assert((nr + 1) < MAX_ARRAY_SIZE);
66 if ((nr + 1) > *max) {
67 if (*max > MAX_ARRAY_SIZE/2)
68 *max = MAX_ARRAY_SIZE;
69 else if ((*max * 2) < (nr + 1))
70 *max = nr + 5;
71 else
72 *max = *max * 2;
73 *ptr = realloc(*ptr, *max * sz);
74 }
75 }
76
77 #define DECLARE_ARRAY(type, name) \
78 unsigned short nr_##name, max_##name; \
79 type *name;
80
81 #define APPEND(x, name, ...) \
82 ({ \
83 grow((void **)&(x)->name, (x)->nr_##name, &(x)->max_##name, \
84 sizeof((x)->name[0])); \
85 (x)->name[(x)->nr_##name] = __VA_ARGS__; \
86 (x)->nr_##name++; \
87 })
88
89 #define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x))
90
91
92 struct fd_device_funcs {
93 /* Create a new buffer object:
94 */
95 struct fd_bo *(*bo_new)(struct fd_device *dev, uint32_t size, uint32_t flags);
96
97 /* Create a new buffer object from existing handle (ie. dma-buf or
98 * flink import):
99 */
100 struct fd_bo *(*bo_from_handle)(struct fd_device *dev, uint32_t size,
101 uint32_t handle);
102
103 struct fd_pipe *(*pipe_new)(struct fd_device *dev, enum fd_pipe_id id,
104 unsigned prio);
105 void (*destroy)(struct fd_device *dev);
106 };
107
108 struct fd_bo_bucket {
109 uint32_t size;
110 struct list_head list;
111 };
112
113 struct fd_bo_cache {
114 struct fd_bo_bucket cache_bucket[14 * 4];
115 int num_buckets;
116 time_t time;
117 };
118
119 struct fd_device {
120 int fd;
121 enum fd_version version;
122 int32_t refcnt;
123
124 /* tables to keep track of bo's, to avoid "evil-twin" fd_bo objects:
125 *
126 * handle_table: maps handle to fd_bo
127 * name_table: maps flink name to fd_bo
128 *
129 * We end up needing two tables, because DRM_IOCTL_GEM_OPEN always
130 * returns a new handle. So we need to figure out if the bo is already
131 * open in the process first, before calling gem-open.
132 */
133 struct hash_table *handle_table, *name_table;
134
135 const struct fd_device_funcs *funcs;
136
137 struct fd_bo_cache bo_cache;
138 struct fd_bo_cache ring_cache;
139
140 bool has_cached_coherent;
141
142 bool closefd; /* call close(fd) upon destruction */
143
144 /* just for valgrind: */
145 int bo_size;
146
147 /**
148 * List of deferred submits, protected by submit_lock. The deferred
149 * submits are tracked globally per-device, even if they execute in
150 * different order on the kernel side (ie. due to different priority
151 * submitqueues, etc) to preserve the order that they are passed off
152 * to the kernel. Once the kernel has them, it is the fences' job
153 * to preserve correct order of execution.
154 */
155 struct list_head deferred_submits;
156 unsigned deferred_cmds;
157 simple_mtx_t submit_lock;
158
159 /**
160 * BO for suballocating long-lived state objects.
161 *
162 * Note: one would be tempted to put this in fd_pipe to avoid locking.
163 * But that is a bad idea for a couple of reasons:
164 *
165 * 1) With TC, stateobj allocation can happen in either frontend thread
166 * (ie. most CSOs), and also driver thread (a6xx cached tex state)
167 * 2) It is best for fd_pipe to not hold a reference to a BO that can
168 * be free'd to bo cache, as that can cause unexpected re-entrancy
169 * (fd_bo_cache_alloc() -> find_in_bucket() -> fd_bo_state() ->
170 * cleanup_fences() -> drop pipe ref which free's bo's).
171 */
172 struct fd_bo *suballoc_bo;
173 uint32_t suballoc_offset;
174 simple_mtx_t suballoc_lock;
175
176 struct util_queue submit_queue;
177 };
178
179 #define foreach_submit(name, list) \
180 list_for_each_entry(struct fd_submit, name, list, node)
181 #define foreach_submit_safe(name, list) \
182 list_for_each_entry_safe(struct fd_submit, name, list, node)
183 #define last_submit(list) \
184 list_last_entry(list, struct fd_submit, node)
185
186 void fd_bo_cache_init(struct fd_bo_cache *cache, int coarse);
187 void fd_bo_cache_cleanup(struct fd_bo_cache *cache, time_t time);
188 struct fd_bo *fd_bo_cache_alloc(struct fd_bo_cache *cache, uint32_t *size,
189 uint32_t flags);
190 int fd_bo_cache_free(struct fd_bo_cache *cache, struct fd_bo *bo);
191
192 /* for where @table_lock is already held: */
193 void fd_bo_del_locked(struct fd_bo *bo);
194 void fd_device_del_locked(struct fd_device *dev);
195 void fd_pipe_del_locked(struct fd_pipe *pipe);
196
197 struct fd_pipe_funcs {
198 struct fd_ringbuffer *(*ringbuffer_new_object)(struct fd_pipe *pipe,
199 uint32_t size);
200 struct fd_submit *(*submit_new)(struct fd_pipe *pipe);
201
202 /**
203 * Flush any deferred submits (if deferred submits are supported by
204 * the pipe implementation)
205 */
206 void (*flush)(struct fd_pipe *pipe, uint32_t fence);
207
208 int (*get_param)(struct fd_pipe *pipe, enum fd_param_id param,
209 uint64_t *value);
210 int (*set_param)(struct fd_pipe *pipe, enum fd_param_id param,
211 uint64_t value);
212 int (*wait)(struct fd_pipe *pipe, const struct fd_fence *fence,
213 uint64_t timeout);
214 void (*destroy)(struct fd_pipe *pipe);
215 };
216
217 struct fd_pipe_control {
218 uint32_t fence;
219 };
220 #define control_ptr(pipe, member) \
221 (pipe)->control_mem, offsetof(struct fd_pipe_control, member), 0, 0
222
223 struct fd_pipe {
224 struct fd_device *dev;
225 enum fd_pipe_id id;
226 struct fd_dev_id dev_id;
227
228 /**
229 * Note refcnt is *not* atomic, but protected by table_lock, since the
230 * table_lock is held in fd_bo_add_fence(), which is the hotpath.
231 */
232 int32_t refcnt;
233
234 /**
235 * Previous fence seqno allocated for this pipe. The fd_pipe represents
236 * a single timeline, fences allocated by this pipe can be compared to
237 * each other, but fences from different pipes are not comparable (as
238 * there could be preemption of multiple priority level submitqueues at
239 * play)
240 */
241 uint32_t last_fence;
242
243 /**
244 * The last fence seqno that was flushed to kernel (doesn't mean that it
245 * is complete, just that the kernel knows about it)
246 */
247 uint32_t last_submit_fence;
248
249 uint32_t last_enqueue_fence; /* just for debugging */
250
251 struct fd_bo *control_mem;
252 volatile struct fd_pipe_control *control;
253
254 struct slab_parent_pool ring_pool;
255
256 const struct fd_pipe_funcs *funcs;
257 };
258
259 uint32_t fd_pipe_emit_fence(struct fd_pipe *pipe, struct fd_ringbuffer *ring);
260
261 static inline void
fd_pipe_flush(struct fd_pipe * pipe,uint32_t fence)262 fd_pipe_flush(struct fd_pipe *pipe, uint32_t fence)
263 {
264 if (!pipe->funcs->flush)
265 return;
266 pipe->funcs->flush(pipe, fence);
267 }
268
269 struct fd_submit_funcs {
270 struct fd_ringbuffer *(*new_ringbuffer)(struct fd_submit *submit,
271 uint32_t size,
272 enum fd_ringbuffer_flags flags);
273 int (*flush)(struct fd_submit *submit, int in_fence_fd,
274 struct fd_submit_fence *out_fence);
275 void (*destroy)(struct fd_submit *submit);
276 };
277
278 struct fd_submit {
279 int32_t refcnt;
280 struct fd_pipe *pipe;
281 const struct fd_submit_funcs *funcs;
282
283 struct fd_ringbuffer *primary;
284 uint32_t fence;
285 struct list_head node; /* node in fd_pipe::deferred_submits */
286 };
287
288 static inline unsigned
fd_dev_count_deferred_cmds(struct fd_device * dev)289 fd_dev_count_deferred_cmds(struct fd_device *dev)
290 {
291 unsigned nr = 0;
292
293 simple_mtx_assert_locked(&dev->submit_lock);
294
295 list_for_each_entry (struct fd_submit, submit, &dev->deferred_submits, node) {
296 nr += fd_ringbuffer_cmd_count(submit->primary);
297 }
298
299 return nr;
300 }
301
302 struct fd_bo_funcs {
303 int (*offset)(struct fd_bo *bo, uint64_t *offset);
304 int (*cpu_prep)(struct fd_bo *bo, struct fd_pipe *pipe, uint32_t op);
305 void (*cpu_fini)(struct fd_bo *bo);
306 int (*madvise)(struct fd_bo *bo, int willneed);
307 uint64_t (*iova)(struct fd_bo *bo);
308 void (*set_name)(struct fd_bo *bo, const char *fmt, va_list ap);
309 void (*destroy)(struct fd_bo *bo);
310
311 /**
312 * Optional, copy data into bo, falls back to mmap+memcpy. If not
313 * implemented, it must be possible to mmap all buffers
314 */
315 void (*upload)(struct fd_bo *bo, void *src, unsigned len);
316 };
317
318 struct fd_bo_fence {
319 /* For non-shared buffers, track the last pipe the buffer was active
320 * on, and the per-pipe fence value that indicates when the buffer is
321 * idle:
322 */
323 uint32_t fence;
324 struct fd_pipe *pipe;
325 };
326
327 struct fd_bo {
328 struct fd_device *dev;
329 uint32_t size;
330 uint32_t handle;
331 uint32_t name;
332 int32_t refcnt;
333 uint32_t reloc_flags; /* flags like FD_RELOC_DUMP to use for relocs to this BO */
334 uint32_t alloc_flags; /* flags that control allocation/mapping, ie. FD_BO_x */
335 uint64_t iova;
336 void *map;
337 const struct fd_bo_funcs *funcs;
338
339 enum {
340 NO_CACHE = 0,
341 BO_CACHE = 1,
342 RING_CACHE = 2,
343 } bo_reuse : 2;
344
345 /* Buffers that are shared (imported or exported) may be used in
346 * other processes, so we need to fallback to kernel to determine
347 * busyness.
348 */
349 bool shared : 1;
350
351 /* We need to be able to disable userspace fence synchronization for
352 * special internal buffers, namely the pipe->control buffer, to avoid
353 * a circular reference loop.
354 */
355 bool nosync : 1;
356
357 /* Most recent index in submit's bo table, used to optimize the common
358 * case where a bo is used many times in the same submit.
359 */
360 uint32_t idx;
361
362 struct list_head list; /* bucket-list entry */
363 time_t free_time; /* time when added to bucket-list */
364
365 DECLARE_ARRAY(struct fd_bo_fence, fences);
366
367 /* In the common case, there is no more than one fence attached.
368 * This provides storage for the fences table until it grows to
369 * be larger than a single element.
370 */
371 struct fd_bo_fence _inline_fence;
372 };
373
374 void fd_bo_add_fence(struct fd_bo *bo, struct fd_pipe *pipe, uint32_t fence);
375
376 enum fd_bo_state {
377 FD_BO_STATE_IDLE,
378 FD_BO_STATE_BUSY,
379 FD_BO_STATE_UNKNOWN,
380 };
381 enum fd_bo_state fd_bo_state(struct fd_bo *bo);
382
383 void fd_bo_init_common(struct fd_bo *bo, struct fd_device *dev);
384
385 struct fd_bo *fd_bo_new_ring(struct fd_device *dev, uint32_t size);
386
387 #define enable_debug 0 /* TODO make dynamic */
388
389 bool fd_dbg(void);
390
391 #define INFO_MSG(fmt, ...) \
392 do { \
393 if (fd_dbg()) \
394 mesa_logi("%s:%d: " fmt, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
395 } while (0)
396 #define DEBUG_MSG(fmt, ...) \
397 do \
398 if (enable_debug) { \
399 mesa_logd("%s:%d: " fmt, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
400 } \
401 while (0)
402 #define WARN_MSG(fmt, ...) \
403 do { \
404 mesa_logw("%s:%d: " fmt, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
405 } while (0)
406 #define ERROR_MSG(fmt, ...) \
407 do { \
408 mesa_loge("%s:%d: " fmt, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
409 } while (0)
410
411 #define U642VOID(x) ((void *)(unsigned long)(x))
412 #define VOID2U64(x) ((uint64_t)(unsigned long)(x))
413
414 #if HAVE_VALGRIND
415 #include <memcheck.h>
416
417 /*
418 * For tracking the backing memory (if valgrind enabled, we force a mmap
419 * for the purposes of tracking)
420 */
421 static inline void
VG_BO_ALLOC(struct fd_bo * bo)422 VG_BO_ALLOC(struct fd_bo *bo)
423 {
424 if (bo && RUNNING_ON_VALGRIND) {
425 VALGRIND_MALLOCLIKE_BLOCK(fd_bo_map(bo), bo->size, 0, 1);
426 }
427 }
428
429 static inline void
VG_BO_FREE(struct fd_bo * bo)430 VG_BO_FREE(struct fd_bo *bo)
431 {
432 VALGRIND_FREELIKE_BLOCK(bo->map, 0);
433 }
434
435 /*
436 * For tracking bo structs that are in the buffer-cache, so that valgrind
437 * doesn't attribute ownership to the first one to allocate the recycled
438 * bo.
439 *
440 * Note that the list_head in fd_bo is used to track the buffers in cache
441 * so disable error reporting on the range while they are in cache so
442 * valgrind doesn't squawk about list traversal.
443 *
444 */
445 static inline void
VG_BO_RELEASE(struct fd_bo * bo)446 VG_BO_RELEASE(struct fd_bo *bo)
447 {
448 if (RUNNING_ON_VALGRIND) {
449 VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(bo, bo->dev->bo_size);
450 VALGRIND_MAKE_MEM_NOACCESS(bo, bo->dev->bo_size);
451 VALGRIND_FREELIKE_BLOCK(bo->map, 0);
452 }
453 }
454 static inline void
VG_BO_OBTAIN(struct fd_bo * bo)455 VG_BO_OBTAIN(struct fd_bo *bo)
456 {
457 if (RUNNING_ON_VALGRIND) {
458 VALGRIND_MAKE_MEM_DEFINED(bo, bo->dev->bo_size);
459 VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(bo, bo->dev->bo_size);
460 VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, 1);
461 }
462 }
463 #else
464 static inline void
VG_BO_ALLOC(struct fd_bo * bo)465 VG_BO_ALLOC(struct fd_bo *bo)
466 {
467 }
468 static inline void
VG_BO_FREE(struct fd_bo * bo)469 VG_BO_FREE(struct fd_bo *bo)
470 {
471 }
472 static inline void
VG_BO_RELEASE(struct fd_bo * bo)473 VG_BO_RELEASE(struct fd_bo *bo)
474 {
475 }
476 static inline void
VG_BO_OBTAIN(struct fd_bo * bo)477 VG_BO_OBTAIN(struct fd_bo *bo)
478 {
479 }
480 #endif
481
482 #define FD_DEFINE_CAST(parent, child) \
483 static inline struct child *to_##child(struct parent *x) \
484 { \
485 return (struct child *)x; \
486 }
487
488 #endif /* FREEDRENO_PRIV_H_ */
489