1 /*
2 * Copyright © 2014-2017 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <errno.h>
25 #include <err.h>
26 #include <sys/mman.h>
27 #include <fcntl.h>
28 #include <xf86drm.h>
29 #include <xf86drmMode.h>
30
31 #include "util/u_hash_table.h"
32 #include "util/u_memory.h"
33 #include "util/ralloc.h"
34
35 #include "vc5_context.h"
36 #include "vc5_screen.h"
37
38 #ifdef HAVE_VALGRIND
39 #include <valgrind.h>
40 #include <memcheck.h>
41 #define VG(x) x
42 #else
43 #define VG(x)
44 #endif
45
46 static bool dump_stats = false;
47
48 static void
49 vc5_bo_cache_free_all(struct vc5_bo_cache *cache);
50
51 static void
vc5_bo_dump_stats(struct vc5_screen * screen)52 vc5_bo_dump_stats(struct vc5_screen *screen)
53 {
54 struct vc5_bo_cache *cache = &screen->bo_cache;
55
56 fprintf(stderr, " BOs allocated: %d\n", screen->bo_count);
57 fprintf(stderr, " BOs size: %dkb\n", screen->bo_size / 1024);
58 fprintf(stderr, " BOs cached: %d\n", cache->bo_count);
59 fprintf(stderr, " BOs cached size: %dkb\n", cache->bo_size / 1024);
60
61 if (!list_empty(&cache->time_list)) {
62 struct vc5_bo *first = LIST_ENTRY(struct vc5_bo,
63 cache->time_list.next,
64 time_list);
65 struct vc5_bo *last = LIST_ENTRY(struct vc5_bo,
66 cache->time_list.prev,
67 time_list);
68
69 fprintf(stderr, " oldest cache time: %ld\n",
70 (long)first->free_time);
71 fprintf(stderr, " newest cache time: %ld\n",
72 (long)last->free_time);
73
74 struct timespec time;
75 clock_gettime(CLOCK_MONOTONIC, &time);
76 fprintf(stderr, " now: %ld\n",
77 time.tv_sec);
78 }
79 }
80
81 static void
vc5_bo_remove_from_cache(struct vc5_bo_cache * cache,struct vc5_bo * bo)82 vc5_bo_remove_from_cache(struct vc5_bo_cache *cache, struct vc5_bo *bo)
83 {
84 list_del(&bo->time_list);
85 list_del(&bo->size_list);
86 cache->bo_count--;
87 cache->bo_size -= bo->size;
88 }
89
90 static struct vc5_bo *
vc5_bo_from_cache(struct vc5_screen * screen,uint32_t size,const char * name)91 vc5_bo_from_cache(struct vc5_screen *screen, uint32_t size, const char *name)
92 {
93 struct vc5_bo_cache *cache = &screen->bo_cache;
94 uint32_t page_index = size / 4096 - 1;
95
96 if (cache->size_list_size <= page_index)
97 return NULL;
98
99 struct vc5_bo *bo = NULL;
100 mtx_lock(&cache->lock);
101 if (!list_empty(&cache->size_list[page_index])) {
102 bo = LIST_ENTRY(struct vc5_bo, cache->size_list[page_index].next,
103 size_list);
104
105 /* Check that the BO has gone idle. If not, then we want to
106 * allocate something new instead, since we assume that the
107 * user will proceed to CPU map it and fill it with stuff.
108 */
109 if (!vc5_bo_wait(bo, 0, NULL)) {
110 mtx_unlock(&cache->lock);
111 return NULL;
112 }
113
114 pipe_reference_init(&bo->reference, 1);
115 vc5_bo_remove_from_cache(cache, bo);
116
117 bo->name = name;
118 }
119 mtx_unlock(&cache->lock);
120 return bo;
121 }
122
123 struct vc5_bo *
vc5_bo_alloc(struct vc5_screen * screen,uint32_t size,const char * name)124 vc5_bo_alloc(struct vc5_screen *screen, uint32_t size, const char *name)
125 {
126 struct vc5_bo *bo;
127 int ret;
128
129 size = align(size, 4096);
130
131 bo = vc5_bo_from_cache(screen, size, name);
132 if (bo) {
133 if (dump_stats) {
134 fprintf(stderr, "Allocated %s %dkb from cache:\n",
135 name, size / 1024);
136 vc5_bo_dump_stats(screen);
137 }
138 return bo;
139 }
140
141 bo = CALLOC_STRUCT(vc5_bo);
142 if (!bo)
143 return NULL;
144
145 pipe_reference_init(&bo->reference, 1);
146 bo->screen = screen;
147 bo->size = size;
148 bo->name = name;
149 bo->private = true;
150
151 retry:
152 ;
153
154 bool cleared_and_retried = false;
155 struct drm_vc5_create_bo create = {
156 .size = size
157 };
158
159 ret = vc5_ioctl(screen->fd, DRM_IOCTL_VC5_CREATE_BO, &create);
160 bo->handle = create.handle;
161 bo->offset = create.offset;
162
163 if (ret != 0) {
164 if (!list_empty(&screen->bo_cache.time_list) &&
165 !cleared_and_retried) {
166 cleared_and_retried = true;
167 vc5_bo_cache_free_all(&screen->bo_cache);
168 goto retry;
169 }
170
171 free(bo);
172 return NULL;
173 }
174
175 screen->bo_count++;
176 screen->bo_size += bo->size;
177 if (dump_stats) {
178 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
179 vc5_bo_dump_stats(screen);
180 }
181
182 return bo;
183 }
184
185 void
vc5_bo_last_unreference(struct vc5_bo * bo)186 vc5_bo_last_unreference(struct vc5_bo *bo)
187 {
188 struct vc5_screen *screen = bo->screen;
189
190 struct timespec time;
191 clock_gettime(CLOCK_MONOTONIC, &time);
192 mtx_lock(&screen->bo_cache.lock);
193 vc5_bo_last_unreference_locked_timed(bo, time.tv_sec);
194 mtx_unlock(&screen->bo_cache.lock);
195 }
196
197 static void
vc5_bo_free(struct vc5_bo * bo)198 vc5_bo_free(struct vc5_bo *bo)
199 {
200 struct vc5_screen *screen = bo->screen;
201
202 if (bo->map) {
203 if (using_vc5_simulator && bo->name &&
204 strcmp(bo->name, "winsys") == 0) {
205 free(bo->map);
206 } else {
207 munmap(bo->map, bo->size);
208 VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
209 }
210 }
211
212 struct drm_gem_close c;
213 memset(&c, 0, sizeof(c));
214 c.handle = bo->handle;
215 int ret = vc5_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
216 if (ret != 0)
217 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
218
219 screen->bo_count--;
220 screen->bo_size -= bo->size;
221
222 if (dump_stats) {
223 fprintf(stderr, "Freed %s%s%dkb:\n",
224 bo->name ? bo->name : "",
225 bo->name ? " " : "",
226 bo->size / 1024);
227 vc5_bo_dump_stats(screen);
228 }
229
230 free(bo);
231 }
232
233 static void
free_stale_bos(struct vc5_screen * screen,time_t time)234 free_stale_bos(struct vc5_screen *screen, time_t time)
235 {
236 struct vc5_bo_cache *cache = &screen->bo_cache;
237 bool freed_any = false;
238
239 list_for_each_entry_safe(struct vc5_bo, bo, &cache->time_list,
240 time_list) {
241 if (dump_stats && !freed_any) {
242 fprintf(stderr, "Freeing stale BOs:\n");
243 vc5_bo_dump_stats(screen);
244 freed_any = true;
245 }
246
247 /* If it's more than a second old, free it. */
248 if (time - bo->free_time > 2) {
249 vc5_bo_remove_from_cache(cache, bo);
250 vc5_bo_free(bo);
251 } else {
252 break;
253 }
254 }
255
256 if (dump_stats && freed_any) {
257 fprintf(stderr, "Freed stale BOs:\n");
258 vc5_bo_dump_stats(screen);
259 }
260 }
261
262 static void
vc5_bo_cache_free_all(struct vc5_bo_cache * cache)263 vc5_bo_cache_free_all(struct vc5_bo_cache *cache)
264 {
265 mtx_lock(&cache->lock);
266 list_for_each_entry_safe(struct vc5_bo, bo, &cache->time_list,
267 time_list) {
268 vc5_bo_remove_from_cache(cache, bo);
269 vc5_bo_free(bo);
270 }
271 mtx_unlock(&cache->lock);
272 }
273
274 void
vc5_bo_last_unreference_locked_timed(struct vc5_bo * bo,time_t time)275 vc5_bo_last_unreference_locked_timed(struct vc5_bo *bo, time_t time)
276 {
277 struct vc5_screen *screen = bo->screen;
278 struct vc5_bo_cache *cache = &screen->bo_cache;
279 uint32_t page_index = bo->size / 4096 - 1;
280
281 if (!bo->private) {
282 vc5_bo_free(bo);
283 return;
284 }
285
286 if (cache->size_list_size <= page_index) {
287 struct list_head *new_list =
288 ralloc_array(screen, struct list_head, page_index + 1);
289
290 /* Move old list contents over (since the array has moved, and
291 * therefore the pointers to the list heads have to change).
292 */
293 for (int i = 0; i < cache->size_list_size; i++) {
294 struct list_head *old_head = &cache->size_list[i];
295 if (list_empty(old_head))
296 list_inithead(&new_list[i]);
297 else {
298 new_list[i].next = old_head->next;
299 new_list[i].prev = old_head->prev;
300 new_list[i].next->prev = &new_list[i];
301 new_list[i].prev->next = &new_list[i];
302 }
303 }
304 for (int i = cache->size_list_size; i < page_index + 1; i++)
305 list_inithead(&new_list[i]);
306
307 cache->size_list = new_list;
308 cache->size_list_size = page_index + 1;
309 }
310
311 bo->free_time = time;
312 list_addtail(&bo->size_list, &cache->size_list[page_index]);
313 list_addtail(&bo->time_list, &cache->time_list);
314 cache->bo_count++;
315 cache->bo_size += bo->size;
316 if (dump_stats) {
317 fprintf(stderr, "Freed %s %dkb to cache:\n",
318 bo->name, bo->size / 1024);
319 vc5_bo_dump_stats(screen);
320 }
321 bo->name = NULL;
322
323 free_stale_bos(screen, time);
324 }
325
326 static struct vc5_bo *
vc5_bo_open_handle(struct vc5_screen * screen,uint32_t winsys_stride,uint32_t handle,uint32_t size)327 vc5_bo_open_handle(struct vc5_screen *screen,
328 uint32_t winsys_stride,
329 uint32_t handle, uint32_t size)
330 {
331 struct vc5_bo *bo;
332
333 assert(size);
334
335 mtx_lock(&screen->bo_handles_mutex);
336
337 bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
338 if (bo) {
339 pipe_reference(NULL, &bo->reference);
340 goto done;
341 }
342
343 bo = CALLOC_STRUCT(vc5_bo);
344 pipe_reference_init(&bo->reference, 1);
345 bo->screen = screen;
346 bo->handle = handle;
347 bo->size = size;
348 bo->name = "winsys";
349 bo->private = false;
350
351 #ifdef USE_VC5_SIMULATOR
352 vc5_simulator_open_from_handle(screen->fd, winsys_stride,
353 bo->handle, bo->size);
354 bo->map = malloc(bo->size);
355 #endif
356
357 util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo);
358
359 done:
360 mtx_unlock(&screen->bo_handles_mutex);
361 return bo;
362 }
363
364 struct vc5_bo *
vc5_bo_open_name(struct vc5_screen * screen,uint32_t name,uint32_t winsys_stride)365 vc5_bo_open_name(struct vc5_screen *screen, uint32_t name,
366 uint32_t winsys_stride)
367 {
368 struct drm_gem_open o = {
369 .name = name
370 };
371 int ret = vc5_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
372 if (ret) {
373 fprintf(stderr, "Failed to open bo %d: %s\n",
374 name, strerror(errno));
375 return NULL;
376 }
377
378 return vc5_bo_open_handle(screen, winsys_stride, o.handle, o.size);
379 }
380
381 struct vc5_bo *
vc5_bo_open_dmabuf(struct vc5_screen * screen,int fd,uint32_t winsys_stride)382 vc5_bo_open_dmabuf(struct vc5_screen *screen, int fd, uint32_t winsys_stride)
383 {
384 uint32_t handle;
385 int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
386 int size;
387 if (ret) {
388 fprintf(stderr, "Failed to get vc5 handle for dmabuf %d\n", fd);
389 return NULL;
390 }
391
392 /* Determine the size of the bo we were handed. */
393 size = lseek(fd, 0, SEEK_END);
394 if (size == -1) {
395 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
396 return NULL;
397 }
398
399 return vc5_bo_open_handle(screen, winsys_stride, handle, size);
400 }
401
402 int
vc5_bo_get_dmabuf(struct vc5_bo * bo)403 vc5_bo_get_dmabuf(struct vc5_bo *bo)
404 {
405 int fd;
406 int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
407 O_CLOEXEC, &fd);
408 if (ret != 0) {
409 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
410 bo->handle);
411 return -1;
412 }
413
414 mtx_lock(&bo->screen->bo_handles_mutex);
415 bo->private = false;
416 util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
417 mtx_unlock(&bo->screen->bo_handles_mutex);
418
419 return fd;
420 }
421
422 bool
vc5_bo_flink(struct vc5_bo * bo,uint32_t * name)423 vc5_bo_flink(struct vc5_bo *bo, uint32_t *name)
424 {
425 struct drm_gem_flink flink = {
426 .handle = bo->handle,
427 };
428 int ret = vc5_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
429 if (ret) {
430 fprintf(stderr, "Failed to flink bo %d: %s\n",
431 bo->handle, strerror(errno));
432 free(bo);
433 return false;
434 }
435
436 bo->private = false;
437 *name = flink.name;
438
439 return true;
440 }
441
vc5_wait_seqno_ioctl(int fd,uint64_t seqno,uint64_t timeout_ns)442 static int vc5_wait_seqno_ioctl(int fd, uint64_t seqno, uint64_t timeout_ns)
443 {
444 struct drm_vc5_wait_seqno wait = {
445 .seqno = seqno,
446 .timeout_ns = timeout_ns,
447 };
448 int ret = vc5_ioctl(fd, DRM_IOCTL_VC5_WAIT_SEQNO, &wait);
449 if (ret == -1)
450 return -errno;
451 else
452 return 0;
453
454 }
455
456 bool
vc5_wait_seqno(struct vc5_screen * screen,uint64_t seqno,uint64_t timeout_ns,const char * reason)457 vc5_wait_seqno(struct vc5_screen *screen, uint64_t seqno, uint64_t timeout_ns,
458 const char *reason)
459 {
460 if (screen->finished_seqno >= seqno)
461 return true;
462
463 if (unlikely(V3D_DEBUG & V3D_DEBUG_PERF) && timeout_ns && reason) {
464 if (vc5_wait_seqno_ioctl(screen->fd, seqno, 0) == -ETIME) {
465 fprintf(stderr, "Blocking on seqno %lld for %s\n",
466 (long long)seqno, reason);
467 }
468 }
469
470 int ret = vc5_wait_seqno_ioctl(screen->fd, seqno, timeout_ns);
471 if (ret) {
472 if (ret != -ETIME) {
473 fprintf(stderr, "wait failed: %d\n", ret);
474 abort();
475 }
476
477 return false;
478 }
479
480 screen->finished_seqno = seqno;
481 return true;
482 }
483
vc5_wait_bo_ioctl(int fd,uint32_t handle,uint64_t timeout_ns)484 static int vc5_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
485 {
486 struct drm_vc5_wait_bo wait = {
487 .handle = handle,
488 .timeout_ns = timeout_ns,
489 };
490 int ret = vc5_ioctl(fd, DRM_IOCTL_VC5_WAIT_BO, &wait);
491 if (ret == -1)
492 return -errno;
493 else
494 return 0;
495
496 }
497
498 bool
vc5_bo_wait(struct vc5_bo * bo,uint64_t timeout_ns,const char * reason)499 vc5_bo_wait(struct vc5_bo *bo, uint64_t timeout_ns, const char *reason)
500 {
501 struct vc5_screen *screen = bo->screen;
502
503 if (unlikely(V3D_DEBUG & V3D_DEBUG_PERF) && timeout_ns && reason) {
504 if (vc5_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
505 fprintf(stderr, "Blocking on %s BO for %s\n",
506 bo->name, reason);
507 }
508 }
509
510 int ret = vc5_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
511 if (ret) {
512 if (ret != -ETIME) {
513 fprintf(stderr, "wait failed: %d\n", ret);
514 abort();
515 }
516
517 return false;
518 }
519
520 return true;
521 }
522
523 void *
vc5_bo_map_unsynchronized(struct vc5_bo * bo)524 vc5_bo_map_unsynchronized(struct vc5_bo *bo)
525 {
526 uint64_t offset;
527 int ret;
528
529 if (bo->map)
530 return bo->map;
531
532 struct drm_vc5_mmap_bo map;
533 memset(&map, 0, sizeof(map));
534 map.handle = bo->handle;
535 ret = vc5_ioctl(bo->screen->fd, DRM_IOCTL_VC5_MMAP_BO, &map);
536 offset = map.offset;
537 if (ret != 0) {
538 fprintf(stderr, "map ioctl failure\n");
539 abort();
540 }
541
542 bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
543 bo->screen->fd, offset);
544 if (bo->map == MAP_FAILED) {
545 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
546 bo->handle, (long long)offset, bo->size);
547 abort();
548 }
549 VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
550
551 return bo->map;
552 }
553
554 void *
vc5_bo_map(struct vc5_bo * bo)555 vc5_bo_map(struct vc5_bo *bo)
556 {
557 void *map = vc5_bo_map_unsynchronized(bo);
558
559 bool ok = vc5_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
560 if (!ok) {
561 fprintf(stderr, "BO wait for map failed\n");
562 abort();
563 }
564
565 return map;
566 }
567
568 void
vc5_bufmgr_destroy(struct pipe_screen * pscreen)569 vc5_bufmgr_destroy(struct pipe_screen *pscreen)
570 {
571 struct vc5_screen *screen = vc5_screen(pscreen);
572 struct vc5_bo_cache *cache = &screen->bo_cache;
573
574 vc5_bo_cache_free_all(cache);
575
576 if (dump_stats) {
577 fprintf(stderr, "BO stats after screen destroy:\n");
578 vc5_bo_dump_stats(screen);
579 }
580 }
581