1 /*
2 * Copyright © 2014-2015 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <errno.h>
25 #include <err.h>
26 #include <sys/mman.h>
27 #include <fcntl.h>
28 #include <xf86drm.h>
29 #include <xf86drmMode.h>
30
31 #include "util/u_hash_table.h"
32 #include "util/u_memory.h"
33 #include "util/ralloc.h"
34
35 #include "vc4_context.h"
36 #include "vc4_screen.h"
37
38 #ifdef HAVE_VALGRIND
39 #include <valgrind.h>
40 #include <memcheck.h>
41 #define VG(x) x
42 #else
43 #define VG(x)
44 #endif
45
46 static bool dump_stats = false;
47
48 static void
49 vc4_bo_cache_free_all(struct vc4_bo_cache *cache);
50
51 static void
vc4_bo_dump_stats(struct vc4_screen * screen)52 vc4_bo_dump_stats(struct vc4_screen *screen)
53 {
54 struct vc4_bo_cache *cache = &screen->bo_cache;
55
56 fprintf(stderr, " BOs allocated: %d\n", screen->bo_count);
57 fprintf(stderr, " BOs size: %dkb\n", screen->bo_size / 1024);
58 fprintf(stderr, " BOs cached: %d\n", cache->bo_count);
59 fprintf(stderr, " BOs cached size: %dkb\n", cache->bo_size / 1024);
60
61 if (!list_empty(&cache->time_list)) {
62 struct vc4_bo *first = LIST_ENTRY(struct vc4_bo,
63 cache->time_list.next,
64 time_list);
65 struct vc4_bo *last = LIST_ENTRY(struct vc4_bo,
66 cache->time_list.prev,
67 time_list);
68
69 fprintf(stderr, " oldest cache time: %ld\n",
70 (long)first->free_time);
71 fprintf(stderr, " newest cache time: %ld\n",
72 (long)last->free_time);
73
74 struct timespec time;
75 clock_gettime(CLOCK_MONOTONIC, &time);
76 fprintf(stderr, " now: %ld\n",
77 time.tv_sec);
78 }
79 }
80
81 static void
vc4_bo_remove_from_cache(struct vc4_bo_cache * cache,struct vc4_bo * bo)82 vc4_bo_remove_from_cache(struct vc4_bo_cache *cache, struct vc4_bo *bo)
83 {
84 list_del(&bo->time_list);
85 list_del(&bo->size_list);
86 cache->bo_count--;
87 cache->bo_size -= bo->size;
88 }
89
90 static struct vc4_bo *
vc4_bo_from_cache(struct vc4_screen * screen,uint32_t size,const char * name)91 vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name)
92 {
93 struct vc4_bo_cache *cache = &screen->bo_cache;
94 uint32_t page_index = size / 4096 - 1;
95
96 if (cache->size_list_size <= page_index)
97 return NULL;
98
99 struct vc4_bo *bo = NULL;
100 pipe_mutex_lock(cache->lock);
101 if (!list_empty(&cache->size_list[page_index])) {
102 bo = LIST_ENTRY(struct vc4_bo, cache->size_list[page_index].next,
103 size_list);
104
105 /* Check that the BO has gone idle. If not, then we want to
106 * allocate something new instead, since we assume that the
107 * user will proceed to CPU map it and fill it with stuff.
108 */
109 if (!vc4_bo_wait(bo, 0, NULL)) {
110 pipe_mutex_unlock(cache->lock);
111 return NULL;
112 }
113
114 pipe_reference_init(&bo->reference, 1);
115 vc4_bo_remove_from_cache(cache, bo);
116
117 bo->name = name;
118 }
119 pipe_mutex_unlock(cache->lock);
120 return bo;
121 }
122
123 struct vc4_bo *
vc4_bo_alloc(struct vc4_screen * screen,uint32_t size,const char * name)124 vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name)
125 {
126 struct vc4_bo *bo;
127 int ret;
128
129 size = align(size, 4096);
130
131 bo = vc4_bo_from_cache(screen, size, name);
132 if (bo) {
133 if (dump_stats) {
134 fprintf(stderr, "Allocated %s %dkb from cache:\n",
135 name, size / 1024);
136 vc4_bo_dump_stats(screen);
137 }
138 return bo;
139 }
140
141 bo = CALLOC_STRUCT(vc4_bo);
142 if (!bo)
143 return NULL;
144
145 pipe_reference_init(&bo->reference, 1);
146 bo->screen = screen;
147 bo->size = size;
148 bo->name = name;
149 bo->private = true;
150
151 retry:
152 ;
153
154 bool cleared_and_retried = false;
155 struct drm_vc4_create_bo create = {
156 .size = size
157 };
158
159 ret = vc4_ioctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create);
160 bo->handle = create.handle;
161
162 if (ret != 0) {
163 if (!list_empty(&screen->bo_cache.time_list) &&
164 !cleared_and_retried) {
165 cleared_and_retried = true;
166 vc4_bo_cache_free_all(&screen->bo_cache);
167 goto retry;
168 }
169
170 free(bo);
171 return NULL;
172 }
173
174 screen->bo_count++;
175 screen->bo_size += bo->size;
176 if (dump_stats) {
177 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
178 vc4_bo_dump_stats(screen);
179 }
180
181 return bo;
182 }
183
184 void
vc4_bo_last_unreference(struct vc4_bo * bo)185 vc4_bo_last_unreference(struct vc4_bo *bo)
186 {
187 struct vc4_screen *screen = bo->screen;
188
189 struct timespec time;
190 clock_gettime(CLOCK_MONOTONIC, &time);
191 pipe_mutex_lock(screen->bo_cache.lock);
192 vc4_bo_last_unreference_locked_timed(bo, time.tv_sec);
193 pipe_mutex_unlock(screen->bo_cache.lock);
194 }
195
196 static void
vc4_bo_free(struct vc4_bo * bo)197 vc4_bo_free(struct vc4_bo *bo)
198 {
199 struct vc4_screen *screen = bo->screen;
200
201 if (bo->map) {
202 if (using_vc4_simulator && bo->name &&
203 strcmp(bo->name, "winsys") == 0) {
204 free(bo->map);
205 } else {
206 munmap(bo->map, bo->size);
207 VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
208 }
209 }
210
211 struct drm_gem_close c;
212 memset(&c, 0, sizeof(c));
213 c.handle = bo->handle;
214 int ret = vc4_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
215 if (ret != 0)
216 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
217
218 screen->bo_count--;
219 screen->bo_size -= bo->size;
220
221 if (dump_stats) {
222 fprintf(stderr, "Freed %s%s%dkb:\n",
223 bo->name ? bo->name : "",
224 bo->name ? " " : "",
225 bo->size / 1024);
226 vc4_bo_dump_stats(screen);
227 }
228
229 free(bo);
230 }
231
232 static void
free_stale_bos(struct vc4_screen * screen,time_t time)233 free_stale_bos(struct vc4_screen *screen, time_t time)
234 {
235 struct vc4_bo_cache *cache = &screen->bo_cache;
236 bool freed_any = false;
237
238 list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
239 time_list) {
240 if (dump_stats && !freed_any) {
241 fprintf(stderr, "Freeing stale BOs:\n");
242 vc4_bo_dump_stats(screen);
243 freed_any = true;
244 }
245
246 /* If it's more than a second old, free it. */
247 if (time - bo->free_time > 2) {
248 vc4_bo_remove_from_cache(cache, bo);
249 vc4_bo_free(bo);
250 } else {
251 break;
252 }
253 }
254
255 if (dump_stats && freed_any) {
256 fprintf(stderr, "Freed stale BOs:\n");
257 vc4_bo_dump_stats(screen);
258 }
259 }
260
261 static void
vc4_bo_cache_free_all(struct vc4_bo_cache * cache)262 vc4_bo_cache_free_all(struct vc4_bo_cache *cache)
263 {
264 pipe_mutex_lock(cache->lock);
265 list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
266 time_list) {
267 vc4_bo_remove_from_cache(cache, bo);
268 vc4_bo_free(bo);
269 }
270 pipe_mutex_unlock(cache->lock);
271 }
272
273 void
vc4_bo_last_unreference_locked_timed(struct vc4_bo * bo,time_t time)274 vc4_bo_last_unreference_locked_timed(struct vc4_bo *bo, time_t time)
275 {
276 struct vc4_screen *screen = bo->screen;
277 struct vc4_bo_cache *cache = &screen->bo_cache;
278 uint32_t page_index = bo->size / 4096 - 1;
279
280 if (!bo->private) {
281 vc4_bo_free(bo);
282 return;
283 }
284
285 if (cache->size_list_size <= page_index) {
286 struct list_head *new_list =
287 ralloc_array(screen, struct list_head, page_index + 1);
288
289 /* Move old list contents over (since the array has moved, and
290 * therefore the pointers to the list heads have to change).
291 */
292 for (int i = 0; i < cache->size_list_size; i++)
293 list_replace(&cache->size_list[i], &new_list[i]);
294 for (int i = cache->size_list_size; i < page_index + 1; i++)
295 list_inithead(&new_list[i]);
296
297 cache->size_list = new_list;
298 cache->size_list_size = page_index + 1;
299 }
300
301 bo->free_time = time;
302 list_addtail(&bo->size_list, &cache->size_list[page_index]);
303 list_addtail(&bo->time_list, &cache->time_list);
304 cache->bo_count++;
305 cache->bo_size += bo->size;
306 if (dump_stats) {
307 fprintf(stderr, "Freed %s %dkb to cache:\n",
308 bo->name, bo->size / 1024);
309 vc4_bo_dump_stats(screen);
310 }
311 bo->name = NULL;
312
313 free_stale_bos(screen, time);
314 }
315
316 static struct vc4_bo *
vc4_bo_open_handle(struct vc4_screen * screen,uint32_t winsys_stride,uint32_t handle,uint32_t size)317 vc4_bo_open_handle(struct vc4_screen *screen,
318 uint32_t winsys_stride,
319 uint32_t handle, uint32_t size)
320 {
321 struct vc4_bo *bo;
322
323 assert(size);
324
325 pipe_mutex_lock(screen->bo_handles_mutex);
326
327 bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
328 if (bo) {
329 pipe_reference(NULL, &bo->reference);
330 goto done;
331 }
332
333 bo = CALLOC_STRUCT(vc4_bo);
334 pipe_reference_init(&bo->reference, 1);
335 bo->screen = screen;
336 bo->handle = handle;
337 bo->size = size;
338 bo->name = "winsys";
339 bo->private = false;
340
341 #ifdef USE_VC4_SIMULATOR
342 vc4_simulator_open_from_handle(screen->fd, winsys_stride,
343 bo->handle, bo->size);
344 bo->map = malloc(bo->size);
345 #endif
346
347 util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo);
348
349 done:
350 pipe_mutex_unlock(screen->bo_handles_mutex);
351 return bo;
352 }
353
354 struct vc4_bo *
vc4_bo_open_name(struct vc4_screen * screen,uint32_t name,uint32_t winsys_stride)355 vc4_bo_open_name(struct vc4_screen *screen, uint32_t name,
356 uint32_t winsys_stride)
357 {
358 struct drm_gem_open o = {
359 .name = name
360 };
361 int ret = vc4_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
362 if (ret) {
363 fprintf(stderr, "Failed to open bo %d: %s\n",
364 name, strerror(errno));
365 return NULL;
366 }
367
368 return vc4_bo_open_handle(screen, winsys_stride, o.handle, o.size);
369 }
370
371 struct vc4_bo *
vc4_bo_open_dmabuf(struct vc4_screen * screen,int fd,uint32_t winsys_stride)372 vc4_bo_open_dmabuf(struct vc4_screen *screen, int fd, uint32_t winsys_stride)
373 {
374 uint32_t handle;
375 int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
376 int size;
377 if (ret) {
378 fprintf(stderr, "Failed to get vc4 handle for dmabuf %d\n", fd);
379 return NULL;
380 }
381
382 /* Determine the size of the bo we were handed. */
383 size = lseek(fd, 0, SEEK_END);
384 if (size == -1) {
385 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
386 return NULL;
387 }
388
389 return vc4_bo_open_handle(screen, winsys_stride, handle, size);
390 }
391
392 int
vc4_bo_get_dmabuf(struct vc4_bo * bo)393 vc4_bo_get_dmabuf(struct vc4_bo *bo)
394 {
395 int fd;
396 int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
397 O_CLOEXEC, &fd);
398 if (ret != 0) {
399 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
400 bo->handle);
401 return -1;
402 }
403
404 pipe_mutex_lock(bo->screen->bo_handles_mutex);
405 bo->private = false;
406 util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
407 pipe_mutex_unlock(bo->screen->bo_handles_mutex);
408
409 return fd;
410 }
411
412 struct vc4_bo *
vc4_bo_alloc_shader(struct vc4_screen * screen,const void * data,uint32_t size)413 vc4_bo_alloc_shader(struct vc4_screen *screen, const void *data, uint32_t size)
414 {
415 struct vc4_bo *bo;
416 int ret;
417
418 bo = CALLOC_STRUCT(vc4_bo);
419 if (!bo)
420 return NULL;
421
422 pipe_reference_init(&bo->reference, 1);
423 bo->screen = screen;
424 bo->size = align(size, 4096);
425 bo->name = "code";
426 bo->private = false; /* Make sure it doesn't go back to the cache. */
427
428 struct drm_vc4_create_shader_bo create = {
429 .size = size,
430 .data = (uintptr_t)data,
431 };
432
433 ret = vc4_ioctl(screen->fd, DRM_IOCTL_VC4_CREATE_SHADER_BO,
434 &create);
435 bo->handle = create.handle;
436
437 if (ret != 0) {
438 fprintf(stderr, "create shader ioctl failure\n");
439 abort();
440 }
441
442 screen->bo_count++;
443 screen->bo_size += bo->size;
444 if (dump_stats) {
445 fprintf(stderr, "Allocated shader %dkb:\n", bo->size / 1024);
446 vc4_bo_dump_stats(screen);
447 }
448
449 return bo;
450 }
451
452 bool
vc4_bo_flink(struct vc4_bo * bo,uint32_t * name)453 vc4_bo_flink(struct vc4_bo *bo, uint32_t *name)
454 {
455 struct drm_gem_flink flink = {
456 .handle = bo->handle,
457 };
458 int ret = vc4_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
459 if (ret) {
460 fprintf(stderr, "Failed to flink bo %d: %s\n",
461 bo->handle, strerror(errno));
462 free(bo);
463 return false;
464 }
465
466 bo->private = false;
467 *name = flink.name;
468
469 return true;
470 }
471
vc4_wait_seqno_ioctl(int fd,uint64_t seqno,uint64_t timeout_ns)472 static int vc4_wait_seqno_ioctl(int fd, uint64_t seqno, uint64_t timeout_ns)
473 {
474 struct drm_vc4_wait_seqno wait = {
475 .seqno = seqno,
476 .timeout_ns = timeout_ns,
477 };
478 int ret = vc4_ioctl(fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait);
479 if (ret == -1)
480 return -errno;
481 else
482 return 0;
483
484 }
485
486 bool
vc4_wait_seqno(struct vc4_screen * screen,uint64_t seqno,uint64_t timeout_ns,const char * reason)487 vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns,
488 const char *reason)
489 {
490 if (screen->finished_seqno >= seqno)
491 return true;
492
493 if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
494 if (vc4_wait_seqno_ioctl(screen->fd, seqno, 0) == -ETIME) {
495 fprintf(stderr, "Blocking on seqno %lld for %s\n",
496 (long long)seqno, reason);
497 }
498 }
499
500 int ret = vc4_wait_seqno_ioctl(screen->fd, seqno, timeout_ns);
501 if (ret) {
502 if (ret != -ETIME) {
503 fprintf(stderr, "wait failed: %d\n", ret);
504 abort();
505 }
506
507 return false;
508 }
509
510 screen->finished_seqno = seqno;
511 return true;
512 }
513
vc4_wait_bo_ioctl(int fd,uint32_t handle,uint64_t timeout_ns)514 static int vc4_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
515 {
516 struct drm_vc4_wait_bo wait = {
517 .handle = handle,
518 .timeout_ns = timeout_ns,
519 };
520 int ret = vc4_ioctl(fd, DRM_IOCTL_VC4_WAIT_BO, &wait);
521 if (ret == -1)
522 return -errno;
523 else
524 return 0;
525
526 }
527
528 bool
vc4_bo_wait(struct vc4_bo * bo,uint64_t timeout_ns,const char * reason)529 vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns, const char *reason)
530 {
531 struct vc4_screen *screen = bo->screen;
532
533 if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
534 if (vc4_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
535 fprintf(stderr, "Blocking on %s BO for %s\n",
536 bo->name, reason);
537 }
538 }
539
540 int ret = vc4_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
541 if (ret) {
542 if (ret != -ETIME) {
543 fprintf(stderr, "wait failed: %d\n", ret);
544 abort();
545 }
546
547 return false;
548 }
549
550 return true;
551 }
552
553 void *
vc4_bo_map_unsynchronized(struct vc4_bo * bo)554 vc4_bo_map_unsynchronized(struct vc4_bo *bo)
555 {
556 uint64_t offset;
557 int ret;
558
559 if (bo->map)
560 return bo->map;
561
562 struct drm_vc4_mmap_bo map;
563 memset(&map, 0, sizeof(map));
564 map.handle = bo->handle;
565 ret = vc4_ioctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map);
566 offset = map.offset;
567 if (ret != 0) {
568 fprintf(stderr, "map ioctl failure\n");
569 abort();
570 }
571
572 bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
573 bo->screen->fd, offset);
574 if (bo->map == MAP_FAILED) {
575 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
576 bo->handle, (long long)offset, bo->size);
577 abort();
578 }
579 VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
580
581 return bo->map;
582 }
583
584 void *
vc4_bo_map(struct vc4_bo * bo)585 vc4_bo_map(struct vc4_bo *bo)
586 {
587 void *map = vc4_bo_map_unsynchronized(bo);
588
589 bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
590 if (!ok) {
591 fprintf(stderr, "BO wait for map failed\n");
592 abort();
593 }
594
595 return map;
596 }
597
598 void
vc4_bufmgr_destroy(struct pipe_screen * pscreen)599 vc4_bufmgr_destroy(struct pipe_screen *pscreen)
600 {
601 struct vc4_screen *screen = vc4_screen(pscreen);
602 struct vc4_bo_cache *cache = &screen->bo_cache;
603
604 vc4_bo_cache_free_all(cache);
605
606 if (dump_stats) {
607 fprintf(stderr, "BO stats after screen destroy:\n");
608 vc4_bo_dump_stats(screen);
609 }
610 }
611