1 /*
2 * Copyright (C) 2017-2019 Lima Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <stdlib.h>
25 #include <sys/types.h>
26 #include <unistd.h>
27 #include <fcntl.h>
28
29 #include "xf86drm.h"
30 #include "drm-uapi/lima_drm.h"
31
32 #include "util/u_hash_table.h"
33 #include "util/u_math.h"
34 #include "util/os_time.h"
35 #include "os/os_mman.h"
36
37 #include "frontend/drm_driver.h"
38
39 #include "lima_screen.h"
40 #include "lima_bo.h"
41 #include "lima_util.h"
42
lima_bo_table_init(struct lima_screen * screen)43 bool lima_bo_table_init(struct lima_screen *screen)
44 {
45 screen->bo_handles = util_hash_table_create_ptr_keys();
46 if (!screen->bo_handles)
47 return false;
48
49 screen->bo_flink_names = util_hash_table_create_ptr_keys();
50 if (!screen->bo_flink_names)
51 goto err_out0;
52
53 mtx_init(&screen->bo_table_lock, mtx_plain);
54 return true;
55
56 err_out0:
57 _mesa_hash_table_destroy(screen->bo_handles, NULL);
58 return false;
59 }
60
lima_bo_cache_init(struct lima_screen * screen)61 bool lima_bo_cache_init(struct lima_screen *screen)
62 {
63 mtx_init(&screen->bo_cache_lock, mtx_plain);
64 list_inithead(&screen->bo_cache_time);
65 for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++)
66 list_inithead(&screen->bo_cache_buckets[i]);
67
68 return true;
69 }
70
lima_bo_table_fini(struct lima_screen * screen)71 void lima_bo_table_fini(struct lima_screen *screen)
72 {
73 mtx_destroy(&screen->bo_table_lock);
74 _mesa_hash_table_destroy(screen->bo_handles, NULL);
75 _mesa_hash_table_destroy(screen->bo_flink_names, NULL);
76 }
77
78 static void
lima_bo_cache_remove(struct lima_bo * bo)79 lima_bo_cache_remove(struct lima_bo *bo)
80 {
81 list_del(&bo->size_list);
82 list_del(&bo->time_list);
83 }
84
lima_close_kms_handle(struct lima_screen * screen,uint32_t handle)85 static void lima_close_kms_handle(struct lima_screen *screen, uint32_t handle)
86 {
87 struct drm_gem_close args = {
88 .handle = handle,
89 };
90
91 drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &args);
92 }
93
94 static void
lima_bo_free(struct lima_bo * bo)95 lima_bo_free(struct lima_bo *bo)
96 {
97 struct lima_screen *screen = bo->screen;
98
99 if (lima_debug & LIMA_DEBUG_BO_CACHE)
100 fprintf(stderr, "%s: %p (size=%d)\n", __func__,
101 bo, bo->size);
102
103 mtx_lock(&screen->bo_table_lock);
104 _mesa_hash_table_remove_key(screen->bo_handles,
105 (void *)(uintptr_t)bo->handle);
106 if (bo->flink_name)
107 _mesa_hash_table_remove_key(screen->bo_flink_names,
108 (void *)(uintptr_t)bo->flink_name);
109 mtx_unlock(&screen->bo_table_lock);
110
111 if (bo->map)
112 lima_bo_unmap(bo);
113
114 lima_close_kms_handle(screen, bo->handle);
115 free(bo);
116 }
117
lima_bo_cache_fini(struct lima_screen * screen)118 void lima_bo_cache_fini(struct lima_screen *screen)
119 {
120 mtx_destroy(&screen->bo_cache_lock);
121
122 list_for_each_entry_safe(struct lima_bo, entry,
123 &screen->bo_cache_time, time_list) {
124 lima_bo_cache_remove(entry);
125 lima_bo_free(entry);
126 }
127 }
128
lima_bo_get_info(struct lima_bo * bo)129 static bool lima_bo_get_info(struct lima_bo *bo)
130 {
131 struct drm_lima_gem_info req = {
132 .handle = bo->handle,
133 };
134
135 if(drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_INFO, &req))
136 return false;
137
138 bo->offset = req.offset;
139 bo->va = req.va;
140 return true;
141 }
142
143 static unsigned
lima_bucket_index(unsigned size)144 lima_bucket_index(unsigned size)
145 {
146 /* Round down to POT to compute a bucket index */
147
148 unsigned bucket_index = util_logbase2(size);
149
150 /* Clamp the bucket index; all huge allocations will be
151 * sorted into the largest bucket */
152 bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET,
153 MAX_BO_CACHE_BUCKET);
154
155 /* Reindex from 0 */
156 return (bucket_index - MIN_BO_CACHE_BUCKET);
157 }
158
159 static struct list_head *
lima_bo_cache_get_bucket(struct lima_screen * screen,unsigned size)160 lima_bo_cache_get_bucket(struct lima_screen *screen, unsigned size)
161 {
162 return &screen->bo_cache_buckets[lima_bucket_index(size)];
163 }
164
165 static void
lima_bo_cache_free_stale_bos(struct lima_screen * screen,time_t time)166 lima_bo_cache_free_stale_bos(struct lima_screen *screen, time_t time)
167 {
168 unsigned cnt = 0;
169 list_for_each_entry_safe(struct lima_bo, entry,
170 &screen->bo_cache_time, time_list) {
171 /* Free BOs that are sitting idle for longer than 5 seconds */
172 if (time - entry->free_time > 6) {
173 lima_bo_cache_remove(entry);
174 lima_bo_free(entry);
175 cnt++;
176 } else
177 break;
178 }
179 if ((lima_debug & LIMA_DEBUG_BO_CACHE) && cnt)
180 fprintf(stderr, "%s: freed %d stale BOs\n", __func__, cnt);
181 }
182
183 static void
lima_bo_cache_print_stats(struct lima_screen * screen)184 lima_bo_cache_print_stats(struct lima_screen *screen)
185 {
186 fprintf(stderr, "===============\n");
187 fprintf(stderr, "BO cache stats:\n");
188 unsigned total_size = 0;
189 for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++) {
190 struct list_head *bucket = &screen->bo_cache_buckets[i];
191 unsigned bucket_size = 0;
192 list_for_each_entry(struct lima_bo, entry, bucket, size_list) {
193 bucket_size += entry->size;
194 total_size += entry->size;
195 }
196 fprintf(stderr, "Bucket #%d, BOs: %d, size: %u\n", i,
197 list_length(bucket),
198 bucket_size);
199 }
200 fprintf(stderr, "Total size: %u\n", total_size);
201 }
202
203 static bool
lima_bo_cache_put(struct lima_bo * bo)204 lima_bo_cache_put(struct lima_bo *bo)
205 {
206 if (!bo->cacheable)
207 return false;
208
209 struct lima_screen *screen = bo->screen;
210
211 mtx_lock(&screen->bo_cache_lock);
212 struct list_head *bucket = lima_bo_cache_get_bucket(screen, bo->size);
213
214 if (!bucket) {
215 mtx_unlock(&screen->bo_cache_lock);
216 return false;
217 }
218
219 struct timespec time;
220 clock_gettime(CLOCK_MONOTONIC, &time);
221 bo->free_time = time.tv_sec;
222 list_addtail(&bo->size_list, bucket);
223 list_addtail(&bo->time_list, &screen->bo_cache_time);
224 lima_bo_cache_free_stale_bos(screen, time.tv_sec);
225 if (lima_debug & LIMA_DEBUG_BO_CACHE) {
226 fprintf(stderr, "%s: put BO: %p (size=%d)\n", __func__, bo, bo->size);
227 lima_bo_cache_print_stats(screen);
228 }
229 mtx_unlock(&screen->bo_cache_lock);
230
231 return true;
232 }
233
234 static struct lima_bo *
lima_bo_cache_get(struct lima_screen * screen,uint32_t size,uint32_t flags)235 lima_bo_cache_get(struct lima_screen *screen, uint32_t size, uint32_t flags)
236 {
237 /* we won't cache heap buffer */
238 if (flags & LIMA_BO_FLAG_HEAP)
239 return NULL;
240
241 struct lima_bo *bo = NULL;
242 mtx_lock(&screen->bo_cache_lock);
243 struct list_head *bucket = lima_bo_cache_get_bucket(screen, size);
244
245 if (!bucket) {
246 mtx_unlock(&screen->bo_cache_lock);
247 return false;
248 }
249
250 list_for_each_entry_safe(struct lima_bo, entry, bucket, size_list) {
251 if (entry->size >= size) {
252 /* Check if BO is idle. If it's not it's better to allocate new one */
253 if (!lima_bo_wait(entry, LIMA_GEM_WAIT_WRITE, 0)) {
254 if (lima_debug & LIMA_DEBUG_BO_CACHE) {
255 fprintf(stderr, "%s: found BO %p but it's busy\n", __func__,
256 entry);
257 }
258 break;
259 }
260
261 lima_bo_cache_remove(entry);
262 p_atomic_set(&entry->refcnt, 1);
263 entry->flags = flags;
264 bo = entry;
265 if (lima_debug & LIMA_DEBUG_BO_CACHE) {
266 fprintf(stderr, "%s: got BO: %p (size=%d), requested size %d\n",
267 __func__, bo, bo->size, size);
268 lima_bo_cache_print_stats(screen);
269 }
270 break;
271 }
272 }
273
274 mtx_unlock(&screen->bo_cache_lock);
275
276 return bo;
277 }
278
lima_bo_create(struct lima_screen * screen,uint32_t size,uint32_t flags)279 struct lima_bo *lima_bo_create(struct lima_screen *screen,
280 uint32_t size, uint32_t flags)
281 {
282 struct lima_bo *bo;
283
284 size = align(size, LIMA_PAGE_SIZE);
285
286 /* Try to get bo from cache first */
287 bo = lima_bo_cache_get(screen, size, flags);
288 if (bo)
289 return bo;
290
291 struct drm_lima_gem_create req = {
292 .size = size,
293 .flags = flags,
294 };
295
296 if (!(bo = calloc(1, sizeof(*bo))))
297 return NULL;
298
299 list_inithead(&bo->time_list);
300 list_inithead(&bo->size_list);
301
302 if (drmIoctl(screen->fd, DRM_IOCTL_LIMA_GEM_CREATE, &req))
303 goto err_out0;
304
305 bo->screen = screen;
306 bo->size = req.size;
307 bo->flags = req.flags;
308 bo->handle = req.handle;
309 bo->cacheable = !(lima_debug & LIMA_DEBUG_NO_BO_CACHE ||
310 flags & LIMA_BO_FLAG_HEAP);
311 p_atomic_set(&bo->refcnt, 1);
312
313 if (!lima_bo_get_info(bo))
314 goto err_out1;
315
316 if (lima_debug & LIMA_DEBUG_BO_CACHE)
317 fprintf(stderr, "%s: %p (size=%d)\n", __func__,
318 bo, bo->size);
319
320 return bo;
321
322 err_out1:
323 lima_close_kms_handle(screen, bo->handle);
324 err_out0:
325 free(bo);
326 return NULL;
327 }
328
lima_bo_unreference(struct lima_bo * bo)329 void lima_bo_unreference(struct lima_bo *bo)
330 {
331 if (!p_atomic_dec_zero(&bo->refcnt))
332 return;
333
334 /* Try to put it into cache */
335 if (lima_bo_cache_put(bo))
336 return;
337
338 lima_bo_free(bo);
339 }
340
lima_bo_map(struct lima_bo * bo)341 void *lima_bo_map(struct lima_bo *bo)
342 {
343 if (!bo->map) {
344 bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE,
345 MAP_SHARED, bo->screen->fd, bo->offset);
346 if (bo->map == MAP_FAILED)
347 bo->map = NULL;
348 }
349
350 return bo->map;
351 }
352
lima_bo_unmap(struct lima_bo * bo)353 void lima_bo_unmap(struct lima_bo *bo)
354 {
355 if (bo->map) {
356 os_munmap(bo->map, bo->size);
357 bo->map = NULL;
358 }
359 }
360
lima_bo_export(struct lima_bo * bo,struct winsys_handle * handle)361 bool lima_bo_export(struct lima_bo *bo, struct winsys_handle *handle)
362 {
363 struct lima_screen *screen = bo->screen;
364
365 /* Don't cache exported BOs */
366 bo->cacheable = false;
367
368 switch (handle->type) {
369 case WINSYS_HANDLE_TYPE_SHARED:
370 if (!bo->flink_name) {
371 struct drm_gem_flink flink = {
372 .handle = bo->handle,
373 .name = 0,
374 };
375 if (drmIoctl(screen->fd, DRM_IOCTL_GEM_FLINK, &flink))
376 return false;
377
378 bo->flink_name = flink.name;
379
380 mtx_lock(&screen->bo_table_lock);
381 _mesa_hash_table_insert(screen->bo_flink_names,
382 (void *)(uintptr_t)bo->flink_name, bo);
383 mtx_unlock(&screen->bo_table_lock);
384 }
385 handle->handle = bo->flink_name;
386 return true;
387
388 case WINSYS_HANDLE_TYPE_KMS:
389 mtx_lock(&screen->bo_table_lock);
390 _mesa_hash_table_insert(screen->bo_handles,
391 (void *)(uintptr_t)bo->handle, bo);
392 mtx_unlock(&screen->bo_table_lock);
393
394 handle->handle = bo->handle;
395 return true;
396
397 case WINSYS_HANDLE_TYPE_FD:
398 if (drmPrimeHandleToFD(screen->fd, bo->handle, DRM_CLOEXEC,
399 (int*)&handle->handle))
400 return false;
401
402 mtx_lock(&screen->bo_table_lock);
403 _mesa_hash_table_insert(screen->bo_handles,
404 (void *)(uintptr_t)bo->handle, bo);
405 mtx_unlock(&screen->bo_table_lock);
406 return true;
407
408 default:
409 return false;
410 }
411 }
412
lima_bo_import(struct lima_screen * screen,struct winsys_handle * handle)413 struct lima_bo *lima_bo_import(struct lima_screen *screen,
414 struct winsys_handle *handle)
415 {
416 struct lima_bo *bo = NULL;
417 struct drm_gem_open req = {0};
418 uint32_t dma_buf_size = 0;
419 unsigned h = handle->handle;
420
421 mtx_lock(&screen->bo_table_lock);
422
423 /* Convert a DMA buf handle to a KMS handle now. */
424 if (handle->type == WINSYS_HANDLE_TYPE_FD) {
425 uint32_t prime_handle;
426 off_t size;
427
428 /* Get a KMS handle. */
429 if (drmPrimeFDToHandle(screen->fd, h, &prime_handle)) {
430 mtx_unlock(&screen->bo_table_lock);
431 return NULL;
432 }
433
434 /* Query the buffer size. */
435 size = lseek(h, 0, SEEK_END);
436 if (size == (off_t)-1) {
437 mtx_unlock(&screen->bo_table_lock);
438 lima_close_kms_handle(screen, prime_handle);
439 return NULL;
440 }
441 lseek(h, 0, SEEK_SET);
442
443 dma_buf_size = size;
444 h = prime_handle;
445 }
446
447 switch (handle->type) {
448 case WINSYS_HANDLE_TYPE_SHARED:
449 bo = util_hash_table_get(screen->bo_flink_names,
450 (void *)(uintptr_t)h);
451 break;
452 case WINSYS_HANDLE_TYPE_KMS:
453 case WINSYS_HANDLE_TYPE_FD:
454 bo = util_hash_table_get(screen->bo_handles,
455 (void *)(uintptr_t)h);
456 break;
457 default:
458 mtx_unlock(&screen->bo_table_lock);
459 return NULL;
460 }
461
462 if (bo) {
463 p_atomic_inc(&bo->refcnt);
464 /* Don't cache imported BOs */
465 bo->cacheable = false;
466 mtx_unlock(&screen->bo_table_lock);
467 return bo;
468 }
469
470 if (!(bo = calloc(1, sizeof(*bo)))) {
471 mtx_unlock(&screen->bo_table_lock);
472 if (handle->type == WINSYS_HANDLE_TYPE_FD)
473 lima_close_kms_handle(screen, h);
474 return NULL;
475 }
476
477 /* Don't cache imported BOs */
478 bo->cacheable = false;
479 list_inithead(&bo->time_list);
480 list_inithead(&bo->size_list);
481 bo->screen = screen;
482 p_atomic_set(&bo->refcnt, 1);
483
484 switch (handle->type) {
485 case WINSYS_HANDLE_TYPE_SHARED:
486 req.name = h;
487 if (drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &req)) {
488 mtx_unlock(&screen->bo_table_lock);
489 free(bo);
490 return NULL;
491 }
492 bo->handle = req.handle;
493 bo->flink_name = h;
494 bo->size = req.size;
495 break;
496 case WINSYS_HANDLE_TYPE_FD:
497 bo->handle = h;
498 bo->size = dma_buf_size;
499 break;
500 default:
501 /* not possible */
502 assert(0);
503 }
504
505 if (lima_bo_get_info(bo)) {
506 if (handle->type == WINSYS_HANDLE_TYPE_SHARED)
507 _mesa_hash_table_insert(screen->bo_flink_names,
508 (void *)(uintptr_t)bo->flink_name, bo);
509 _mesa_hash_table_insert(screen->bo_handles,
510 (void*)(uintptr_t)bo->handle, bo);
511 }
512 else {
513 lima_close_kms_handle(screen, bo->handle);
514 free(bo);
515 bo = NULL;
516 }
517
518 mtx_unlock(&screen->bo_table_lock);
519
520 return bo;
521 }
522
lima_bo_wait(struct lima_bo * bo,uint32_t op,uint64_t timeout_ns)523 bool lima_bo_wait(struct lima_bo *bo, uint32_t op, uint64_t timeout_ns)
524 {
525 int64_t abs_timeout;
526
527 if (timeout_ns == 0)
528 abs_timeout = 0;
529 else
530 abs_timeout = os_time_get_absolute_timeout(timeout_ns);
531
532 if (abs_timeout == OS_TIMEOUT_INFINITE)
533 abs_timeout = INT64_MAX;
534
535 struct drm_lima_gem_wait req = {
536 .handle = bo->handle,
537 .op = op,
538 .timeout_ns = abs_timeout,
539 };
540
541 return drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_WAIT, &req) == 0;
542 }
543