• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright © 2017 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included
12  * in all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20  * DEALINGS IN THE SOFTWARE.
21  */
22 
23 /**
24  * @file iris_bufmgr.c
25  *
26  * The Iris buffer manager.
27  *
28  * XXX: write better comments
29  * - BOs
30  * - Explain BO cache
31  * - main interface to GEM in the kernel
32  */
33 
34 #include <xf86drm.h>
35 #include <util/u_atomic.h>
36 #include <fcntl.h>
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <string.h>
40 #include <unistd.h>
41 #include <assert.h>
42 #include <sys/ioctl.h>
43 #include <sys/mman.h>
44 #include <sys/stat.h>
45 #include <sys/types.h>
46 #include <stdbool.h>
47 #include <time.h>
48 #include <unistd.h>
49 
50 #include "errno.h"
51 #include "common/intel_aux_map.h"
52 #include "common/intel_clflush.h"
53 #include "dev/intel_debug.h"
54 #include "common/intel_gem.h"
55 #include "dev/intel_device_info.h"
56 #include "isl/isl.h"
57 #include "os/os_mman.h"
58 #include "util/debug.h"
59 #include "util/macros.h"
60 #include "util/hash_table.h"
61 #include "util/list.h"
62 #include "util/os_file.h"
63 #include "util/u_dynarray.h"
64 #include "util/vma.h"
65 #include "iris_bufmgr.h"
66 #include "iris_context.h"
67 #include "string.h"
68 
69 #include "drm-uapi/i915_drm.h"
70 
71 #ifdef HAVE_VALGRIND
72 #include <valgrind.h>
73 #include <memcheck.h>
74 #define VG(x) x
75 #else
76 #define VG(x)
77 #endif
78 
79 /* VALGRIND_FREELIKE_BLOCK unfortunately does not actually undo the earlier
80  * VALGRIND_MALLOCLIKE_BLOCK but instead leaves vg convinced the memory is
81  * leaked. All because it does not call VG(cli_free) from its
82  * VG_USERREQ__FREELIKE_BLOCK handler. Instead of treating the memory like
83  * and allocation, we mark it available for use upon mmapping and remove
84  * it upon unmapping.
85  */
86 #define VG_DEFINED(ptr, size) VG(VALGRIND_MAKE_MEM_DEFINED(ptr, size))
87 #define VG_NOACCESS(ptr, size) VG(VALGRIND_MAKE_MEM_NOACCESS(ptr, size))
88 
89 /* On FreeBSD PAGE_SIZE is already defined in
90  * /usr/include/machine/param.h that is indirectly
91  * included here.
92  */
93 #ifndef PAGE_SIZE
94 #define PAGE_SIZE 4096
95 #endif
96 
97 #define WARN_ONCE(cond, fmt...) do {                            \
98    if (unlikely(cond)) {                                        \
99       static bool _warned = false;                              \
100       if (!_warned) {                                           \
101          fprintf(stderr, "WARNING: ");                          \
102          fprintf(stderr, fmt);                                  \
103          _warned = true;                                        \
104       }                                                         \
105    }                                                            \
106 } while (0)
107 
108 #define FILE_DEBUG_FLAG DEBUG_BUFMGR
109 
110 /**
111  * For debugging purposes, this returns a time in seconds.
112  */
113 static double
get_time(void)114 get_time(void)
115 {
116    struct timespec tp;
117 
118    clock_gettime(CLOCK_MONOTONIC, &tp);
119 
120    return tp.tv_sec + tp.tv_nsec / 1000000000.0;
121 }
122 
123 static inline int
atomic_add_unless(int * v,int add,int unless)124 atomic_add_unless(int *v, int add, int unless)
125 {
126    int c, old;
127    c = p_atomic_read(v);
128    while (c != unless && (old = p_atomic_cmpxchg(v, c, c + add)) != c)
129       c = old;
130    return c == unless;
131 }
132 
133 static const char *
memzone_name(enum iris_memory_zone memzone)134 memzone_name(enum iris_memory_zone memzone)
135 {
136    const char *names[] = {
137       [IRIS_MEMZONE_SHADER]   = "shader",
138       [IRIS_MEMZONE_BINDER]   = "binder",
139       [IRIS_MEMZONE_BINDLESS] = "scratchsurf",
140       [IRIS_MEMZONE_SURFACE]  = "surface",
141       [IRIS_MEMZONE_DYNAMIC]  = "dynamic",
142       [IRIS_MEMZONE_OTHER]    = "other",
143       [IRIS_MEMZONE_BORDER_COLOR_POOL] = "bordercolor",
144    };
145    assert(memzone < ARRAY_SIZE(names));
146    return names[memzone];
147 }
148 
149 struct bo_cache_bucket {
150    /** List of cached BOs. */
151    struct list_head head;
152 
153    /** Size of this bucket, in bytes. */
154    uint64_t size;
155 };
156 
157 struct bo_export {
158    /** File descriptor associated with a handle export. */
159    int drm_fd;
160 
161    /** GEM handle in drm_fd */
162    uint32_t gem_handle;
163 
164    struct list_head link;
165 };
166 
167 struct iris_memregion {
168    struct drm_i915_gem_memory_class_instance region;
169    uint64_t size;
170 };
171 
172 #define NUM_SLAB_ALLOCATORS 3
173 
174 struct iris_slab {
175    struct pb_slab base;
176 
177    unsigned entry_size;
178 
179    /** The BO representing the entire slab */
180    struct iris_bo *bo;
181 
182    /** Array of iris_bo structs representing BOs allocated out of this slab */
183    struct iris_bo *entries;
184 };
185 
186 #define BUCKET_ARRAY_SIZE (14 * 4)
187 
188 struct iris_bufmgr {
189    /**
190     * List into the list of bufmgr.
191     */
192    struct list_head link;
193 
194    uint32_t refcount;
195 
196    int fd;
197 
198    simple_mtx_t lock;
199    simple_mtx_t bo_deps_lock;
200 
201    /** Array of lists of cached gem objects of power-of-two sizes */
202    struct bo_cache_bucket cache_bucket[BUCKET_ARRAY_SIZE];
203    int num_buckets;
204 
205    /** Same as cache_bucket, but for local memory gem objects */
206    struct bo_cache_bucket local_cache_bucket[BUCKET_ARRAY_SIZE];
207    int num_local_buckets;
208 
209    /** Same as cache_bucket, but for local-preferred memory gem objects */
210    struct bo_cache_bucket local_preferred_cache_bucket[BUCKET_ARRAY_SIZE];
211    int num_local_preferred_buckets;
212 
213    time_t time;
214 
215    struct hash_table *name_table;
216    struct hash_table *handle_table;
217 
218    /**
219     * List of BOs which we've effectively freed, but are hanging on to
220     * until they're idle before closing and returning the VMA.
221     */
222    struct list_head zombie_list;
223 
224    struct util_vma_heap vma_allocator[IRIS_MEMZONE_COUNT];
225 
226    uint64_t vma_min_align;
227    struct iris_memregion vram, sys;
228 
229    /* Used only when use_global_vm is true. */
230    uint32_t global_vm_id;
231 
232    int next_screen_id;
233 
234    bool has_llc:1;
235    bool has_local_mem:1;
236    bool has_mmap_offset:1;
237    bool has_tiling_uapi:1;
238    bool has_userptr_probe:1;
239    bool bo_reuse:1;
240    bool use_global_vm:1;
241    bool all_vram_mappable:1;
242 
243    struct intel_aux_map_context *aux_map_ctx;
244 
245    struct pb_slabs bo_slabs[NUM_SLAB_ALLOCATORS];
246 
247    struct iris_border_color_pool border_color_pool;
248 };
249 
250 static simple_mtx_t global_bufmgr_list_mutex = _SIMPLE_MTX_INITIALIZER_NP;
251 static struct list_head global_bufmgr_list = {
252    .next = &global_bufmgr_list,
253    .prev = &global_bufmgr_list,
254 };
255 
256 static void bo_free(struct iris_bo *bo);
257 
258 static struct iris_bo *
find_and_ref_external_bo(struct hash_table * ht,unsigned int key)259 find_and_ref_external_bo(struct hash_table *ht, unsigned int key)
260 {
261    struct hash_entry *entry = _mesa_hash_table_search(ht, &key);
262    struct iris_bo *bo = entry ? entry->data : NULL;
263 
264    if (bo) {
265       assert(iris_bo_is_external(bo));
266       assert(iris_bo_is_real(bo));
267       assert(!bo->real.reusable);
268 
269       /* Being non-reusable, the BO cannot be in the cache lists, but it
270        * may be in the zombie list if it had reached zero references, but
271        * we hadn't yet closed it...and then reimported the same BO.  If it
272        * is, then remove it since it's now been resurrected.
273        */
274       if (list_is_linked(&bo->head))
275          list_del(&bo->head);
276 
277       iris_bo_reference(bo);
278    }
279 
280    return bo;
281 }
282 
283 static void
bucket_info_for_heap(struct iris_bufmgr * bufmgr,enum iris_heap heap,struct bo_cache_bucket ** cache_bucket,int ** num_buckets)284 bucket_info_for_heap(struct iris_bufmgr *bufmgr, enum iris_heap heap,
285                      struct bo_cache_bucket **cache_bucket, int **num_buckets)
286 {
287    switch (heap) {
288    case IRIS_HEAP_SYSTEM_MEMORY:
289       *cache_bucket = bufmgr->cache_bucket;
290       *num_buckets = &bufmgr->num_buckets;
291       break;
292    case IRIS_HEAP_DEVICE_LOCAL:
293       *cache_bucket = bufmgr->local_cache_bucket;
294       *num_buckets = &bufmgr->num_local_buckets;
295       break;
296    case IRIS_HEAP_DEVICE_LOCAL_PREFERRED:
297       *cache_bucket = bufmgr->local_preferred_cache_bucket;
298       *num_buckets = &bufmgr->num_local_preferred_buckets;
299       break;
300    case IRIS_HEAP_MAX:
301    default:
302       *cache_bucket = NULL;
303       *num_buckets = NULL;
304       unreachable("invalid heap");
305    }
306 
307    assert(**num_buckets < BUCKET_ARRAY_SIZE);
308 }
309 /**
310  * This function finds the correct bucket fit for the input size.
311  * The function works with O(1) complexity when the requested size
312  * was queried instead of iterating the size through all the buckets.
313  */
314 static struct bo_cache_bucket *
bucket_for_size(struct iris_bufmgr * bufmgr,uint64_t size,enum iris_heap heap)315 bucket_for_size(struct iris_bufmgr *bufmgr, uint64_t size,
316                 enum iris_heap heap)
317 {
318    /* Calculating the pages and rounding up to the page size. */
319    const unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
320 
321    /* Row  Bucket sizes    clz((x-1) | 3)   Row    Column
322     *        in pages                      stride   size
323     *   0:   1  2  3  4 -> 30 30 30 30        4       1
324     *   1:   5  6  7  8 -> 29 29 29 29        4       1
325     *   2:  10 12 14 16 -> 28 28 28 28        8       2
326     *   3:  20 24 28 32 -> 27 27 27 27       16       4
327     */
328    const unsigned row = 30 - __builtin_clz((pages - 1) | 3);
329    const unsigned row_max_pages = 4 << row;
330 
331    /* The '& ~2' is the special case for row 1. In row 1, max pages /
332     * 2 is 2, but the previous row maximum is zero (because there is
333     * no previous row). All row maximum sizes are power of 2, so that
334     * is the only case where that bit will be set.
335     */
336    const unsigned prev_row_max_pages = (row_max_pages / 2) & ~2;
337    int col_size_log2 = row - 1;
338    col_size_log2 += (col_size_log2 < 0);
339 
340    const unsigned col = (pages - prev_row_max_pages +
341                         ((1 << col_size_log2) - 1)) >> col_size_log2;
342 
343    /* Calculating the index based on the row and column. */
344    const unsigned index = (row * 4) + (col - 1);
345 
346    int *num_buckets;
347    struct bo_cache_bucket *buckets;
348    bucket_info_for_heap(bufmgr, heap, &buckets, &num_buckets);
349 
350    return (index < *num_buckets) ? &buckets[index] : NULL;
351 }
352 
353 enum iris_memory_zone
iris_memzone_for_address(uint64_t address)354 iris_memzone_for_address(uint64_t address)
355 {
356    STATIC_ASSERT(IRIS_MEMZONE_OTHER_START    > IRIS_MEMZONE_DYNAMIC_START);
357    STATIC_ASSERT(IRIS_MEMZONE_DYNAMIC_START  > IRIS_MEMZONE_SURFACE_START);
358    STATIC_ASSERT(IRIS_MEMZONE_SURFACE_START  > IRIS_MEMZONE_BINDLESS_START);
359    STATIC_ASSERT(IRIS_MEMZONE_BINDLESS_START > IRIS_MEMZONE_BINDER_START);
360    STATIC_ASSERT(IRIS_MEMZONE_BINDER_START   > IRIS_MEMZONE_SHADER_START);
361    STATIC_ASSERT(IRIS_BORDER_COLOR_POOL_ADDRESS == IRIS_MEMZONE_DYNAMIC_START);
362 
363    if (address >= IRIS_MEMZONE_OTHER_START)
364       return IRIS_MEMZONE_OTHER;
365 
366    if (address == IRIS_BORDER_COLOR_POOL_ADDRESS)
367       return IRIS_MEMZONE_BORDER_COLOR_POOL;
368 
369    if (address > IRIS_MEMZONE_DYNAMIC_START)
370       return IRIS_MEMZONE_DYNAMIC;
371 
372    if (address >= IRIS_MEMZONE_SURFACE_START)
373       return IRIS_MEMZONE_SURFACE;
374 
375    if (address >= IRIS_MEMZONE_BINDLESS_START)
376       return IRIS_MEMZONE_BINDLESS;
377 
378    if (address >= IRIS_MEMZONE_BINDER_START)
379       return IRIS_MEMZONE_BINDER;
380 
381    return IRIS_MEMZONE_SHADER;
382 }
383 
384 /**
385  * Allocate a section of virtual memory for a buffer, assigning an address.
386  *
387  * This uses either the bucket allocator for the given size, or the large
388  * object allocator (util_vma).
389  */
390 static uint64_t
vma_alloc(struct iris_bufmgr * bufmgr,enum iris_memory_zone memzone,uint64_t size,uint64_t alignment)391 vma_alloc(struct iris_bufmgr *bufmgr,
392           enum iris_memory_zone memzone,
393           uint64_t size,
394           uint64_t alignment)
395 {
396    simple_mtx_assert_locked(&bufmgr->lock);
397 
398    /* Force minimum alignment based on device requirements */
399    assert((alignment & (alignment - 1)) == 0);
400    alignment = MAX2(alignment, bufmgr->vma_min_align);
401 
402    if (memzone == IRIS_MEMZONE_BORDER_COLOR_POOL)
403       return IRIS_BORDER_COLOR_POOL_ADDRESS;
404 
405    uint64_t addr =
406       util_vma_heap_alloc(&bufmgr->vma_allocator[memzone], size, alignment);
407 
408    assert((addr >> 48ull) == 0);
409    assert((addr % alignment) == 0);
410 
411    return intel_canonical_address(addr);
412 }
413 
414 static void
vma_free(struct iris_bufmgr * bufmgr,uint64_t address,uint64_t size)415 vma_free(struct iris_bufmgr *bufmgr,
416          uint64_t address,
417          uint64_t size)
418 {
419    simple_mtx_assert_locked(&bufmgr->lock);
420 
421    if (address == IRIS_BORDER_COLOR_POOL_ADDRESS)
422       return;
423 
424    /* Un-canonicalize the address. */
425    address = intel_48b_address(address);
426 
427    if (address == 0ull)
428       return;
429 
430    enum iris_memory_zone memzone = iris_memzone_for_address(address);
431 
432    assert(memzone < ARRAY_SIZE(bufmgr->vma_allocator));
433 
434    util_vma_heap_free(&bufmgr->vma_allocator[memzone], address, size);
435 }
436 
437 static bool
iris_bo_busy_gem(struct iris_bo * bo)438 iris_bo_busy_gem(struct iris_bo *bo)
439 {
440    assert(iris_bo_is_real(bo));
441 
442    struct iris_bufmgr *bufmgr = bo->bufmgr;
443    struct drm_i915_gem_busy busy = { .handle = bo->gem_handle };
444 
445    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
446    if (ret == 0) {
447       return busy.busy;
448    }
449    return false;
450 }
451 
452 /* A timeout of 0 just checks for busyness. */
453 static int
iris_bo_wait_syncobj(struct iris_bo * bo,int64_t timeout_ns)454 iris_bo_wait_syncobj(struct iris_bo *bo, int64_t timeout_ns)
455 {
456    int ret = 0;
457    struct iris_bufmgr *bufmgr = bo->bufmgr;
458 
459    /* If we know it's idle, don't bother with the kernel round trip */
460    if (bo->idle)
461       return 0;
462 
463    simple_mtx_lock(&bufmgr->bo_deps_lock);
464 
465    uint32_t handles[bo->deps_size * IRIS_BATCH_COUNT * 2];
466    int handle_count = 0;
467 
468    for (int d = 0; d < bo->deps_size; d++) {
469       for (int b = 0; b < IRIS_BATCH_COUNT; b++) {
470          struct iris_syncobj *r = bo->deps[d].read_syncobjs[b];
471          struct iris_syncobj *w = bo->deps[d].write_syncobjs[b];
472          if (r)
473             handles[handle_count++] = r->handle;
474          if (w)
475             handles[handle_count++] = w->handle;
476       }
477    }
478 
479    if (handle_count == 0)
480       goto out;
481 
482    /* Unlike the gem wait, negative values are not infinite here. */
483    int64_t timeout_abs = os_time_get_absolute_timeout(timeout_ns);
484    if (timeout_abs < 0)
485       timeout_abs = INT64_MAX;
486 
487    struct drm_syncobj_wait args = {
488       .handles = (uintptr_t) handles,
489       .timeout_nsec = timeout_abs,
490       .count_handles = handle_count,
491       .flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL,
492    };
493 
494    ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args);
495    if (ret != 0) {
496       ret = -errno;
497       goto out;
498    }
499 
500    /* We just waited everything, so clean all the deps. */
501    for (int d = 0; d < bo->deps_size; d++) {
502       for (int b = 0; b < IRIS_BATCH_COUNT; b++) {
503          iris_syncobj_reference(bufmgr, &bo->deps[d].write_syncobjs[b], NULL);
504          iris_syncobj_reference(bufmgr, &bo->deps[d].read_syncobjs[b], NULL);
505       }
506    }
507 
508 out:
509    simple_mtx_unlock(&bufmgr->bo_deps_lock);
510    return ret;
511 }
512 
513 static bool
iris_bo_busy_syncobj(struct iris_bo * bo)514 iris_bo_busy_syncobj(struct iris_bo *bo)
515 {
516    return iris_bo_wait_syncobj(bo, 0) == -ETIME;
517 }
518 
519 bool
iris_bo_busy(struct iris_bo * bo)520 iris_bo_busy(struct iris_bo *bo)
521 {
522    bool busy;
523    if (iris_bo_is_external(bo))
524       busy = iris_bo_busy_gem(bo);
525    else
526       busy = iris_bo_busy_syncobj(bo);
527 
528    bo->idle = !busy;
529 
530    return busy;
531 }
532 
533 int
iris_bo_madvise(struct iris_bo * bo,int state)534 iris_bo_madvise(struct iris_bo *bo, int state)
535 {
536    /* We can't madvise suballocated BOs. */
537    assert(iris_bo_is_real(bo));
538 
539    struct drm_i915_gem_madvise madv = {
540       .handle = bo->gem_handle,
541       .madv = state,
542       .retained = 1,
543    };
544 
545    intel_ioctl(bo->bufmgr->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
546 
547    return madv.retained;
548 }
549 
550 static struct iris_bo *
bo_calloc(void)551 bo_calloc(void)
552 {
553    struct iris_bo *bo = calloc(1, sizeof(*bo));
554    if (!bo)
555       return NULL;
556 
557    list_inithead(&bo->real.exports);
558 
559    bo->hash = _mesa_hash_pointer(bo);
560 
561    return bo;
562 }
563 
564 static void
bo_unmap(struct iris_bo * bo)565 bo_unmap(struct iris_bo *bo)
566 {
567    assert(iris_bo_is_real(bo));
568 
569    VG_NOACCESS(bo->real.map, bo->size);
570    os_munmap(bo->real.map, bo->size);
571    bo->real.map = NULL;
572 }
573 
574 static struct pb_slabs *
get_slabs(struct iris_bufmgr * bufmgr,uint64_t size)575 get_slabs(struct iris_bufmgr *bufmgr, uint64_t size)
576 {
577    for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
578       struct pb_slabs *slabs = &bufmgr->bo_slabs[i];
579 
580       if (size <= 1ull << (slabs->min_order + slabs->num_orders - 1))
581          return slabs;
582    }
583 
584    unreachable("should have found a valid slab for this size");
585 }
586 
587 /* Return the power of two size of a slab entry matching the input size. */
588 static unsigned
get_slab_pot_entry_size(struct iris_bufmgr * bufmgr,unsigned size)589 get_slab_pot_entry_size(struct iris_bufmgr *bufmgr, unsigned size)
590 {
591    unsigned entry_size = util_next_power_of_two(size);
592    unsigned min_entry_size = 1 << bufmgr->bo_slabs[0].min_order;
593 
594    return MAX2(entry_size, min_entry_size);
595 }
596 
597 /* Return the slab entry alignment. */
598 static unsigned
get_slab_entry_alignment(struct iris_bufmgr * bufmgr,unsigned size)599 get_slab_entry_alignment(struct iris_bufmgr *bufmgr, unsigned size)
600 {
601    unsigned entry_size = get_slab_pot_entry_size(bufmgr, size);
602 
603    if (size <= entry_size * 3 / 4)
604       return entry_size / 4;
605 
606    return entry_size;
607 }
608 
609 static bool
iris_can_reclaim_slab(void * priv,struct pb_slab_entry * entry)610 iris_can_reclaim_slab(void *priv, struct pb_slab_entry *entry)
611 {
612    struct iris_bo *bo = container_of(entry, struct iris_bo, slab.entry);
613 
614    return !iris_bo_busy(bo);
615 }
616 
617 static void
iris_slab_free(void * priv,struct pb_slab * pslab)618 iris_slab_free(void *priv, struct pb_slab *pslab)
619 {
620    struct iris_bufmgr *bufmgr = priv;
621    struct iris_slab *slab = (void *) pslab;
622    struct intel_aux_map_context *aux_map_ctx = bufmgr->aux_map_ctx;
623 
624    assert(!slab->bo->aux_map_address);
625 
626    /* Since we're freeing the whole slab, all buffers allocated out of it
627     * must be reclaimable.  We require buffers to be idle to be reclaimed
628     * (see iris_can_reclaim_slab()), so we know all entries must be idle.
629     * Therefore, we can safely unmap their aux table entries.
630     */
631    for (unsigned i = 0; i < pslab->num_entries; i++) {
632       struct iris_bo *bo = &slab->entries[i];
633       if (aux_map_ctx && bo->aux_map_address) {
634          intel_aux_map_unmap_range(aux_map_ctx, bo->address, bo->size);
635          bo->aux_map_address = 0;
636       }
637 
638       /* Unref read/write dependency syncobjs and free the array. */
639       for (int d = 0; d < bo->deps_size; d++) {
640          for (int b = 0; b < IRIS_BATCH_COUNT; b++) {
641             iris_syncobj_reference(bufmgr, &bo->deps[d].write_syncobjs[b], NULL);
642             iris_syncobj_reference(bufmgr, &bo->deps[d].read_syncobjs[b], NULL);
643          }
644       }
645       free(bo->deps);
646    }
647 
648    iris_bo_unreference(slab->bo);
649 
650    free(slab->entries);
651    free(slab);
652 }
653 
654 static struct pb_slab *
iris_slab_alloc(void * priv,unsigned heap,unsigned entry_size,unsigned group_index)655 iris_slab_alloc(void *priv,
656                 unsigned heap,
657                 unsigned entry_size,
658                 unsigned group_index)
659 {
660    struct iris_bufmgr *bufmgr = priv;
661    struct iris_slab *slab = calloc(1, sizeof(struct iris_slab));
662    unsigned flags = heap == IRIS_HEAP_SYSTEM_MEMORY ? BO_ALLOC_SMEM :
663                     heap == IRIS_HEAP_DEVICE_LOCAL ? BO_ALLOC_LMEM : 0;
664    unsigned slab_size = 0;
665    /* We only support slab allocation for IRIS_MEMZONE_OTHER */
666    enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER;
667 
668    if (!slab)
669       return NULL;
670 
671    struct pb_slabs *slabs = bufmgr->bo_slabs;
672 
673    /* Determine the slab buffer size. */
674    for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
675       unsigned max_entry_size =
676          1 << (slabs[i].min_order + slabs[i].num_orders - 1);
677 
678       if (entry_size <= max_entry_size) {
679          /* The slab size is twice the size of the largest possible entry. */
680          slab_size = max_entry_size * 2;
681 
682          if (!util_is_power_of_two_nonzero(entry_size)) {
683             assert(util_is_power_of_two_nonzero(entry_size * 4 / 3));
684 
685             /* If the entry size is 3/4 of a power of two, we would waste
686              * space and not gain anything if we allocated only twice the
687              * power of two for the backing buffer:
688              *
689              *    2 * 3/4 = 1.5 usable with buffer size 2
690              *
691              * Allocating 5 times the entry size leads us to the next power
692              * of two and results in a much better memory utilization:
693              *
694              *    5 * 3/4 = 3.75 usable with buffer size 4
695              */
696             if (entry_size * 5 > slab_size)
697                slab_size = util_next_power_of_two(entry_size * 5);
698          }
699 
700          /* The largest slab should have the same size as the PTE fragment
701           * size to get faster address translation.
702           *
703           * TODO: move this to intel_device_info?
704           */
705          const unsigned pte_size = 2 * 1024 * 1024;
706 
707          if (i == NUM_SLAB_ALLOCATORS - 1 && slab_size < pte_size)
708             slab_size = pte_size;
709 
710          break;
711       }
712    }
713    assert(slab_size != 0);
714 
715    slab->bo =
716       iris_bo_alloc(bufmgr, "slab", slab_size, slab_size, memzone, flags);
717    if (!slab->bo)
718       goto fail;
719 
720    slab_size = slab->bo->size;
721 
722    slab->base.num_entries = slab_size / entry_size;
723    slab->base.num_free = slab->base.num_entries;
724    slab->entry_size = entry_size;
725    slab->entries = calloc(slab->base.num_entries, sizeof(*slab->entries));
726    if (!slab->entries)
727       goto fail_bo;
728 
729    list_inithead(&slab->base.free);
730 
731    for (unsigned i = 0; i < slab->base.num_entries; i++) {
732       struct iris_bo *bo = &slab->entries[i];
733 
734       bo->size = entry_size;
735       bo->bufmgr = bufmgr;
736       bo->hash = _mesa_hash_pointer(bo);
737       bo->gem_handle = 0;
738       bo->address = slab->bo->address + i * entry_size;
739       bo->aux_map_address = 0;
740       bo->index = -1;
741       bo->refcount = 0;
742       bo->idle = true;
743 
744       bo->slab.entry.slab = &slab->base;
745       bo->slab.entry.group_index = group_index;
746       bo->slab.entry.entry_size = entry_size;
747 
748       bo->slab.real = iris_get_backing_bo(slab->bo);
749 
750       list_addtail(&bo->slab.entry.head, &slab->base.free);
751    }
752 
753    return &slab->base;
754 
755 fail_bo:
756    iris_bo_unreference(slab->bo);
757 fail:
758    free(slab);
759    return NULL;
760 }
761 
762 static enum iris_heap
flags_to_heap(struct iris_bufmgr * bufmgr,unsigned flags)763 flags_to_heap(struct iris_bufmgr *bufmgr, unsigned flags)
764 {
765    if (bufmgr->vram.size > 0 &&
766        !(flags & BO_ALLOC_SMEM) &&
767        !(flags & BO_ALLOC_COHERENT)) {
768       return flags & BO_ALLOC_LMEM ? IRIS_HEAP_DEVICE_LOCAL :
769                                      IRIS_HEAP_DEVICE_LOCAL_PREFERRED;
770    } else {
771       assert(!(flags & BO_ALLOC_LMEM));
772       return IRIS_HEAP_SYSTEM_MEMORY;
773    }
774 }
775 
776 static struct iris_bo *
alloc_bo_from_slabs(struct iris_bufmgr * bufmgr,const char * name,uint64_t size,uint32_t alignment,unsigned flags)777 alloc_bo_from_slabs(struct iris_bufmgr *bufmgr,
778                     const char *name,
779                     uint64_t size,
780                     uint32_t alignment,
781                     unsigned flags)
782 {
783    if (flags & BO_ALLOC_NO_SUBALLOC)
784       return NULL;
785 
786    struct pb_slabs *last_slab = &bufmgr->bo_slabs[NUM_SLAB_ALLOCATORS - 1];
787    unsigned max_slab_entry_size =
788       1 << (last_slab->min_order + last_slab->num_orders - 1);
789 
790    if (size > max_slab_entry_size)
791       return NULL;
792 
793    struct pb_slab_entry *entry;
794 
795    enum iris_heap heap = flags_to_heap(bufmgr, flags);
796 
797    unsigned alloc_size = size;
798 
799    /* Always use slabs for sizes less than 4 KB because the kernel aligns
800     * everything to 4 KB.
801     */
802    if (size < alignment && alignment <= 4 * 1024)
803       alloc_size = alignment;
804 
805    if (alignment > get_slab_entry_alignment(bufmgr, alloc_size)) {
806       /* 3/4 allocations can return too small alignment.
807        * Try again with a power of two allocation size.
808        */
809       unsigned pot_size = get_slab_pot_entry_size(bufmgr, alloc_size);
810 
811       if (alignment <= pot_size) {
812          /* This size works but wastes some memory to fulfill the alignment. */
813          alloc_size = pot_size;
814       } else {
815          /* can't fulfill alignment requirements */
816          return NULL;
817       }
818    }
819 
820    struct pb_slabs *slabs = get_slabs(bufmgr, alloc_size);
821    entry = pb_slab_alloc(slabs, alloc_size, heap);
822    if (!entry) {
823       /* Clean up and try again... */
824       pb_slabs_reclaim(slabs);
825 
826       entry = pb_slab_alloc(slabs, alloc_size, heap);
827    }
828    if (!entry)
829       return NULL;
830 
831    struct iris_bo *bo = container_of(entry, struct iris_bo, slab.entry);
832 
833    if (bo->aux_map_address && bo->bufmgr->aux_map_ctx) {
834       /* This buffer was associated with an aux-buffer range.  We only allow
835        * slab allocated buffers to be reclaimed when idle (not in use by an
836        * executing batch).  (See iris_can_reclaim_slab().)  So we know that
837        * our previous aux mapping is no longer in use, and we can safely
838        * remove it.
839        */
840       intel_aux_map_unmap_range(bo->bufmgr->aux_map_ctx, bo->address,
841                                 bo->size);
842       bo->aux_map_address = 0;
843    }
844 
845    p_atomic_set(&bo->refcount, 1);
846    bo->name = name;
847    bo->size = size;
848 
849    /* Zero the contents if necessary.  If this fails, fall back to
850     * allocating a fresh BO, which will always be zeroed by the kernel.
851     */
852    if (flags & BO_ALLOC_ZEROED) {
853       void *map = iris_bo_map(NULL, bo, MAP_WRITE | MAP_RAW);
854       if (map) {
855          memset(map, 0, bo->size);
856       } else {
857          pb_slab_free(slabs, &bo->slab.entry);
858          return NULL;
859       }
860    }
861 
862    return bo;
863 }
864 
865 static struct iris_bo *
alloc_bo_from_cache(struct iris_bufmgr * bufmgr,struct bo_cache_bucket * bucket,uint32_t alignment,enum iris_memory_zone memzone,enum iris_mmap_mode mmap_mode,unsigned flags,bool match_zone)866 alloc_bo_from_cache(struct iris_bufmgr *bufmgr,
867                     struct bo_cache_bucket *bucket,
868                     uint32_t alignment,
869                     enum iris_memory_zone memzone,
870                     enum iris_mmap_mode mmap_mode,
871                     unsigned flags,
872                     bool match_zone)
873 {
874    if (!bucket)
875       return NULL;
876 
877    struct iris_bo *bo = NULL;
878 
879    simple_mtx_assert_locked(&bufmgr->lock);
880 
881    list_for_each_entry_safe(struct iris_bo, cur, &bucket->head, head) {
882       assert(iris_bo_is_real(cur));
883 
884       /* Find one that's got the right mapping type.  We used to swap maps
885        * around but the kernel doesn't allow this on discrete GPUs.
886        */
887       if (mmap_mode != cur->real.mmap_mode)
888          continue;
889 
890       /* Try a little harder to find one that's already in the right memzone */
891       if (match_zone && memzone != iris_memzone_for_address(cur->address))
892          continue;
893 
894       /* If the last BO in the cache is busy, there are no idle BOs.  Bail,
895        * either falling back to a non-matching memzone, or if that fails,
896        * allocating a fresh buffer.
897        */
898       if (iris_bo_busy(cur))
899          return NULL;
900 
901       list_del(&cur->head);
902 
903       /* Tell the kernel we need this BO.  If it still exists, we're done! */
904       if (iris_bo_madvise(cur, I915_MADV_WILLNEED)) {
905          bo = cur;
906          break;
907       }
908 
909       /* This BO was purged, throw it out and keep looking. */
910       bo_free(cur);
911    }
912 
913    if (!bo)
914       return NULL;
915 
916    if (bo->aux_map_address) {
917       /* This buffer was associated with an aux-buffer range. We make sure
918        * that buffers are not reused from the cache while the buffer is (busy)
919        * being used by an executing batch. Since we are here, the buffer is no
920        * longer being used by a batch and the buffer was deleted (in order to
921        * end up in the cache). Therefore its old aux-buffer range can be
922        * removed from the aux-map.
923        */
924       if (bo->bufmgr->aux_map_ctx)
925          intel_aux_map_unmap_range(bo->bufmgr->aux_map_ctx, bo->address,
926                                    bo->size);
927       bo->aux_map_address = 0;
928    }
929 
930    /* If the cached BO isn't in the right memory zone, or the alignment
931     * isn't sufficient, free the old memory and assign it a new address.
932     */
933    if (memzone != iris_memzone_for_address(bo->address) ||
934        bo->address % alignment != 0) {
935       vma_free(bufmgr, bo->address, bo->size);
936       bo->address = 0ull;
937    }
938 
939    /* Zero the contents if necessary.  If this fails, fall back to
940     * allocating a fresh BO, which will always be zeroed by the kernel.
941     */
942    if (flags & BO_ALLOC_ZEROED) {
943       void *map = iris_bo_map(NULL, bo, MAP_WRITE | MAP_RAW);
944       if (map) {
945          memset(map, 0, bo->size);
946       } else {
947          bo_free(bo);
948          return NULL;
949       }
950    }
951 
952    return bo;
953 }
954 
955 static struct iris_bo *
alloc_fresh_bo(struct iris_bufmgr * bufmgr,uint64_t bo_size,unsigned flags)956 alloc_fresh_bo(struct iris_bufmgr *bufmgr, uint64_t bo_size, unsigned flags)
957 {
958    struct iris_bo *bo = bo_calloc();
959    if (!bo)
960       return NULL;
961 
962    bo->real.heap = flags_to_heap(bufmgr, flags);
963 
964    /* If we have vram size, we have multiple memory regions and should choose
965     * one of them.
966     */
967    if (bufmgr->vram.size > 0) {
968       /* All new BOs we get from the kernel are zeroed, so we don't need to
969        * worry about that here.
970        */
971       struct drm_i915_gem_memory_class_instance regions[2];
972       uint32_t nregions = 0;
973       switch (bo->real.heap) {
974       case IRIS_HEAP_DEVICE_LOCAL_PREFERRED:
975          /* For vram allocations, still use system memory as a fallback. */
976          regions[nregions++] = bufmgr->vram.region;
977          regions[nregions++] = bufmgr->sys.region;
978          break;
979       case IRIS_HEAP_DEVICE_LOCAL:
980          regions[nregions++] = bufmgr->vram.region;
981          break;
982       case IRIS_HEAP_SYSTEM_MEMORY:
983          regions[nregions++] = bufmgr->sys.region;
984          break;
985       case IRIS_HEAP_MAX:
986          unreachable("invalid heap for BO");
987       }
988 
989       struct drm_i915_gem_create_ext_memory_regions ext_regions = {
990          .base = { .name = I915_GEM_CREATE_EXT_MEMORY_REGIONS },
991          .num_regions = nregions,
992          .regions = (uintptr_t)regions,
993       };
994 
995       struct drm_i915_gem_create_ext create = {
996          .size = bo_size,
997          .extensions = (uintptr_t)&ext_regions,
998       };
999 
1000       if (!bufmgr->all_vram_mappable &&
1001           bo->real.heap == IRIS_HEAP_DEVICE_LOCAL_PREFERRED) {
1002          create.flags |= I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS;
1003       }
1004 
1005       /* It should be safe to use GEM_CREATE_EXT without checking, since we are
1006        * in the side of the branch where discrete memory is available. So we
1007        * can assume GEM_CREATE_EXT is supported already.
1008        */
1009       if (intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CREATE_EXT, &create) != 0) {
1010          free(bo);
1011          return NULL;
1012       }
1013       bo->gem_handle = create.handle;
1014    } else {
1015       struct drm_i915_gem_create create = { .size = bo_size };
1016 
1017       /* All new BOs we get from the kernel are zeroed, so we don't need to
1018        * worry about that here.
1019        */
1020       if (intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CREATE, &create) != 0) {
1021          free(bo);
1022          return NULL;
1023       }
1024       bo->gem_handle = create.handle;
1025    }
1026 
1027    bo->bufmgr = bufmgr;
1028    bo->size = bo_size;
1029    bo->idle = true;
1030 
1031    if (bufmgr->vram.size == 0) {
1032       /* Calling set_domain() will allocate pages for the BO outside of the
1033        * struct mutex lock in the kernel, which is more efficient than waiting
1034        * to create them during the first execbuf that uses the BO.
1035        */
1036       struct drm_i915_gem_set_domain sd = {
1037          .handle = bo->gem_handle,
1038          .read_domains = I915_GEM_DOMAIN_CPU,
1039          .write_domain = 0,
1040       };
1041 
1042       intel_ioctl(bo->bufmgr->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &sd);
1043    }
1044 
1045    return bo;
1046 }
1047 
1048 const char *
1049 iris_heap_to_string[IRIS_HEAP_MAX] = {
1050    [IRIS_HEAP_SYSTEM_MEMORY] = "system",
1051    [IRIS_HEAP_DEVICE_LOCAL] = "local",
1052    [IRIS_HEAP_DEVICE_LOCAL_PREFERRED] = "local-preferred",
1053 };
1054 
1055 struct iris_bo *
iris_bo_alloc(struct iris_bufmgr * bufmgr,const char * name,uint64_t size,uint32_t alignment,enum iris_memory_zone memzone,unsigned flags)1056 iris_bo_alloc(struct iris_bufmgr *bufmgr,
1057               const char *name,
1058               uint64_t size,
1059               uint32_t alignment,
1060               enum iris_memory_zone memzone,
1061               unsigned flags)
1062 {
1063    struct iris_bo *bo;
1064    unsigned int page_size = getpagesize();
1065    enum iris_heap heap = flags_to_heap(bufmgr, flags);
1066    bool local = heap != IRIS_HEAP_SYSTEM_MEMORY;
1067    struct bo_cache_bucket *bucket = bucket_for_size(bufmgr, size, heap);
1068 
1069    if (memzone != IRIS_MEMZONE_OTHER || (flags & BO_ALLOC_COHERENT))
1070       flags |= BO_ALLOC_NO_SUBALLOC;
1071 
1072    bo = alloc_bo_from_slabs(bufmgr, name, size, alignment, flags);
1073 
1074    if (bo)
1075       return bo;
1076 
1077    /* Round the size up to the bucket size, or if we don't have caching
1078     * at this size, a multiple of the page size.
1079     */
1080    uint64_t bo_size =
1081       bucket ? bucket->size : MAX2(ALIGN(size, page_size), page_size);
1082 
1083    bool is_coherent = bufmgr->has_llc ||
1084                       (bufmgr->vram.size > 0 && !local) ||
1085                       (flags & BO_ALLOC_COHERENT);
1086    bool is_scanout = (flags & BO_ALLOC_SCANOUT) != 0;
1087 
1088    enum iris_mmap_mode mmap_mode;
1089    if (!bufmgr->all_vram_mappable && heap == IRIS_HEAP_DEVICE_LOCAL)
1090       mmap_mode = IRIS_MMAP_NONE;
1091    else if (!local && is_coherent && !is_scanout)
1092       mmap_mode = IRIS_MMAP_WB;
1093    else
1094       mmap_mode = IRIS_MMAP_WC;
1095 
1096    simple_mtx_lock(&bufmgr->lock);
1097 
1098    /* Get a buffer out of the cache if available.  First, we try to find
1099     * one with a matching memory zone so we can avoid reallocating VMA.
1100     */
1101    bo = alloc_bo_from_cache(bufmgr, bucket, alignment, memzone, mmap_mode,
1102                             flags, true);
1103 
1104    /* If that fails, we try for any cached BO, without matching memzone. */
1105    if (!bo) {
1106       bo = alloc_bo_from_cache(bufmgr, bucket, alignment, memzone, mmap_mode,
1107                                flags, false);
1108    }
1109 
1110    simple_mtx_unlock(&bufmgr->lock);
1111 
1112    if (!bo) {
1113       bo = alloc_fresh_bo(bufmgr, bo_size, flags);
1114       if (!bo)
1115          return NULL;
1116    }
1117 
1118    if (bo->address == 0ull) {
1119       simple_mtx_lock(&bufmgr->lock);
1120       bo->address = vma_alloc(bufmgr, memzone, bo->size, alignment);
1121       simple_mtx_unlock(&bufmgr->lock);
1122 
1123       if (bo->address == 0ull)
1124          goto err_free;
1125    }
1126 
1127    bo->name = name;
1128    p_atomic_set(&bo->refcount, 1);
1129    bo->real.reusable = bucket && bufmgr->bo_reuse;
1130    bo->index = -1;
1131    bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
1132 
1133    /* By default, capture all driver-internal buffers like shader kernels,
1134     * surface states, dynamic states, border colors, and so on.
1135     */
1136    if (memzone < IRIS_MEMZONE_OTHER)
1137       bo->real.kflags |= EXEC_OBJECT_CAPTURE;
1138 
1139    assert(bo->real.map == NULL || bo->real.mmap_mode == mmap_mode);
1140    bo->real.mmap_mode = mmap_mode;
1141 
1142    /* On integrated GPUs, enable snooping to ensure coherency if needed.
1143     * For discrete, we instead use SMEM and avoid WB maps for coherency.
1144     */
1145    if ((flags & BO_ALLOC_COHERENT) &&
1146        !bufmgr->has_llc && bufmgr->vram.size == 0) {
1147       struct drm_i915_gem_caching arg = {
1148          .handle = bo->gem_handle,
1149          .caching = 1,
1150       };
1151       if (intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_SET_CACHING, &arg) != 0)
1152          goto err_free;
1153 
1154       bo->real.reusable = false;
1155    }
1156 
1157    DBG("bo_create: buf %d (%s) (%s memzone) (%s) %llub\n", bo->gem_handle,
1158        bo->name, memzone_name(memzone), iris_heap_to_string[bo->real.heap],
1159        (unsigned long long) size);
1160 
1161    return bo;
1162 
1163 err_free:
1164    simple_mtx_lock(&bufmgr->lock);
1165    bo_free(bo);
1166    simple_mtx_unlock(&bufmgr->lock);
1167    return NULL;
1168 }
1169 
1170 struct iris_bo *
iris_bo_create_userptr(struct iris_bufmgr * bufmgr,const char * name,void * ptr,size_t size,enum iris_memory_zone memzone)1171 iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
1172                        void *ptr, size_t size,
1173                        enum iris_memory_zone memzone)
1174 {
1175    struct drm_gem_close close = { 0, };
1176    struct iris_bo *bo;
1177 
1178    bo = bo_calloc();
1179    if (!bo)
1180       return NULL;
1181 
1182    struct drm_i915_gem_userptr arg = {
1183       .user_ptr = (uintptr_t)ptr,
1184       .user_size = size,
1185       .flags = bufmgr->has_userptr_probe ? I915_USERPTR_PROBE : 0,
1186    };
1187    if (intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_USERPTR, &arg))
1188       goto err_free;
1189    bo->gem_handle = arg.handle;
1190 
1191    if (!bufmgr->has_userptr_probe) {
1192       /* Check the buffer for validity before we try and use it in a batch */
1193       struct drm_i915_gem_set_domain sd = {
1194          .handle = bo->gem_handle,
1195          .read_domains = I915_GEM_DOMAIN_CPU,
1196       };
1197       if (intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &sd))
1198          goto err_close;
1199    }
1200 
1201    bo->name = name;
1202    bo->size = size;
1203    bo->real.map = ptr;
1204 
1205    bo->bufmgr = bufmgr;
1206    bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
1207 
1208    simple_mtx_lock(&bufmgr->lock);
1209    bo->address = vma_alloc(bufmgr, memzone, size, 1);
1210    simple_mtx_unlock(&bufmgr->lock);
1211 
1212    if (bo->address == 0ull)
1213       goto err_close;
1214 
1215    p_atomic_set(&bo->refcount, 1);
1216    bo->real.userptr = true;
1217    bo->index = -1;
1218    bo->idle = true;
1219    bo->real.mmap_mode = IRIS_MMAP_WB;
1220 
1221    return bo;
1222 
1223 err_close:
1224    close.handle = bo->gem_handle;
1225    intel_ioctl(bufmgr->fd, DRM_IOCTL_GEM_CLOSE, &close);
1226 err_free:
1227    free(bo);
1228    return NULL;
1229 }
1230 
1231 /**
1232  * Returns a iris_bo wrapping the given buffer object handle.
1233  *
1234  * This can be used when one application needs to pass a buffer object
1235  * to another.
1236  */
1237 struct iris_bo *
iris_bo_gem_create_from_name(struct iris_bufmgr * bufmgr,const char * name,unsigned int handle)1238 iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr,
1239                              const char *name, unsigned int handle)
1240 {
1241    struct iris_bo *bo;
1242 
1243    /* At the moment most applications only have a few named bo.
1244     * For instance, in a DRI client only the render buffers passed
1245     * between X and the client are named. And since X returns the
1246     * alternating names for the front/back buffer a linear search
1247     * provides a sufficiently fast match.
1248     */
1249    simple_mtx_lock(&bufmgr->lock);
1250    bo = find_and_ref_external_bo(bufmgr->name_table, handle);
1251    if (bo)
1252       goto out;
1253 
1254    struct drm_gem_open open_arg = { .name = handle };
1255    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_GEM_OPEN, &open_arg);
1256    if (ret != 0) {
1257       DBG("Couldn't reference %s handle 0x%08x: %s\n",
1258           name, handle, strerror(errno));
1259       bo = NULL;
1260       goto out;
1261    }
1262    /* Now see if someone has used a prime handle to get this
1263     * object from the kernel before by looking through the list
1264     * again for a matching gem_handle
1265     */
1266    bo = find_and_ref_external_bo(bufmgr->handle_table, open_arg.handle);
1267    if (bo)
1268       goto out;
1269 
1270    bo = bo_calloc();
1271    if (!bo) {
1272       struct drm_gem_close close = { .handle = open_arg.handle, };
1273       intel_ioctl(bufmgr->fd, DRM_IOCTL_GEM_CLOSE, &close);
1274       goto out;
1275    }
1276 
1277    p_atomic_set(&bo->refcount, 1);
1278 
1279    bo->size = open_arg.size;
1280    bo->bufmgr = bufmgr;
1281    bo->gem_handle = open_arg.handle;
1282    bo->name = name;
1283    bo->real.global_name = handle;
1284    bo->real.reusable = false;
1285    bo->real.imported = true;
1286    bo->real.mmap_mode = IRIS_MMAP_NONE;
1287    bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
1288    bo->address = vma_alloc(bufmgr, IRIS_MEMZONE_OTHER, bo->size, 1);
1289 
1290    if (bo->address == 0ull) {
1291       bo_free(bo);
1292       bo = NULL;
1293       goto out;
1294    }
1295 
1296    _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
1297    _mesa_hash_table_insert(bufmgr->name_table, &bo->real.global_name, bo);
1298 
1299    DBG("bo_create_from_handle: %d (%s)\n", handle, bo->name);
1300 
1301 out:
1302    simple_mtx_unlock(&bufmgr->lock);
1303    return bo;
1304 }
1305 
1306 static void
bo_close(struct iris_bo * bo)1307 bo_close(struct iris_bo *bo)
1308 {
1309    struct iris_bufmgr *bufmgr = bo->bufmgr;
1310 
1311    simple_mtx_assert_locked(&bufmgr->lock);
1312    assert(iris_bo_is_real(bo));
1313 
1314    if (iris_bo_is_external(bo)) {
1315       struct hash_entry *entry;
1316 
1317       if (bo->real.global_name) {
1318          entry = _mesa_hash_table_search(bufmgr->name_table,
1319                                          &bo->real.global_name);
1320          _mesa_hash_table_remove(bufmgr->name_table, entry);
1321       }
1322 
1323       entry = _mesa_hash_table_search(bufmgr->handle_table, &bo->gem_handle);
1324       _mesa_hash_table_remove(bufmgr->handle_table, entry);
1325 
1326       list_for_each_entry_safe(struct bo_export, export, &bo->real.exports, link) {
1327          struct drm_gem_close close = { .handle = export->gem_handle };
1328          intel_ioctl(export->drm_fd, DRM_IOCTL_GEM_CLOSE, &close);
1329 
1330          list_del(&export->link);
1331          free(export);
1332       }
1333    } else {
1334       assert(list_is_empty(&bo->real.exports));
1335    }
1336 
1337    /* Close this object */
1338    struct drm_gem_close close = { .handle = bo->gem_handle };
1339    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_GEM_CLOSE, &close);
1340    if (ret != 0) {
1341       DBG("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n",
1342           bo->gem_handle, bo->name, strerror(errno));
1343    }
1344 
1345    if (bo->aux_map_address && bo->bufmgr->aux_map_ctx) {
1346       intel_aux_map_unmap_range(bo->bufmgr->aux_map_ctx, bo->address,
1347                                 bo->size);
1348    }
1349 
1350    /* Return the VMA for reuse */
1351    vma_free(bo->bufmgr, bo->address, bo->size);
1352 
1353    for (int d = 0; d < bo->deps_size; d++) {
1354       for (int b = 0; b < IRIS_BATCH_COUNT; b++) {
1355          iris_syncobj_reference(bufmgr, &bo->deps[d].write_syncobjs[b], NULL);
1356          iris_syncobj_reference(bufmgr, &bo->deps[d].read_syncobjs[b], NULL);
1357       }
1358    }
1359    free(bo->deps);
1360 
1361    free(bo);
1362 }
1363 
1364 static void
bo_free(struct iris_bo * bo)1365 bo_free(struct iris_bo *bo)
1366 {
1367    struct iris_bufmgr *bufmgr = bo->bufmgr;
1368 
1369    simple_mtx_assert_locked(&bufmgr->lock);
1370    assert(iris_bo_is_real(bo));
1371 
1372    if (!bo->real.userptr && bo->real.map)
1373       bo_unmap(bo);
1374 
1375    if (bo->idle) {
1376       bo_close(bo);
1377    } else {
1378       /* Defer closing the GEM BO and returning the VMA for reuse until the
1379        * BO is idle.  Just move it to the dead list for now.
1380        */
1381       list_addtail(&bo->head, &bufmgr->zombie_list);
1382    }
1383 }
1384 
1385 /** Frees all cached buffers significantly older than @time. */
1386 static void
cleanup_bo_cache(struct iris_bufmgr * bufmgr,time_t time)1387 cleanup_bo_cache(struct iris_bufmgr *bufmgr, time_t time)
1388 {
1389    int i;
1390 
1391    simple_mtx_assert_locked(&bufmgr->lock);
1392 
1393    if (bufmgr->time == time)
1394       return;
1395 
1396    for (i = 0; i < bufmgr->num_buckets; i++) {
1397       struct bo_cache_bucket *bucket = &bufmgr->cache_bucket[i];
1398 
1399       list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1400          if (time - bo->real.free_time <= 1)
1401             break;
1402 
1403          list_del(&bo->head);
1404 
1405          bo_free(bo);
1406       }
1407    }
1408 
1409    for (i = 0; i < bufmgr->num_local_buckets; i++) {
1410       struct bo_cache_bucket *bucket = &bufmgr->local_cache_bucket[i];
1411 
1412       list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1413          if (time - bo->real.free_time <= 1)
1414             break;
1415 
1416          list_del(&bo->head);
1417 
1418          bo_free(bo);
1419       }
1420    }
1421 
1422    for (i = 0; i < bufmgr->num_local_preferred_buckets; i++) {
1423       struct bo_cache_bucket *bucket = &bufmgr->local_preferred_cache_bucket[i];
1424 
1425       list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1426          if (time - bo->real.free_time <= 1)
1427             break;
1428 
1429          list_del(&bo->head);
1430 
1431          bo_free(bo);
1432       }
1433    }
1434 
1435    list_for_each_entry_safe(struct iris_bo, bo, &bufmgr->zombie_list, head) {
1436       /* Stop once we reach a busy BO - all others past this point were
1437        * freed more recently so are likely also busy.
1438        */
1439       if (!bo->idle && iris_bo_busy(bo))
1440          break;
1441 
1442       list_del(&bo->head);
1443       bo_close(bo);
1444    }
1445 
1446    bufmgr->time = time;
1447 }
1448 
1449 static void
bo_unreference_final(struct iris_bo * bo,time_t time)1450 bo_unreference_final(struct iris_bo *bo, time_t time)
1451 {
1452    struct iris_bufmgr *bufmgr = bo->bufmgr;
1453    struct bo_cache_bucket *bucket;
1454 
1455    DBG("bo_unreference final: %d (%s)\n", bo->gem_handle, bo->name);
1456 
1457    assert(iris_bo_is_real(bo));
1458 
1459    bucket = NULL;
1460    if (bo->real.reusable)
1461       bucket = bucket_for_size(bufmgr, bo->size, bo->real.heap);
1462    /* Put the buffer into our internal cache for reuse if we can. */
1463    if (bucket && iris_bo_madvise(bo, I915_MADV_DONTNEED)) {
1464       bo->real.free_time = time;
1465       bo->name = NULL;
1466 
1467       list_addtail(&bo->head, &bucket->head);
1468    } else {
1469       bo_free(bo);
1470    }
1471 }
1472 
1473 void
iris_bo_unreference(struct iris_bo * bo)1474 iris_bo_unreference(struct iris_bo *bo)
1475 {
1476    if (bo == NULL)
1477       return;
1478 
1479    assert(p_atomic_read(&bo->refcount) > 0);
1480 
1481    if (atomic_add_unless(&bo->refcount, -1, 1)) {
1482       struct iris_bufmgr *bufmgr = bo->bufmgr;
1483       struct timespec time;
1484 
1485       clock_gettime(CLOCK_MONOTONIC, &time);
1486 
1487       if (bo->gem_handle == 0) {
1488          pb_slab_free(get_slabs(bufmgr, bo->size), &bo->slab.entry);
1489       } else {
1490          simple_mtx_lock(&bufmgr->lock);
1491 
1492          if (p_atomic_dec_zero(&bo->refcount)) {
1493             bo_unreference_final(bo, time.tv_sec);
1494             cleanup_bo_cache(bufmgr, time.tv_sec);
1495          }
1496 
1497          simple_mtx_unlock(&bufmgr->lock);
1498       }
1499    }
1500 }
1501 
1502 static void
bo_wait_with_stall_warning(struct util_debug_callback * dbg,struct iris_bo * bo,const char * action)1503 bo_wait_with_stall_warning(struct util_debug_callback *dbg,
1504                            struct iris_bo *bo,
1505                            const char *action)
1506 {
1507    bool busy = dbg && !bo->idle;
1508    double elapsed = unlikely(busy) ? -get_time() : 0.0;
1509 
1510    iris_bo_wait_rendering(bo);
1511 
1512    if (unlikely(busy)) {
1513       elapsed += get_time();
1514       if (elapsed > 1e-5) /* 0.01ms */ {
1515          perf_debug(dbg, "%s a busy \"%s\" BO stalled and took %.03f ms.\n",
1516                     action, bo->name, elapsed * 1000);
1517       }
1518    }
1519 }
1520 
1521 static void
print_flags(unsigned flags)1522 print_flags(unsigned flags)
1523 {
1524    if (flags & MAP_READ)
1525       DBG("READ ");
1526    if (flags & MAP_WRITE)
1527       DBG("WRITE ");
1528    if (flags & MAP_ASYNC)
1529       DBG("ASYNC ");
1530    if (flags & MAP_PERSISTENT)
1531       DBG("PERSISTENT ");
1532    if (flags & MAP_COHERENT)
1533       DBG("COHERENT ");
1534    if (flags & MAP_RAW)
1535       DBG("RAW ");
1536    DBG("\n");
1537 }
1538 
1539 static void *
iris_bo_gem_mmap_legacy(struct util_debug_callback * dbg,struct iris_bo * bo)1540 iris_bo_gem_mmap_legacy(struct util_debug_callback *dbg, struct iris_bo *bo)
1541 {
1542    struct iris_bufmgr *bufmgr = bo->bufmgr;
1543 
1544    assert(bufmgr->vram.size == 0);
1545    assert(iris_bo_is_real(bo));
1546    assert(bo->real.mmap_mode == IRIS_MMAP_WB ||
1547           bo->real.mmap_mode == IRIS_MMAP_WC);
1548 
1549    struct drm_i915_gem_mmap mmap_arg = {
1550       .handle = bo->gem_handle,
1551       .size = bo->size,
1552       .flags = bo->real.mmap_mode == IRIS_MMAP_WC ? I915_MMAP_WC : 0,
1553    };
1554 
1555    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg);
1556    if (ret != 0) {
1557       DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1558           __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
1559       return NULL;
1560    }
1561    void *map = (void *) (uintptr_t) mmap_arg.addr_ptr;
1562 
1563    return map;
1564 }
1565 
1566 static void *
iris_bo_gem_mmap_offset(struct util_debug_callback * dbg,struct iris_bo * bo)1567 iris_bo_gem_mmap_offset(struct util_debug_callback *dbg, struct iris_bo *bo)
1568 {
1569    struct iris_bufmgr *bufmgr = bo->bufmgr;
1570 
1571    assert(iris_bo_is_real(bo));
1572 
1573    struct drm_i915_gem_mmap_offset mmap_arg = {
1574       .handle = bo->gem_handle,
1575    };
1576 
1577    if (bufmgr->has_local_mem) {
1578       /* On discrete memory platforms, we cannot control the mmap caching mode
1579        * at mmap time.  Instead, it's fixed when the object is created (this
1580        * is a limitation of TTM).
1581        *
1582        * On DG1, our only currently enabled discrete platform, there is no
1583        * control over what mode we get.  For SMEM, we always get WB because
1584        * it's fast (probably what we want) and when the device views SMEM
1585        * across PCIe, it's always snooped.  The only caching mode allowed by
1586        * DG1 hardware for LMEM is WC.
1587        */
1588       if (bo->real.heap != IRIS_HEAP_SYSTEM_MEMORY)
1589          assert(bo->real.mmap_mode == IRIS_MMAP_WC);
1590       else
1591          assert(bo->real.mmap_mode == IRIS_MMAP_WB);
1592 
1593       mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
1594    } else {
1595       /* Only integrated platforms get to select a mmap caching mode here */
1596       static const uint32_t mmap_offset_for_mode[] = {
1597          [IRIS_MMAP_UC]    = I915_MMAP_OFFSET_UC,
1598          [IRIS_MMAP_WC]    = I915_MMAP_OFFSET_WC,
1599          [IRIS_MMAP_WB]    = I915_MMAP_OFFSET_WB,
1600       };
1601       assert(bo->real.mmap_mode != IRIS_MMAP_NONE);
1602       assert(bo->real.mmap_mode < ARRAY_SIZE(mmap_offset_for_mode));
1603       mmap_arg.flags = mmap_offset_for_mode[bo->real.mmap_mode];
1604    }
1605 
1606    /* Get the fake offset back */
1607    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP_OFFSET, &mmap_arg);
1608    if (ret != 0) {
1609       DBG("%s:%d: Error preparing buffer %d (%s): %s .\n",
1610           __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
1611       return NULL;
1612    }
1613 
1614    /* And map it */
1615    void *map = mmap(0, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
1616                     bufmgr->fd, mmap_arg.offset);
1617    if (map == MAP_FAILED) {
1618       DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1619           __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
1620       return NULL;
1621    }
1622 
1623    return map;
1624 }
1625 
1626 void *
iris_bo_map(struct util_debug_callback * dbg,struct iris_bo * bo,unsigned flags)1627 iris_bo_map(struct util_debug_callback *dbg,
1628             struct iris_bo *bo, unsigned flags)
1629 {
1630    struct iris_bufmgr *bufmgr = bo->bufmgr;
1631    void *map = NULL;
1632 
1633    if (bo->gem_handle == 0) {
1634       struct iris_bo *real = iris_get_backing_bo(bo);
1635       uint64_t offset = bo->address - real->address;
1636       map = iris_bo_map(dbg, real, flags | MAP_ASYNC) + offset;
1637    } else {
1638       assert(bo->real.mmap_mode != IRIS_MMAP_NONE);
1639       if (bo->real.mmap_mode == IRIS_MMAP_NONE)
1640          return NULL;
1641 
1642       if (!bo->real.map) {
1643          DBG("iris_bo_map: %d (%s)\n", bo->gem_handle, bo->name);
1644          map = bufmgr->has_mmap_offset ? iris_bo_gem_mmap_offset(dbg, bo)
1645                                        : iris_bo_gem_mmap_legacy(dbg, bo);
1646          if (!map) {
1647             return NULL;
1648          }
1649 
1650          VG_DEFINED(map, bo->size);
1651 
1652          if (p_atomic_cmpxchg(&bo->real.map, NULL, map)) {
1653             VG_NOACCESS(map, bo->size);
1654             os_munmap(map, bo->size);
1655          }
1656       }
1657       assert(bo->real.map);
1658       map = bo->real.map;
1659    }
1660 
1661    DBG("iris_bo_map: %d (%s) -> %p\n",
1662        bo->gem_handle, bo->name, bo->real.map);
1663    print_flags(flags);
1664 
1665    if (!(flags & MAP_ASYNC)) {
1666       bo_wait_with_stall_warning(dbg, bo, "memory mapping");
1667    }
1668 
1669    return map;
1670 }
1671 
1672 /** Waits for all GPU rendering with the object to have completed. */
1673 void
iris_bo_wait_rendering(struct iris_bo * bo)1674 iris_bo_wait_rendering(struct iris_bo *bo)
1675 {
1676    /* We require a kernel recent enough for WAIT_IOCTL support.
1677     * See intel_init_bufmgr()
1678     */
1679    iris_bo_wait(bo, -1);
1680 }
1681 
1682 static int
iris_bo_wait_gem(struct iris_bo * bo,int64_t timeout_ns)1683 iris_bo_wait_gem(struct iris_bo *bo, int64_t timeout_ns)
1684 {
1685    assert(iris_bo_is_real(bo));
1686 
1687    struct iris_bufmgr *bufmgr = bo->bufmgr;
1688    struct drm_i915_gem_wait wait = {
1689       .bo_handle = bo->gem_handle,
1690       .timeout_ns = timeout_ns,
1691    };
1692 
1693    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
1694    if (ret != 0)
1695       return -errno;
1696 
1697    return 0;
1698 }
1699 
1700 /**
1701  * Waits on a BO for the given amount of time.
1702  *
1703  * @bo: buffer object to wait for
1704  * @timeout_ns: amount of time to wait in nanoseconds.
1705  *   If value is less than 0, an infinite wait will occur.
1706  *
1707  * Returns 0 if the wait was successful ie. the last batch referencing the
1708  * object has completed within the allotted time. Otherwise some negative return
1709  * value describes the error. Of particular interest is -ETIME when the wait has
1710  * failed to yield the desired result.
1711  *
1712  * Similar to iris_bo_wait_rendering except a timeout parameter allows
1713  * the operation to give up after a certain amount of time. Another subtle
1714  * difference is the internal locking semantics are different (this variant does
1715  * not hold the lock for the duration of the wait). This makes the wait subject
1716  * to a larger userspace race window.
1717  *
1718  * The implementation shall wait until the object is no longer actively
1719  * referenced within a batch buffer at the time of the call. The wait will
1720  * not guarantee that the buffer is re-issued via another thread, or an flinked
1721  * handle. Userspace must make sure this race does not occur if such precision
1722  * is important.
1723  *
1724  * Note that some kernels have broken the infinite wait for negative values
1725  * promise, upgrade to latest stable kernels if this is the case.
1726  */
1727 int
iris_bo_wait(struct iris_bo * bo,int64_t timeout_ns)1728 iris_bo_wait(struct iris_bo *bo, int64_t timeout_ns)
1729 {
1730    int ret;
1731 
1732    if (iris_bo_is_external(bo))
1733       ret = iris_bo_wait_gem(bo, timeout_ns);
1734    else
1735       ret = iris_bo_wait_syncobj(bo, timeout_ns);
1736 
1737    if (ret != 0)
1738       return -errno;
1739 
1740    bo->idle = true;
1741 
1742    return ret;
1743 }
1744 
1745 static void
iris_bufmgr_destroy(struct iris_bufmgr * bufmgr)1746 iris_bufmgr_destroy(struct iris_bufmgr *bufmgr)
1747 {
1748    iris_destroy_border_color_pool(&bufmgr->border_color_pool);
1749 
1750    /* Free aux-map buffers */
1751    intel_aux_map_finish(bufmgr->aux_map_ctx);
1752 
1753    /* bufmgr will no longer try to free VMA entries in the aux-map */
1754    bufmgr->aux_map_ctx = NULL;
1755 
1756    for (int i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
1757       if (bufmgr->bo_slabs[i].groups)
1758          pb_slabs_deinit(&bufmgr->bo_slabs[i]);
1759    }
1760 
1761    simple_mtx_lock(&bufmgr->lock);
1762    /* Free any cached buffer objects we were going to reuse */
1763    for (int i = 0; i < bufmgr->num_buckets; i++) {
1764       struct bo_cache_bucket *bucket = &bufmgr->cache_bucket[i];
1765 
1766       list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1767          list_del(&bo->head);
1768 
1769          bo_free(bo);
1770       }
1771    }
1772 
1773    for (int i = 0; i < bufmgr->num_local_buckets; i++) {
1774       struct bo_cache_bucket *bucket = &bufmgr->local_cache_bucket[i];
1775 
1776       list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1777          list_del(&bo->head);
1778 
1779          bo_free(bo);
1780       }
1781    }
1782 
1783    for (int i = 0; i < bufmgr->num_local_preferred_buckets; i++) {
1784       struct bo_cache_bucket *bucket = &bufmgr->local_preferred_cache_bucket[i];
1785 
1786       list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1787          list_del(&bo->head);
1788 
1789          bo_free(bo);
1790       }
1791    }
1792 
1793    /* Close any buffer objects on the dead list. */
1794    list_for_each_entry_safe(struct iris_bo, bo, &bufmgr->zombie_list, head) {
1795       list_del(&bo->head);
1796       bo_close(bo);
1797    }
1798 
1799    _mesa_hash_table_destroy(bufmgr->name_table, NULL);
1800    _mesa_hash_table_destroy(bufmgr->handle_table, NULL);
1801 
1802    for (int z = 0; z < IRIS_MEMZONE_COUNT; z++)
1803          util_vma_heap_finish(&bufmgr->vma_allocator[z]);
1804 
1805    close(bufmgr->fd);
1806 
1807    simple_mtx_unlock(&bufmgr->lock);
1808 
1809    simple_mtx_destroy(&bufmgr->lock);
1810    simple_mtx_destroy(&bufmgr->bo_deps_lock);
1811 
1812    free(bufmgr);
1813 }
1814 
1815 int
iris_gem_get_tiling(struct iris_bo * bo,uint32_t * tiling)1816 iris_gem_get_tiling(struct iris_bo *bo, uint32_t *tiling)
1817 {
1818    struct iris_bufmgr *bufmgr = bo->bufmgr;
1819 
1820    if (!bufmgr->has_tiling_uapi) {
1821       *tiling = I915_TILING_NONE;
1822       return 0;
1823    }
1824 
1825    struct drm_i915_gem_get_tiling ti = { .handle = bo->gem_handle };
1826    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_GET_TILING, &ti);
1827 
1828    if (ret) {
1829       DBG("gem_get_tiling failed for BO %u: %s\n",
1830           bo->gem_handle, strerror(errno));
1831    }
1832 
1833    *tiling = ti.tiling_mode;
1834 
1835    return ret;
1836 }
1837 
1838 int
iris_gem_set_tiling(struct iris_bo * bo,const struct isl_surf * surf)1839 iris_gem_set_tiling(struct iris_bo *bo, const struct isl_surf *surf)
1840 {
1841    struct iris_bufmgr *bufmgr = bo->bufmgr;
1842    uint32_t tiling_mode = isl_tiling_to_i915_tiling(surf->tiling);
1843    int ret;
1844 
1845    /* If we can't do map_gtt, the set/get_tiling API isn't useful. And it's
1846     * actually not supported by the kernel in those cases.
1847     */
1848    if (!bufmgr->has_tiling_uapi)
1849       return 0;
1850 
1851    /* GEM_SET_TILING is slightly broken and overwrites the input on the
1852     * error path, so we have to open code intel_ioctl().
1853     */
1854    do {
1855       struct drm_i915_gem_set_tiling set_tiling = {
1856          .handle = bo->gem_handle,
1857          .tiling_mode = tiling_mode,
1858          .stride = surf->row_pitch_B,
1859       };
1860       ret = ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling);
1861    } while (ret == -1 && (errno == EINTR || errno == EAGAIN));
1862 
1863    if (ret) {
1864       DBG("gem_set_tiling failed for BO %u: %s\n",
1865           bo->gem_handle, strerror(errno));
1866    }
1867 
1868    return ret;
1869 }
1870 
1871 struct iris_bo *
iris_bo_import_dmabuf(struct iris_bufmgr * bufmgr,int prime_fd)1872 iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd)
1873 {
1874    uint32_t handle;
1875    struct iris_bo *bo;
1876 
1877    simple_mtx_lock(&bufmgr->lock);
1878    int ret = drmPrimeFDToHandle(bufmgr->fd, prime_fd, &handle);
1879    if (ret) {
1880       DBG("import_dmabuf: failed to obtain handle from fd: %s\n",
1881           strerror(errno));
1882       simple_mtx_unlock(&bufmgr->lock);
1883       return NULL;
1884    }
1885 
1886    /*
1887     * See if the kernel has already returned this buffer to us. Just as
1888     * for named buffers, we must not create two bo's pointing at the same
1889     * kernel object
1890     */
1891    bo = find_and_ref_external_bo(bufmgr->handle_table, handle);
1892    if (bo)
1893       goto out;
1894 
1895    bo = bo_calloc();
1896    if (!bo)
1897       goto out;
1898 
1899    p_atomic_set(&bo->refcount, 1);
1900 
1901    /* Determine size of bo.  The fd-to-handle ioctl really should
1902     * return the size, but it doesn't.  If we have kernel 3.12 or
1903     * later, we can lseek on the prime fd to get the size.  Older
1904     * kernels will just fail, in which case we fall back to the
1905     * provided (estimated or guess size). */
1906    ret = lseek(prime_fd, 0, SEEK_END);
1907    if (ret != -1)
1908       bo->size = ret;
1909 
1910    bo->bufmgr = bufmgr;
1911    bo->name = "prime";
1912    bo->real.reusable = false;
1913    bo->real.imported = true;
1914    bo->real.mmap_mode = IRIS_MMAP_NONE;
1915    bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
1916    bo->gem_handle = handle;
1917 
1918    /* From the Bspec, Memory Compression - Gfx12:
1919     *
1920     *    The base address for the surface has to be 64K page aligned and the
1921     *    surface is expected to be padded in the virtual domain to be 4 4K
1922     *    pages.
1923     *
1924     * The dmabuf may contain a compressed surface. Align the BO to 64KB just
1925     * in case. We always align to 64KB even on platforms where we don't need
1926     * to, because it's a fairly reasonable thing to do anyway.
1927     */
1928    bo->address = vma_alloc(bufmgr, IRIS_MEMZONE_OTHER, bo->size, 64 * 1024);
1929 
1930    if (bo->address == 0ull) {
1931       bo_free(bo);
1932       bo = NULL;
1933       goto out;
1934    }
1935 
1936    _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
1937 
1938 out:
1939    simple_mtx_unlock(&bufmgr->lock);
1940    return bo;
1941 }
1942 
1943 static void
iris_bo_mark_exported_locked(struct iris_bo * bo)1944 iris_bo_mark_exported_locked(struct iris_bo *bo)
1945 {
1946    struct iris_bufmgr *bufmgr = bo->bufmgr;
1947 
1948    /* We cannot export suballocated BOs. */
1949    assert(iris_bo_is_real(bo));
1950    simple_mtx_assert_locked(&bufmgr->lock);
1951 
1952    if (!iris_bo_is_external(bo))
1953       _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
1954 
1955    if (!bo->real.exported) {
1956       /* If a BO is going to be used externally, it could be sent to the
1957        * display HW. So make sure our CPU mappings don't assume cache
1958        * coherency since display is outside that cache.
1959        */
1960       bo->real.exported = true;
1961       bo->real.reusable = false;
1962    }
1963 }
1964 
1965 void
iris_bo_mark_exported(struct iris_bo * bo)1966 iris_bo_mark_exported(struct iris_bo *bo)
1967 {
1968    struct iris_bufmgr *bufmgr = bo->bufmgr;
1969 
1970    /* We cannot export suballocated BOs. */
1971    assert(iris_bo_is_real(bo));
1972 
1973    if (bo->real.exported) {
1974       assert(!bo->real.reusable);
1975       return;
1976    }
1977 
1978    simple_mtx_lock(&bufmgr->lock);
1979    iris_bo_mark_exported_locked(bo);
1980    simple_mtx_unlock(&bufmgr->lock);
1981 }
1982 
1983 int
iris_bo_export_dmabuf(struct iris_bo * bo,int * prime_fd)1984 iris_bo_export_dmabuf(struct iris_bo *bo, int *prime_fd)
1985 {
1986    struct iris_bufmgr *bufmgr = bo->bufmgr;
1987 
1988    /* We cannot export suballocated BOs. */
1989    assert(iris_bo_is_real(bo));
1990 
1991    iris_bo_mark_exported(bo);
1992 
1993    if (drmPrimeHandleToFD(bufmgr->fd, bo->gem_handle,
1994                           DRM_CLOEXEC | DRM_RDWR, prime_fd) != 0)
1995       return -errno;
1996 
1997    return 0;
1998 }
1999 
2000 uint32_t
iris_bo_export_gem_handle(struct iris_bo * bo)2001 iris_bo_export_gem_handle(struct iris_bo *bo)
2002 {
2003    /* We cannot export suballocated BOs. */
2004    assert(iris_bo_is_real(bo));
2005 
2006    iris_bo_mark_exported(bo);
2007 
2008    return bo->gem_handle;
2009 }
2010 
2011 int
iris_bo_flink(struct iris_bo * bo,uint32_t * name)2012 iris_bo_flink(struct iris_bo *bo, uint32_t *name)
2013 {
2014    struct iris_bufmgr *bufmgr = bo->bufmgr;
2015 
2016    /* We cannot export suballocated BOs. */
2017    assert(iris_bo_is_real(bo));
2018 
2019    if (!bo->real.global_name) {
2020       struct drm_gem_flink flink = { .handle = bo->gem_handle };
2021 
2022       if (intel_ioctl(bufmgr->fd, DRM_IOCTL_GEM_FLINK, &flink))
2023          return -errno;
2024 
2025       simple_mtx_lock(&bufmgr->lock);
2026       if (!bo->real.global_name) {
2027          iris_bo_mark_exported_locked(bo);
2028          bo->real.global_name = flink.name;
2029          _mesa_hash_table_insert(bufmgr->name_table, &bo->real.global_name, bo);
2030       }
2031       simple_mtx_unlock(&bufmgr->lock);
2032    }
2033 
2034    *name = bo->real.global_name;
2035    return 0;
2036 }
2037 
2038 int
iris_bo_export_gem_handle_for_device(struct iris_bo * bo,int drm_fd,uint32_t * out_handle)2039 iris_bo_export_gem_handle_for_device(struct iris_bo *bo, int drm_fd,
2040                                      uint32_t *out_handle)
2041 {
2042    /* We cannot export suballocated BOs. */
2043    assert(iris_bo_is_real(bo));
2044 
2045    /* Only add the new GEM handle to the list of export if it belongs to a
2046     * different GEM device. Otherwise we might close the same buffer multiple
2047     * times.
2048     */
2049    struct iris_bufmgr *bufmgr = bo->bufmgr;
2050    int ret = os_same_file_description(drm_fd, bufmgr->fd);
2051    WARN_ONCE(ret < 0,
2052              "Kernel has no file descriptor comparison support: %s\n",
2053              strerror(errno));
2054    if (ret == 0) {
2055       *out_handle = iris_bo_export_gem_handle(bo);
2056       return 0;
2057    }
2058 
2059    struct bo_export *export = calloc(1, sizeof(*export));
2060    if (!export)
2061       return -ENOMEM;
2062 
2063    export->drm_fd = drm_fd;
2064 
2065    int dmabuf_fd = -1;
2066    int err = iris_bo_export_dmabuf(bo, &dmabuf_fd);
2067    if (err) {
2068       free(export);
2069       return err;
2070    }
2071 
2072    simple_mtx_lock(&bufmgr->lock);
2073    err = drmPrimeFDToHandle(drm_fd, dmabuf_fd, &export->gem_handle);
2074    close(dmabuf_fd);
2075    if (err) {
2076       simple_mtx_unlock(&bufmgr->lock);
2077       free(export);
2078       return err;
2079    }
2080 
2081    bool found = false;
2082    list_for_each_entry(struct bo_export, iter, &bo->real.exports, link) {
2083       if (iter->drm_fd != drm_fd)
2084          continue;
2085       /* Here we assume that for a given DRM fd, we'll always get back the
2086        * same GEM handle for a given buffer.
2087        */
2088       assert(iter->gem_handle == export->gem_handle);
2089       free(export);
2090       export = iter;
2091       found = true;
2092       break;
2093    }
2094    if (!found)
2095       list_addtail(&export->link, &bo->real.exports);
2096 
2097    simple_mtx_unlock(&bufmgr->lock);
2098 
2099    *out_handle = export->gem_handle;
2100 
2101    return 0;
2102 }
2103 
2104 static void
add_bucket(struct iris_bufmgr * bufmgr,int size,enum iris_heap heap)2105 add_bucket(struct iris_bufmgr *bufmgr, int size, enum iris_heap heap)
2106 {
2107    int *num_buckets;
2108    struct bo_cache_bucket *buckets;
2109    bucket_info_for_heap(bufmgr, heap, &buckets, &num_buckets);
2110 
2111    unsigned int i = (*num_buckets)++;
2112 
2113    list_inithead(&buckets[i].head);
2114    buckets[i].size = size;
2115 
2116    assert(bucket_for_size(bufmgr, size, heap) == &buckets[i]);
2117    assert(bucket_for_size(bufmgr, size - 2048, heap) == &buckets[i]);
2118    assert(bucket_for_size(bufmgr, size + 1, heap) != &buckets[i]);
2119 }
2120 
2121 static void
init_cache_buckets(struct iris_bufmgr * bufmgr,enum iris_heap heap)2122 init_cache_buckets(struct iris_bufmgr *bufmgr, enum iris_heap heap)
2123 {
2124    uint64_t size, cache_max_size = 64 * 1024 * 1024;
2125 
2126    /* OK, so power of two buckets was too wasteful of memory.
2127     * Give 3 other sizes between each power of two, to hopefully
2128     * cover things accurately enough.  (The alternative is
2129     * probably to just go for exact matching of sizes, and assume
2130     * that for things like composited window resize the tiled
2131     * width/height alignment and rounding of sizes to pages will
2132     * get us useful cache hit rates anyway)
2133     */
2134    add_bucket(bufmgr, PAGE_SIZE, heap);
2135    add_bucket(bufmgr, PAGE_SIZE * 2, heap);
2136    add_bucket(bufmgr, PAGE_SIZE * 3, heap);
2137 
2138    /* Initialize the linked lists for BO reuse cache. */
2139    for (size = 4 * PAGE_SIZE; size <= cache_max_size; size *= 2) {
2140       add_bucket(bufmgr, size, heap);
2141 
2142       add_bucket(bufmgr, size + size * 1 / 4, heap);
2143       add_bucket(bufmgr, size + size * 2 / 4, heap);
2144       add_bucket(bufmgr, size + size * 3 / 4, heap);
2145    }
2146 }
2147 
2148 void
iris_hw_context_set_unrecoverable(struct iris_bufmgr * bufmgr,uint32_t ctx_id)2149 iris_hw_context_set_unrecoverable(struct iris_bufmgr *bufmgr,
2150                                   uint32_t ctx_id)
2151 {
2152    /* Upon declaring a GPU hang, the kernel will zap the guilty context
2153     * back to the default logical HW state and attempt to continue on to
2154     * our next submitted batchbuffer.  However, our render batches assume
2155     * the previous GPU state is preserved, and only emit commands needed
2156     * to incrementally change that state.  In particular, we inherit the
2157     * STATE_BASE_ADDRESS and PIPELINE_SELECT settings, which are critical.
2158     * With default base addresses, our next batches will almost certainly
2159     * cause more GPU hangs, leading to repeated hangs until we're banned
2160     * or the machine is dead.
2161     *
2162     * Here we tell the kernel not to attempt to recover our context but
2163     * immediately (on the next batchbuffer submission) report that the
2164     * context is lost, and we will do the recovery ourselves.  Ideally,
2165     * we'll have two lost batches instead of a continual stream of hangs.
2166     */
2167    struct drm_i915_gem_context_param p = {
2168       .ctx_id = ctx_id,
2169       .param = I915_CONTEXT_PARAM_RECOVERABLE,
2170       .value = false,
2171    };
2172    intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM, &p);
2173 }
2174 
2175 void
iris_hw_context_set_vm_id(struct iris_bufmgr * bufmgr,uint32_t ctx_id)2176 iris_hw_context_set_vm_id(struct iris_bufmgr *bufmgr, uint32_t ctx_id)
2177 {
2178    if (!bufmgr->use_global_vm)
2179       return;
2180 
2181    struct drm_i915_gem_context_param p = {
2182       .ctx_id = ctx_id,
2183       .param = I915_CONTEXT_PARAM_VM,
2184       .value = bufmgr->global_vm_id,
2185    };
2186    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM, &p);
2187    if (ret != 0) {
2188       DBG("DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM failed: %s\n",
2189           strerror(errno));
2190    }
2191 }
2192 
2193 uint32_t
iris_create_hw_context(struct iris_bufmgr * bufmgr)2194 iris_create_hw_context(struct iris_bufmgr *bufmgr)
2195 {
2196    struct drm_i915_gem_context_create create = { };
2197    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &create);
2198    if (ret != 0) {
2199       DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n", strerror(errno));
2200       return 0;
2201    }
2202 
2203    iris_hw_context_set_unrecoverable(bufmgr, create.ctx_id);
2204    iris_hw_context_set_vm_id(bufmgr, create.ctx_id);
2205 
2206    return create.ctx_id;
2207 }
2208 
2209 int
iris_kernel_context_get_priority(struct iris_bufmgr * bufmgr,uint32_t ctx_id)2210 iris_kernel_context_get_priority(struct iris_bufmgr *bufmgr, uint32_t ctx_id)
2211 {
2212    struct drm_i915_gem_context_param p = {
2213       .ctx_id = ctx_id,
2214       .param = I915_CONTEXT_PARAM_PRIORITY,
2215    };
2216    intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM, &p);
2217    return p.value; /* on error, return 0 i.e. default priority */
2218 }
2219 
2220 int
iris_hw_context_set_priority(struct iris_bufmgr * bufmgr,uint32_t ctx_id,int priority)2221 iris_hw_context_set_priority(struct iris_bufmgr *bufmgr,
2222                             uint32_t ctx_id,
2223                             int priority)
2224 {
2225    struct drm_i915_gem_context_param p = {
2226       .ctx_id = ctx_id,
2227       .param = I915_CONTEXT_PARAM_PRIORITY,
2228       .value = priority,
2229    };
2230    int err;
2231 
2232    err = 0;
2233    if (intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM, &p))
2234       err = -errno;
2235 
2236    return err;
2237 }
2238 
2239 uint32_t
iris_clone_hw_context(struct iris_bufmgr * bufmgr,uint32_t ctx_id)2240 iris_clone_hw_context(struct iris_bufmgr *bufmgr, uint32_t ctx_id)
2241 {
2242    uint32_t new_ctx = iris_create_hw_context(bufmgr);
2243 
2244    if (new_ctx) {
2245       int priority = iris_kernel_context_get_priority(bufmgr, ctx_id);
2246       iris_hw_context_set_priority(bufmgr, new_ctx, priority);
2247    }
2248 
2249    return new_ctx;
2250 }
2251 
2252 void
iris_destroy_kernel_context(struct iris_bufmgr * bufmgr,uint32_t ctx_id)2253 iris_destroy_kernel_context(struct iris_bufmgr *bufmgr, uint32_t ctx_id)
2254 {
2255    struct drm_i915_gem_context_destroy d = { .ctx_id = ctx_id };
2256 
2257    if (ctx_id != 0 &&
2258        intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_DESTROY, &d) != 0) {
2259       fprintf(stderr, "DRM_IOCTL_I915_GEM_CONTEXT_DESTROY failed: %s\n",
2260               strerror(errno));
2261    }
2262 }
2263 
2264 int
iris_reg_read(struct iris_bufmgr * bufmgr,uint32_t offset,uint64_t * result)2265 iris_reg_read(struct iris_bufmgr *bufmgr, uint32_t offset, uint64_t *result)
2266 {
2267    struct drm_i915_reg_read reg_read = { .offset = offset };
2268    int ret = intel_ioctl(bufmgr->fd, DRM_IOCTL_I915_REG_READ, &reg_read);
2269 
2270    *result = reg_read.val;
2271    return ret;
2272 }
2273 
2274 static struct intel_buffer *
intel_aux_map_buffer_alloc(void * driver_ctx,uint32_t size)2275 intel_aux_map_buffer_alloc(void *driver_ctx, uint32_t size)
2276 {
2277    struct intel_buffer *buf = malloc(sizeof(struct intel_buffer));
2278    if (!buf)
2279       return NULL;
2280 
2281    struct iris_bufmgr *bufmgr = (struct iris_bufmgr *)driver_ctx;
2282 
2283    unsigned int page_size = getpagesize();
2284    size = MAX2(ALIGN(size, page_size), page_size);
2285 
2286    struct iris_bo *bo = alloc_fresh_bo(bufmgr, size, 0);
2287    if (!bo) {
2288       free(buf);
2289       return NULL;
2290    }
2291 
2292    simple_mtx_lock(&bufmgr->lock);
2293 
2294    bo->address = vma_alloc(bufmgr, IRIS_MEMZONE_OTHER, bo->size, 64 * 1024);
2295    if (bo->address == 0ull) {
2296       free(buf);
2297       bo_free(bo);
2298       simple_mtx_unlock(&bufmgr->lock);
2299       return NULL;
2300    }
2301 
2302    simple_mtx_unlock(&bufmgr->lock);
2303 
2304    bo->name = "aux-map";
2305    p_atomic_set(&bo->refcount, 1);
2306    bo->index = -1;
2307    bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED |
2308                      EXEC_OBJECT_CAPTURE;
2309    bo->real.mmap_mode =
2310       bo->real.heap != IRIS_HEAP_SYSTEM_MEMORY ? IRIS_MMAP_WC : IRIS_MMAP_WB;
2311 
2312    buf->driver_bo = bo;
2313    buf->gpu = bo->address;
2314    buf->gpu_end = buf->gpu + bo->size;
2315    buf->map = iris_bo_map(NULL, bo, MAP_WRITE | MAP_RAW);
2316    return buf;
2317 }
2318 
2319 static void
intel_aux_map_buffer_free(void * driver_ctx,struct intel_buffer * buffer)2320 intel_aux_map_buffer_free(void *driver_ctx, struct intel_buffer *buffer)
2321 {
2322    iris_bo_unreference((struct iris_bo*)buffer->driver_bo);
2323    free(buffer);
2324 }
2325 
2326 static struct intel_mapped_pinned_buffer_alloc aux_map_allocator = {
2327    .alloc = intel_aux_map_buffer_alloc,
2328    .free = intel_aux_map_buffer_free,
2329 };
2330 
2331 static int
gem_param(int fd,int name)2332 gem_param(int fd, int name)
2333 {
2334    int v = -1; /* No param uses (yet) the sign bit, reserve it for errors */
2335 
2336    struct drm_i915_getparam gp = { .param = name, .value = &v };
2337    if (intel_ioctl(fd, DRM_IOCTL_I915_GETPARAM, &gp))
2338       return -1;
2339 
2340    return v;
2341 }
2342 
2343 static bool
iris_bufmgr_get_meminfo(struct iris_bufmgr * bufmgr,struct intel_device_info * devinfo)2344 iris_bufmgr_get_meminfo(struct iris_bufmgr *bufmgr,
2345                         struct intel_device_info *devinfo)
2346 {
2347    bufmgr->sys.region.memory_class = devinfo->mem.sram.mem_class;
2348    bufmgr->sys.region.memory_instance = devinfo->mem.sram.mem_instance;
2349    bufmgr->sys.size = devinfo->mem.sram.mappable.size;
2350 
2351    bufmgr->vram.region.memory_class = devinfo->mem.vram.mem_class;
2352    bufmgr->vram.region.memory_instance = devinfo->mem.vram.mem_instance;
2353    bufmgr->vram.size = devinfo->mem.vram.mappable.size;
2354 
2355    return true;
2356 }
2357 
2358 static void
iris_bufmgr_init_global_vm(int fd,struct iris_bufmgr * bufmgr)2359 iris_bufmgr_init_global_vm(int fd, struct iris_bufmgr *bufmgr)
2360 {
2361    struct drm_i915_gem_context_param gcp = {
2362       .ctx_id = 0,
2363       .param = I915_CONTEXT_PARAM_VM,
2364    };
2365 
2366    if (intel_ioctl(fd, DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM, &gcp)) {
2367       bufmgr->use_global_vm = false;
2368       bufmgr->global_vm_id = 0;
2369    } else {
2370       bufmgr->use_global_vm = true;
2371       bufmgr->global_vm_id = gcp.value;
2372    }
2373 }
2374 
2375 /**
2376  * Initializes the GEM buffer manager, which uses the kernel to allocate, map,
2377  * and manage map buffer objections.
2378  *
2379  * \param fd File descriptor of the opened DRM device.
2380  */
2381 static struct iris_bufmgr *
iris_bufmgr_create(struct intel_device_info * devinfo,int fd,bool bo_reuse)2382 iris_bufmgr_create(struct intel_device_info *devinfo, int fd, bool bo_reuse)
2383 {
2384    if (devinfo->gtt_size <= IRIS_MEMZONE_OTHER_START)
2385       return NULL;
2386 
2387    struct iris_bufmgr *bufmgr = calloc(1, sizeof(*bufmgr));
2388    if (bufmgr == NULL)
2389       return NULL;
2390 
2391    /* Handles to buffer objects belong to the device fd and are not
2392     * reference counted by the kernel.  If the same fd is used by
2393     * multiple parties (threads sharing the same screen bufmgr, or
2394     * even worse the same device fd passed to multiple libraries)
2395     * ownership of those handles is shared by those independent parties.
2396     *
2397     * Don't do this! Ensure that each library/bufmgr has its own device
2398     * fd so that its namespace does not clash with another.
2399     */
2400    bufmgr->fd = os_dupfd_cloexec(fd);
2401 
2402    p_atomic_set(&bufmgr->refcount, 1);
2403 
2404    simple_mtx_init(&bufmgr->lock, mtx_plain);
2405    simple_mtx_init(&bufmgr->bo_deps_lock, mtx_plain);
2406 
2407    iris_bufmgr_init_global_vm(fd, bufmgr);
2408 
2409    list_inithead(&bufmgr->zombie_list);
2410 
2411    bufmgr->has_llc = devinfo->has_llc;
2412    bufmgr->has_local_mem = devinfo->has_local_mem;
2413    bufmgr->has_tiling_uapi = devinfo->has_tiling_uapi;
2414    bufmgr->bo_reuse = bo_reuse;
2415    bufmgr->has_mmap_offset = gem_param(fd, I915_PARAM_MMAP_GTT_VERSION) >= 4;
2416    bufmgr->has_userptr_probe =
2417       gem_param(fd, I915_PARAM_HAS_USERPTR_PROBE) >= 1;
2418    iris_bufmgr_get_meminfo(bufmgr, devinfo);
2419    bufmgr->all_vram_mappable = intel_vram_all_mappable(devinfo);
2420 
2421    STATIC_ASSERT(IRIS_MEMZONE_SHADER_START == 0ull);
2422    const uint64_t _4GB = 1ull << 32;
2423    const uint64_t _2GB = 1ul << 31;
2424 
2425    /* The STATE_BASE_ADDRESS size field can only hold 1 page shy of 4GB */
2426    const uint64_t _4GB_minus_1 = _4GB - PAGE_SIZE;
2427 
2428    util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SHADER],
2429                       PAGE_SIZE, _4GB_minus_1 - PAGE_SIZE);
2430    util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_BINDER],
2431                       IRIS_MEMZONE_BINDER_START, IRIS_BINDER_ZONE_SIZE);
2432    util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_BINDLESS],
2433                       IRIS_MEMZONE_BINDLESS_START, IRIS_BINDLESS_SIZE);
2434    util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SURFACE],
2435                       IRIS_MEMZONE_SURFACE_START, _4GB_minus_1 -
2436                       IRIS_BINDER_ZONE_SIZE - IRIS_BINDLESS_SIZE);
2437 
2438    /* Wa_2209859288: the Tigerlake PRM's workarounds volume says:
2439     *
2440     *    "PSDunit is dropping MSB of the blend state pointer from SD FIFO"
2441     *    "Limit the Blend State Pointer to < 2G"
2442     *
2443     * We restrict the dynamic state pool to 2GB so that we don't ever get a
2444     * BLEND_STATE pointer with the MSB set.  We aren't likely to need the
2445     * full 4GB for dynamic state anyway.
2446     */
2447    const uint64_t dynamic_pool_size =
2448       (devinfo->ver >= 12 ? _2GB : _4GB_minus_1) - IRIS_BORDER_COLOR_POOL_SIZE;
2449    util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_DYNAMIC],
2450                       IRIS_MEMZONE_DYNAMIC_START + IRIS_BORDER_COLOR_POOL_SIZE,
2451                       dynamic_pool_size);
2452 
2453    /* Leave the last 4GB out of the high vma range, so that no state
2454     * base address + size can overflow 48 bits.
2455     */
2456    util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_OTHER],
2457                       IRIS_MEMZONE_OTHER_START,
2458                       (devinfo->gtt_size - _4GB) - IRIS_MEMZONE_OTHER_START);
2459 
2460    init_cache_buckets(bufmgr, IRIS_HEAP_SYSTEM_MEMORY);
2461    init_cache_buckets(bufmgr, IRIS_HEAP_DEVICE_LOCAL);
2462    init_cache_buckets(bufmgr, IRIS_HEAP_DEVICE_LOCAL_PREFERRED);
2463 
2464    unsigned min_slab_order = 8;  /* 256 bytes */
2465    unsigned max_slab_order = 20; /* 1 MB (slab size = 2 MB) */
2466    unsigned num_slab_orders_per_allocator =
2467       (max_slab_order - min_slab_order) / NUM_SLAB_ALLOCATORS;
2468 
2469    /* Divide the size order range among slab managers. */
2470    for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
2471       unsigned min_order = min_slab_order;
2472       unsigned max_order =
2473          MIN2(min_order + num_slab_orders_per_allocator, max_slab_order);
2474 
2475       if (!pb_slabs_init(&bufmgr->bo_slabs[i], min_order, max_order,
2476                          IRIS_HEAP_MAX, true, bufmgr,
2477                          iris_can_reclaim_slab,
2478                          iris_slab_alloc,
2479                          (void *) iris_slab_free)) {
2480          free(bufmgr);
2481          return NULL;
2482       }
2483       min_slab_order = max_order + 1;
2484    }
2485 
2486    bufmgr->name_table =
2487       _mesa_hash_table_create(NULL, _mesa_hash_uint, _mesa_key_uint_equal);
2488    bufmgr->handle_table =
2489       _mesa_hash_table_create(NULL, _mesa_hash_uint, _mesa_key_uint_equal);
2490 
2491    bufmgr->vma_min_align =
2492       devinfo->verx10 >= 125 ? 2 * 1024 * 1024 :
2493       (devinfo->has_local_mem ? 64 * 1024 : PAGE_SIZE);
2494 
2495    if (devinfo->has_aux_map) {
2496       bufmgr->aux_map_ctx = intel_aux_map_init(bufmgr, &aux_map_allocator,
2497                                                devinfo);
2498       assert(bufmgr->aux_map_ctx);
2499    }
2500 
2501    iris_init_border_color_pool(bufmgr, &bufmgr->border_color_pool);
2502 
2503    return bufmgr;
2504 }
2505 
2506 static struct iris_bufmgr *
iris_bufmgr_ref(struct iris_bufmgr * bufmgr)2507 iris_bufmgr_ref(struct iris_bufmgr *bufmgr)
2508 {
2509    p_atomic_inc(&bufmgr->refcount);
2510    return bufmgr;
2511 }
2512 
2513 void
iris_bufmgr_unref(struct iris_bufmgr * bufmgr)2514 iris_bufmgr_unref(struct iris_bufmgr *bufmgr)
2515 {
2516    simple_mtx_lock(&global_bufmgr_list_mutex);
2517    if (p_atomic_dec_zero(&bufmgr->refcount)) {
2518       list_del(&bufmgr->link);
2519       iris_bufmgr_destroy(bufmgr);
2520    }
2521    simple_mtx_unlock(&global_bufmgr_list_mutex);
2522 }
2523 
2524 /** Returns a new unique id, to be used by screens. */
2525 int
iris_bufmgr_create_screen_id(struct iris_bufmgr * bufmgr)2526 iris_bufmgr_create_screen_id(struct iris_bufmgr *bufmgr)
2527 {
2528    return p_atomic_inc_return(&bufmgr->next_screen_id) - 1;
2529 }
2530 
2531 /**
2532  * Gets an already existing GEM buffer manager or create a new one.
2533  *
2534  * \param fd File descriptor of the opened DRM device.
2535  */
2536 struct iris_bufmgr *
iris_bufmgr_get_for_fd(struct intel_device_info * devinfo,int fd,bool bo_reuse)2537 iris_bufmgr_get_for_fd(struct intel_device_info *devinfo, int fd, bool bo_reuse)
2538 {
2539    struct stat st;
2540 
2541    if (fstat(fd, &st))
2542       return NULL;
2543 
2544    struct iris_bufmgr *bufmgr = NULL;
2545 
2546    simple_mtx_lock(&global_bufmgr_list_mutex);
2547    list_for_each_entry(struct iris_bufmgr, iter_bufmgr, &global_bufmgr_list, link) {
2548       struct stat iter_st;
2549       if (fstat(iter_bufmgr->fd, &iter_st))
2550          continue;
2551 
2552       if (st.st_rdev == iter_st.st_rdev) {
2553          assert(iter_bufmgr->bo_reuse == bo_reuse);
2554          bufmgr = iris_bufmgr_ref(iter_bufmgr);
2555          goto unlock;
2556       }
2557    }
2558 
2559    bufmgr = iris_bufmgr_create(devinfo, fd, bo_reuse);
2560    if (bufmgr)
2561       list_addtail(&bufmgr->link, &global_bufmgr_list);
2562 
2563  unlock:
2564    simple_mtx_unlock(&global_bufmgr_list_mutex);
2565 
2566    return bufmgr;
2567 }
2568 
2569 int
iris_bufmgr_get_fd(struct iris_bufmgr * bufmgr)2570 iris_bufmgr_get_fd(struct iris_bufmgr *bufmgr)
2571 {
2572    return bufmgr->fd;
2573 }
2574 
2575 void*
iris_bufmgr_get_aux_map_context(struct iris_bufmgr * bufmgr)2576 iris_bufmgr_get_aux_map_context(struct iris_bufmgr *bufmgr)
2577 {
2578    return bufmgr->aux_map_ctx;
2579 }
2580 
2581 simple_mtx_t *
iris_bufmgr_get_bo_deps_lock(struct iris_bufmgr * bufmgr)2582 iris_bufmgr_get_bo_deps_lock(struct iris_bufmgr *bufmgr)
2583 {
2584    return &bufmgr->bo_deps_lock;
2585 }
2586 
2587 struct iris_border_color_pool *
iris_bufmgr_get_border_color_pool(struct iris_bufmgr * bufmgr)2588 iris_bufmgr_get_border_color_pool(struct iris_bufmgr *bufmgr)
2589 {
2590    return &bufmgr->border_color_pool;
2591 }
2592 
2593 uint64_t
iris_bufmgr_vram_size(struct iris_bufmgr * bufmgr)2594 iris_bufmgr_vram_size(struct iris_bufmgr *bufmgr)
2595 {
2596    return bufmgr->vram.size;
2597 }
2598 
2599 uint64_t
iris_bufmgr_sram_size(struct iris_bufmgr * bufmgr)2600 iris_bufmgr_sram_size(struct iris_bufmgr *bufmgr)
2601 {
2602    return bufmgr->sys.size;
2603 }
2604