1 #include "nouveau_bo.h"
2
3 #include "drm-uapi/nouveau_drm.h"
4 #include "util/hash_table.h"
5 #include "util/u_math.h"
6
7 #include <errno.h>
8 #include <fcntl.h>
9 #include <stddef.h>
10 #include <sys/mman.h>
11 #include <xf86drm.h>
12
13 static void
bo_bind(struct nouveau_ws_device * dev,uint32_t handle,uint64_t addr,uint64_t range,uint64_t bo_offset,uint32_t flags)14 bo_bind(struct nouveau_ws_device *dev,
15 uint32_t handle, uint64_t addr,
16 uint64_t range, uint64_t bo_offset,
17 uint32_t flags)
18 {
19 int ret;
20
21 struct drm_nouveau_vm_bind_op newbindop = {
22 .op = DRM_NOUVEAU_VM_BIND_OP_MAP,
23 .handle = handle,
24 .addr = addr,
25 .range = range,
26 .bo_offset = bo_offset,
27 .flags = flags,
28 };
29 struct drm_nouveau_vm_bind vmbind = {
30 .op_count = 1,
31 .op_ptr = (uint64_t)(uintptr_t)(void *)&newbindop,
32 };
33 ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_VM_BIND, &vmbind, sizeof(vmbind));
34 if (ret)
35 fprintf(stderr, "vm bind failed %d\n", errno);
36 assert(ret == 0);
37 }
38
39 static void
bo_unbind(struct nouveau_ws_device * dev,uint64_t offset,uint64_t range,uint32_t flags)40 bo_unbind(struct nouveau_ws_device *dev,
41 uint64_t offset, uint64_t range,
42 uint32_t flags)
43 {
44 struct drm_nouveau_vm_bind_op newbindop = {
45 .op = DRM_NOUVEAU_VM_BIND_OP_UNMAP,
46 .addr = offset,
47 .range = range,
48 .flags = flags,
49 };
50 struct drm_nouveau_vm_bind vmbind = {
51 .op_count = 1,
52 .op_ptr = (uint64_t)(uintptr_t)(void *)&newbindop,
53 };
54 ASSERTED int ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_VM_BIND, &vmbind, sizeof(vmbind));
55 assert(ret == 0);
56 }
57
58 uint64_t
nouveau_ws_alloc_vma(struct nouveau_ws_device * dev,uint64_t req_addr,uint64_t size,uint64_t align,bool bda_capture_replay,bool sparse_resident)59 nouveau_ws_alloc_vma(struct nouveau_ws_device *dev,
60 uint64_t req_addr, uint64_t size, uint64_t align,
61 bool bda_capture_replay,
62 bool sparse_resident)
63 {
64 assert(dev->has_vm_bind);
65
66 /* if the caller doesn't care, use the GPU page size */
67 if (align == 0)
68 align = 0x1000;
69
70 uint64_t offset;
71 simple_mtx_lock(&dev->vma_mutex);
72 if (bda_capture_replay) {
73 if (req_addr != 0) {
74 bool found = util_vma_heap_alloc_addr(&dev->bda_heap, req_addr, size);
75 offset = found ? req_addr : 0;
76 } else {
77 offset = util_vma_heap_alloc(&dev->bda_heap, size, align);
78 }
79 } else {
80 offset = util_vma_heap_alloc(&dev->vma_heap, size, align);
81 }
82 simple_mtx_unlock(&dev->vma_mutex);
83
84 if (offset == 0) {
85 if (dev->debug_flags & NVK_DEBUG_VM) {
86 fprintf(stderr, "alloc vma FAILED: %" PRIx64 " sparse: %d\n",
87 size, sparse_resident);
88 }
89 return 0;
90 }
91
92 if (dev->debug_flags & NVK_DEBUG_VM)
93 fprintf(stderr, "alloc vma %" PRIx64 " %" PRIx64 " sparse: %d\n",
94 offset, size, sparse_resident);
95
96 if (sparse_resident)
97 bo_bind(dev, 0, offset, size, 0, DRM_NOUVEAU_VM_BIND_SPARSE);
98
99 return offset;
100 }
101
102 void
nouveau_ws_free_vma(struct nouveau_ws_device * dev,uint64_t offset,uint64_t size,bool bda_capture_replay,bool sparse_resident)103 nouveau_ws_free_vma(struct nouveau_ws_device *dev,
104 uint64_t offset, uint64_t size,
105 bool bda_capture_replay,
106 bool sparse_resident)
107 {
108 assert(dev->has_vm_bind);
109
110 if (dev->debug_flags & NVK_DEBUG_VM)
111 fprintf(stderr, "free vma %" PRIx64 " %" PRIx64 "\n",
112 offset, size);
113
114 if (sparse_resident)
115 bo_unbind(dev, offset, size, DRM_NOUVEAU_VM_BIND_SPARSE);
116
117 simple_mtx_lock(&dev->vma_mutex);
118 if (bda_capture_replay) {
119 util_vma_heap_free(&dev->bda_heap, offset, size);
120 } else {
121 util_vma_heap_free(&dev->vma_heap, offset, size);
122 }
123 simple_mtx_unlock(&dev->vma_mutex);
124 }
125
126 void
nouveau_ws_bo_unbind_vma(struct nouveau_ws_device * dev,uint64_t offset,uint64_t range)127 nouveau_ws_bo_unbind_vma(struct nouveau_ws_device *dev,
128 uint64_t offset, uint64_t range)
129 {
130 assert(dev->has_vm_bind);
131
132 if (dev->debug_flags & NVK_DEBUG_VM)
133 fprintf(stderr, "unbind vma %" PRIx64 " %" PRIx64 "\n",
134 offset, range);
135 bo_unbind(dev, offset, range, 0);
136 }
137
138 void
nouveau_ws_bo_bind_vma(struct nouveau_ws_device * dev,struct nouveau_ws_bo * bo,uint64_t addr,uint64_t range,uint64_t bo_offset,uint32_t pte_kind)139 nouveau_ws_bo_bind_vma(struct nouveau_ws_device *dev,
140 struct nouveau_ws_bo *bo,
141 uint64_t addr,
142 uint64_t range,
143 uint64_t bo_offset,
144 uint32_t pte_kind)
145 {
146 assert(dev->has_vm_bind);
147
148 if (dev->debug_flags & NVK_DEBUG_VM)
149 fprintf(stderr, "bind vma %x %" PRIx64 " %" PRIx64 " %" PRIx64 " %d\n",
150 bo->handle, addr, range, bo_offset, pte_kind);
151 bo_bind(dev, bo->handle, addr, range, bo_offset, pte_kind);
152 }
153
154 struct nouveau_ws_bo *
nouveau_ws_bo_new_mapped(struct nouveau_ws_device * dev,uint64_t size,uint64_t align,enum nouveau_ws_bo_flags flags,enum nouveau_ws_bo_map_flags map_flags,void ** map_out)155 nouveau_ws_bo_new_mapped(struct nouveau_ws_device *dev,
156 uint64_t size, uint64_t align,
157 enum nouveau_ws_bo_flags flags,
158 enum nouveau_ws_bo_map_flags map_flags,
159 void **map_out)
160 {
161 struct nouveau_ws_bo *bo = nouveau_ws_bo_new(dev, size, align,
162 flags | NOUVEAU_WS_BO_MAP);
163 if (!bo)
164 return NULL;
165
166 void *map = nouveau_ws_bo_map(bo, map_flags, NULL);
167 if (map == NULL) {
168 nouveau_ws_bo_destroy(bo);
169 return NULL;
170 }
171
172 *map_out = map;
173 return bo;
174 }
175
176 static struct nouveau_ws_bo *
nouveau_ws_bo_new_locked(struct nouveau_ws_device * dev,uint64_t size,uint64_t align,enum nouveau_ws_bo_flags flags)177 nouveau_ws_bo_new_locked(struct nouveau_ws_device *dev,
178 uint64_t size, uint64_t align,
179 enum nouveau_ws_bo_flags flags)
180 {
181 struct drm_nouveau_gem_new req = {};
182
183 /* if the caller doesn't care, use the GPU page size */
184 if (align == 0)
185 align = 0x1000;
186
187 /* Align the size */
188 size = align64(size, align);
189
190 req.info.domain = 0;
191
192 /* TODO:
193 *
194 * VRAM maps on Kepler appear to be broken and we don't really know why.
195 * My NVIDIA contact doesn't remember them not working so they probably
196 * should but they don't today. Force everything that may be mapped to
197 * use GART for now.
198 */
199 if (flags & NOUVEAU_WS_BO_GART)
200 req.info.domain |= NOUVEAU_GEM_DOMAIN_GART;
201 else if (dev->info.chipset < 0x110 && (flags & NOUVEAU_WS_BO_MAP))
202 req.info.domain |= NOUVEAU_GEM_DOMAIN_GART;
203 else
204 req.info.domain |= dev->local_mem_domain;
205
206 if (flags & NOUVEAU_WS_BO_MAP)
207 req.info.domain |= NOUVEAU_GEM_DOMAIN_MAPPABLE;
208
209 if (flags & NOUVEAU_WS_BO_NO_SHARE)
210 req.info.domain |= NOUVEAU_GEM_DOMAIN_NO_SHARE;
211
212 req.info.size = size;
213 req.align = align;
214
215 int ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_GEM_NEW, &req, sizeof(req));
216 if (ret != 0)
217 return NULL;
218
219 struct nouveau_ws_bo *bo = CALLOC_STRUCT(nouveau_ws_bo);
220 bo->size = size;
221 bo->align = align;
222 bo->offset = -1ULL;
223 bo->handle = req.info.handle;
224 bo->map_handle = req.info.map_handle;
225 bo->dev = dev;
226 bo->flags = flags;
227 bo->refcnt = 1;
228
229 if (dev->has_vm_bind) {
230 bo->offset = nouveau_ws_alloc_vma(dev, 0, bo->size, align, false, false);
231 if (bo->offset == 0)
232 goto fail_gem_new;
233
234 nouveau_ws_bo_bind_vma(dev, bo, bo->offset, bo->size, 0, 0);
235 }
236
237 _mesa_hash_table_insert(dev->bos, (void *)(uintptr_t)bo->handle, bo);
238
239 return bo;
240
241 fail_gem_new:
242 drmCloseBufferHandle(dev->fd, req.info.handle);
243 FREE(bo);
244
245 return NULL;
246 }
247
248 struct nouveau_ws_bo *
nouveau_ws_bo_new(struct nouveau_ws_device * dev,uint64_t size,uint64_t align,enum nouveau_ws_bo_flags flags)249 nouveau_ws_bo_new(struct nouveau_ws_device *dev,
250 uint64_t size, uint64_t align,
251 enum nouveau_ws_bo_flags flags)
252 {
253 struct nouveau_ws_bo *bo;
254
255 simple_mtx_lock(&dev->bos_lock);
256 bo = nouveau_ws_bo_new_locked(dev, size, align, flags);
257 simple_mtx_unlock(&dev->bos_lock);
258
259 return bo;
260 }
261
262 static struct nouveau_ws_bo *
nouveau_ws_bo_from_dma_buf_locked(struct nouveau_ws_device * dev,int fd)263 nouveau_ws_bo_from_dma_buf_locked(struct nouveau_ws_device *dev, int fd)
264 {
265 uint32_t handle;
266 int ret = drmPrimeFDToHandle(dev->fd, fd, &handle);
267 if (ret != 0)
268 return NULL;
269
270 struct hash_entry *entry =
271 _mesa_hash_table_search(dev->bos, (void *)(uintptr_t)handle);
272 if (entry != NULL)
273 return entry->data;
274
275 /*
276 * If we got here, no BO exists for the retrieved handle. If we error
277 * after this point, we need to close the handle.
278 */
279
280 struct drm_nouveau_gem_info info = {
281 .handle = handle
282 };
283 ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_GEM_INFO,
284 &info, sizeof(info));
285 if (ret != 0)
286 goto fail_fd_to_handle;
287
288 enum nouveau_ws_bo_flags flags = 0;
289 if (info.domain & NOUVEAU_GEM_DOMAIN_GART)
290 flags |= NOUVEAU_WS_BO_GART;
291 if (info.map_handle)
292 flags |= NOUVEAU_WS_BO_MAP;
293
294 struct nouveau_ws_bo *bo = CALLOC_STRUCT(nouveau_ws_bo);
295 bo->size = info.size;
296 bo->offset = info.offset;
297 bo->handle = info.handle;
298 bo->map_handle = info.map_handle;
299 bo->dev = dev;
300 bo->flags = flags;
301 bo->refcnt = 1;
302
303 uint64_t align = (1ULL << 12);
304 if (info.domain & NOUVEAU_GEM_DOMAIN_VRAM)
305 align = (1ULL << 16);
306
307 assert(bo->size == align64(bo->size, align));
308
309 bo->offset = nouveau_ws_alloc_vma(dev, 0, bo->size, align, false, false);
310 if (bo->offset == 0)
311 goto fail_calloc;
312
313 nouveau_ws_bo_bind_vma(dev, bo, bo->offset, bo->size, 0, 0);
314 _mesa_hash_table_insert(dev->bos, (void *)(uintptr_t)handle, bo);
315
316 return bo;
317
318 fail_calloc:
319 FREE(bo);
320 fail_fd_to_handle:
321 drmCloseBufferHandle(dev->fd, handle);
322
323 return NULL;
324 }
325
326 struct nouveau_ws_bo *
nouveau_ws_bo_from_dma_buf(struct nouveau_ws_device * dev,int fd)327 nouveau_ws_bo_from_dma_buf(struct nouveau_ws_device *dev, int fd)
328 {
329 struct nouveau_ws_bo *bo;
330
331 simple_mtx_lock(&dev->bos_lock);
332 bo = nouveau_ws_bo_from_dma_buf_locked(dev, fd);
333 simple_mtx_unlock(&dev->bos_lock);
334
335 return bo;
336 }
337
338 void
nouveau_ws_bo_destroy(struct nouveau_ws_bo * bo)339 nouveau_ws_bo_destroy(struct nouveau_ws_bo *bo)
340 {
341 if (--bo->refcnt)
342 return;
343
344 struct nouveau_ws_device *dev = bo->dev;
345
346 simple_mtx_lock(&dev->bos_lock);
347
348 _mesa_hash_table_remove_key(dev->bos, (void *)(uintptr_t)bo->handle);
349
350 if (dev->has_vm_bind) {
351 nouveau_ws_bo_unbind_vma(bo->dev, bo->offset, bo->size);
352 nouveau_ws_free_vma(bo->dev, bo->offset, bo->size, false, false);
353 }
354
355 drmCloseBufferHandle(bo->dev->fd, bo->handle);
356 FREE(bo);
357
358 simple_mtx_unlock(&dev->bos_lock);
359 }
360
361 void *
nouveau_ws_bo_map(struct nouveau_ws_bo * bo,enum nouveau_ws_bo_map_flags flags,void * fixed_addr)362 nouveau_ws_bo_map(struct nouveau_ws_bo *bo,
363 enum nouveau_ws_bo_map_flags flags,
364 void *fixed_addr)
365 {
366 int prot = 0, map_flags = 0;
367
368 if (flags & NOUVEAU_WS_BO_RD)
369 prot |= PROT_READ;
370 if (flags & NOUVEAU_WS_BO_WR)
371 prot |= PROT_WRITE;
372
373 map_flags = MAP_SHARED;
374 if (fixed_addr != NULL)
375 map_flags |= MAP_FIXED;
376
377 void *res = mmap(fixed_addr, bo->size, prot, map_flags,
378 bo->dev->fd, bo->map_handle);
379 if (res == MAP_FAILED)
380 return NULL;
381
382 return res;
383 }
384
385 bool
nouveau_ws_bo_wait(struct nouveau_ws_bo * bo,enum nouveau_ws_bo_map_flags flags)386 nouveau_ws_bo_wait(struct nouveau_ws_bo *bo, enum nouveau_ws_bo_map_flags flags)
387 {
388 struct drm_nouveau_gem_cpu_prep req = {};
389
390 req.handle = bo->handle;
391 if (flags & NOUVEAU_WS_BO_WR)
392 req.flags |= NOUVEAU_GEM_CPU_PREP_WRITE;
393
394 return !drmCommandWrite(bo->dev->fd, DRM_NOUVEAU_GEM_CPU_PREP, &req, sizeof(req));
395 }
396
397 int
nouveau_ws_bo_dma_buf(struct nouveau_ws_bo * bo,int * fd)398 nouveau_ws_bo_dma_buf(struct nouveau_ws_bo *bo, int *fd)
399 {
400 return drmPrimeHandleToFD(bo->dev->fd, bo->handle, DRM_CLOEXEC, fd);
401 }
402