1 /*
2 * Copyright © 2024 Collabora Ltd. and Red Hat Inc.
3 * SPDX-License-Identifier: MIT
4 */
5 #ifndef NVKMD_H
6 #define NVKMD_H 1
7
8 #include "nv_device_info.h"
9 #include "util/list.h"
10 #include "util/simple_mtx.h"
11 #include "util/u_atomic.h"
12
13 #include "../nvk_debug.h"
14 #include "vulkan/vulkan_core.h"
15
16 #include <assert.h>
17 #include <stdbool.h>
18 #include <stdbool.h>
19 #include <sys/types.h>
20
21 struct nvkmd_ctx;
22 struct nvkmd_dev;
23 struct nvkmd_mem;
24 struct nvkmd_pdev;
25 struct nvkmd_va;
26
27 struct _drmDevice;
28 struct vk_object_base;
29 struct vk_sync_wait;
30 struct vk_sync_signal;
31
32 /*
33 * Enums
34 */
35
36 enum nvkmd_mem_flags {
37 /** Place memory as local as possible.
38 *
39 * This should be the default for most memory allocations. On discrete
40 * GPUs, it will default to be placed in VRAM but may be paged out to GART,
41 * depending on system memory pressure.
42 */
43 NVKMD_MEM_LOCAL = 1 << 0,
44
45 /** Place the memory in GART */
46 NVKMD_MEM_GART = 1 << 1,
47
48 /** Place the memory in VRAM */
49 NVKMD_MEM_VRAM = 1 << 2,
50
51 /** This memory object may be mapped */
52 NVKMD_MEM_CAN_MAP = 1 << 3,
53
54 /** This memory object may be shared with other processes */
55 NVKMD_MEM_SHARED = 1 << 4,
56 };
57
58 enum nvkmd_mem_map_flags {
59 NVKMD_MEM_MAP_RD = 1 << 0,
60 NVKMD_MEM_MAP_WR = 1 << 1,
61 NVKMD_MEM_MAP_RDWR = NVKMD_MEM_MAP_RD | NVKMD_MEM_MAP_WR,
62
63 /** Create a client mapping
64 *
65 * This sets nvkmd_mem::client_map instead of nvkmd_mem::map. These
66 * mappings may be different from internal mappings and have different
67 * rules. Only one client mapping may exist at a time but internal
68 * mappings are reference counted. Only client mappings can be used with
69 * MAP_FIXED or unmapped with nvkmd_mem_overmap().
70 */
71 NVKMD_MEM_MAP_CLIENT = 1 << 2,
72
73 NVKMD_MEM_MAP_FIXED = 1 << 3,
74 };
75
76 enum nvkmd_va_flags {
77 /** This VA should be configured for sparse (soft faults) */
78 NVKMD_VA_SPARSE = 1 << 0,
79
80 /** This VA should come from the capture/replay pool */
81 NVKMD_VA_REPLAY = 1 << 1,
82
83 /** Attempt to place this VA at the requested address and fail otherwise */
84 NVKMD_VA_ALLOC_FIXED = 1 << 2,
85
86 /** This VA will only be used with GART.
87 *
88 * Alignment requirements for GART-only are lower.
89 */
90 NVKMD_VA_GART = 1 << 3,
91 };
92
93 enum nvkmd_engines {
94 NVKMD_ENGINE_COPY = 1 << 0,
95 NVKMD_ENGINE_2D = 1 << 1,
96 NVKMD_ENGINE_3D = 1 << 2,
97 NVKMD_ENGINE_M2MF = 1 << 3,
98 NVKMD_ENGINE_COMPUTE = 1 << 4,
99 NVKMD_ENGINE_BIND = 1 << 5,
100 };
101
102 enum nvkmd_bind_op {
103 NVKMD_BIND_OP_BIND,
104 NVKMD_BIND_OP_UNBIND,
105 };
106
107 /*
108 * Structs
109 */
110
111 struct nvkmd_info {
112 bool has_dma_buf;
113 bool has_get_vram_used;
114 bool has_alloc_tiled;
115 bool has_map_fixed;
116 bool has_overmap;
117 };
118
119 struct nvkmd_pdev_ops {
120 void (*destroy)(struct nvkmd_pdev *pdev);
121
122 uint64_t (*get_vram_used)(struct nvkmd_pdev *pdev);
123
124 int (*get_drm_primary_fd)(struct nvkmd_pdev *pdev);
125
126 VkResult (*create_dev)(struct nvkmd_pdev *pdev,
127 struct vk_object_base *log_obj,
128 struct nvkmd_dev **dev_out);
129 };
130
131 struct nvkmd_pdev {
132 const struct nvkmd_pdev_ops *ops;
133
134 enum nvk_debug debug_flags;
135
136 struct nv_device_info dev_info;
137 struct nvkmd_info kmd_info;
138
139 /** Device-wide GPU memory bind alignment
140 *
141 * Individual BOs may have a smaller alignment if, for instance, we know
142 * they are pinned to system ram.
143 */
144 uint32_t bind_align_B;
145
146 struct {
147 dev_t render_dev;
148 dev_t primary_dev;
149 } drm;
150
151 const struct vk_sync_type *const *sync_types;
152 };
153
154 struct nvkmd_dev_ops {
155 void (*destroy)(struct nvkmd_dev *dev);
156
157 uint64_t (*get_gpu_timestamp)(struct nvkmd_dev *dev);
158
159 int (*get_drm_fd)(struct nvkmd_dev *dev);
160
161 VkResult (*alloc_mem)(struct nvkmd_dev *dev,
162 struct vk_object_base *log_obj,
163 uint64_t size_B, uint64_t align_B,
164 enum nvkmd_mem_flags flags,
165 struct nvkmd_mem **mem_out);
166
167 VkResult (*alloc_tiled_mem)(struct nvkmd_dev *dev,
168 struct vk_object_base *log_obj,
169 uint64_t size_B, uint64_t align_B,
170 uint8_t pte_kind, uint16_t tile_mode,
171 enum nvkmd_mem_flags flags,
172 struct nvkmd_mem **mem_out);
173
174 VkResult (*import_dma_buf)(struct nvkmd_dev *dev,
175 struct vk_object_base *log_obj,
176 int fd, struct nvkmd_mem **mem_out);
177
178 VkResult (*alloc_va)(struct nvkmd_dev *dev,
179 struct vk_object_base *log_obj,
180 enum nvkmd_va_flags flags, uint8_t pte_kind,
181 uint64_t size_B, uint64_t align_B,
182 uint64_t fixed_addr, struct nvkmd_va **va_out);
183
184 VkResult (*create_ctx)(struct nvkmd_dev *dev,
185 struct vk_object_base *log_obj,
186 enum nvkmd_engines engines,
187 struct nvkmd_ctx **ctx_out);
188 };
189
190 struct nvkmd_dev {
191 const struct nvkmd_dev_ops *ops;
192 struct nvkmd_pdev *pdev;
193
194 /* Start and end of the usable VA space. All nvkmd_va objects will be
195 * allocated within this range.
196 */
197 uint64_t va_start, va_end;
198
199 struct list_head mems;
200 simple_mtx_t mems_mutex;
201 };
202
203 struct nvkmd_mem_ops {
204 void (*free)(struct nvkmd_mem *mem);
205
206 VkResult (*map)(struct nvkmd_mem *mem,
207 struct vk_object_base *log_obj,
208 enum nvkmd_mem_map_flags flags,
209 void *fixed_addr,
210 void **map_out);
211
212 void (*unmap)(struct nvkmd_mem *mem,
213 enum nvkmd_mem_map_flags flags,
214 void *map);
215
216 VkResult (*overmap)(struct nvkmd_mem *mem,
217 struct vk_object_base *log_obj,
218 enum nvkmd_mem_map_flags flags,
219 void *map);
220
221 VkResult (*export_dma_buf)(struct nvkmd_mem *mem,
222 struct vk_object_base *log_obj,
223 int *fd_out);
224
225 /** Handle to use for NVK_DEBUG_VM logging */
226 uint32_t (*log_handle)(struct nvkmd_mem *mem);
227 };
228
229 struct nvkmd_mem {
230 const struct nvkmd_mem_ops *ops;
231 struct nvkmd_dev *dev;
232
233 /* Optional link in nvkmd_dev::mems */
234 struct list_head link;
235
236 uint32_t refcnt;
237
238 enum nvkmd_mem_flags flags;
239 uint32_t bind_align_B;
240
241 uint64_t size_B;
242 struct nvkmd_va *va;
243
244 simple_mtx_t map_mutex;
245 uint32_t map_cnt;
246 void *map;
247
248 void *client_map;
249 };
250
251 void nvkmd_mem_init(struct nvkmd_dev *dev,
252 struct nvkmd_mem *mem,
253 const struct nvkmd_mem_ops *ops,
254 enum nvkmd_mem_flags flags,
255 uint64_t size_B,
256 uint32_t bind_align_B);
257
258 struct nvkmd_va_ops {
259 void (*free)(struct nvkmd_va *va);
260
261 VkResult (*bind_mem)(struct nvkmd_va *va,
262 struct vk_object_base *log_obj,
263 uint64_t va_offset_B,
264 struct nvkmd_mem *mem,
265 uint64_t mem_offset_B,
266 uint64_t range_B);
267
268 VkResult (*unbind)(struct nvkmd_va *va,
269 struct vk_object_base *log_obj,
270 uint64_t va_offset_B,
271 uint64_t range_B);
272 };
273
274 struct nvkmd_va {
275 const struct nvkmd_va_ops *ops;
276 struct nvkmd_dev *dev;
277
278 enum nvkmd_va_flags flags;
279 uint8_t pte_kind;
280 uint64_t addr;
281 uint64_t size_B;
282 };
283
284 struct nvkmd_ctx_exec {
285 uint64_t addr;
286 uint32_t size_B;
287 bool no_prefetch;
288 };
289
290 struct nvkmd_ctx_bind {
291 enum nvkmd_bind_op op;
292
293 struct nvkmd_va *va;
294 uint64_t va_offset_B;
295
296 /* Ignored if op != NVK_BIND_OP_UNBIND */
297 struct nvkmd_mem *mem;
298 uint64_t mem_offset_B;
299
300 uint64_t range_B;
301 };
302
303 struct nvkmd_ctx_ops {
304 void (*destroy)(struct nvkmd_ctx *ctx);
305
306 VkResult (*wait)(struct nvkmd_ctx *ctx,
307 struct vk_object_base *log_obj,
308 uint32_t wait_count,
309 const struct vk_sync_wait *waits);
310
311 VkResult (*exec)(struct nvkmd_ctx *ctx,
312 struct vk_object_base *log_obj,
313 uint32_t exec_count,
314 const struct nvkmd_ctx_exec *execs);
315
316 VkResult (*bind)(struct nvkmd_ctx *ctx,
317 struct vk_object_base *log_obj,
318 uint32_t bind_count,
319 const struct nvkmd_ctx_bind *binds);
320
321 /* Implies flush() */
322 VkResult (*signal)(struct nvkmd_ctx *ctx,
323 struct vk_object_base *log_obj,
324 uint32_t signal_count,
325 const struct vk_sync_signal *signals);
326
327 VkResult (*flush)(struct nvkmd_ctx *ctx,
328 struct vk_object_base *log_obj);
329
330 /* Implies flush() */
331 VkResult (*sync)(struct nvkmd_ctx *ctx,
332 struct vk_object_base *log_obj);
333 };
334
335 struct nvkmd_ctx {
336 const struct nvkmd_ctx_ops *ops;
337 struct nvkmd_dev *dev;
338 };
339
340 /*
341 * Macros
342 *
343 * All subclassed structs must be named nvkmd_<subcls>_<strct> where the
344 * original struct is named nvkmd_<strct>
345 */
346
347 #define NVKMD_DECL_SUBCLASS(strct, subcls) \
348 extern const struct nvkmd_##strct##_ops nvkmd_##subcls##_##strct##_ops; \
349 static inline struct nvkmd_##subcls##_##strct * \
350 nvkmd_##subcls##_##strct(struct nvkmd_##strct *nvkmd) \
351 { \
352 assert(nvkmd->ops == &nvkmd_##subcls##_##strct##_ops); \
353 return container_of(nvkmd, struct nvkmd_##subcls##_##strct, base); \
354 }
355
356 /*
357 * Methods
358 *
359 * Even though everything goes through a function pointer table, we always add
360 * an inline wrapper in case we want to move something into "core" NVKMD.
361 */
362
363 VkResult MUST_CHECK
364 nvkmd_try_create_pdev_for_drm(struct _drmDevice *drm_device,
365 struct vk_object_base *log_obj,
366 enum nvk_debug debug_flags,
367 struct nvkmd_pdev **pdev_out);
368
369 static inline void
nvkmd_pdev_destroy(struct nvkmd_pdev * pdev)370 nvkmd_pdev_destroy(struct nvkmd_pdev *pdev)
371 {
372 pdev->ops->destroy(pdev);
373 }
374
375 static inline uint64_t
nvkmd_pdev_get_vram_used(struct nvkmd_pdev * pdev)376 nvkmd_pdev_get_vram_used(struct nvkmd_pdev *pdev)
377 {
378 return pdev->ops->get_vram_used(pdev);
379 }
380
381 static inline int
nvkmd_pdev_get_drm_primary_fd(struct nvkmd_pdev * pdev)382 nvkmd_pdev_get_drm_primary_fd(struct nvkmd_pdev *pdev)
383 {
384 if (pdev->ops->get_drm_primary_fd == NULL)
385 return -1;
386
387 return pdev->ops->get_drm_primary_fd(pdev);
388 }
389
390 static inline VkResult MUST_CHECK
nvkmd_pdev_create_dev(struct nvkmd_pdev * pdev,struct vk_object_base * log_obj,struct nvkmd_dev ** dev_out)391 nvkmd_pdev_create_dev(struct nvkmd_pdev *pdev,
392 struct vk_object_base *log_obj,
393 struct nvkmd_dev **dev_out)
394 {
395 return pdev->ops->create_dev(pdev, log_obj, dev_out);
396 }
397
398 static inline void
nvkmd_dev_destroy(struct nvkmd_dev * dev)399 nvkmd_dev_destroy(struct nvkmd_dev *dev)
400 {
401 dev->ops->destroy(dev);
402 }
403
404 static inline uint64_t
nvkmd_dev_get_gpu_timestamp(struct nvkmd_dev * dev)405 nvkmd_dev_get_gpu_timestamp(struct nvkmd_dev *dev)
406 {
407 return dev->ops->get_gpu_timestamp(dev);
408 }
409
410 static inline int
nvkmd_dev_get_drm_fd(struct nvkmd_dev * dev)411 nvkmd_dev_get_drm_fd(struct nvkmd_dev *dev)
412 {
413 if (dev->ops->get_drm_fd == NULL)
414 return -1;
415
416 return dev->ops->get_drm_fd(dev);
417 }
418
419 static inline VkResult MUST_CHECK
nvkmd_dev_alloc_mem(struct nvkmd_dev * dev,struct vk_object_base * log_obj,uint64_t size_B,uint64_t align_B,enum nvkmd_mem_flags flags,struct nvkmd_mem ** mem_out)420 nvkmd_dev_alloc_mem(struct nvkmd_dev *dev,
421 struct vk_object_base *log_obj,
422 uint64_t size_B, uint64_t align_B,
423 enum nvkmd_mem_flags flags,
424 struct nvkmd_mem **mem_out)
425 {
426 return dev->ops->alloc_mem(dev, log_obj, size_B, align_B, flags, mem_out);
427 }
428
429 static inline VkResult MUST_CHECK
nvkmd_dev_alloc_tiled_mem(struct nvkmd_dev * dev,struct vk_object_base * log_obj,uint64_t size_B,uint64_t align_B,uint8_t pte_kind,uint16_t tile_mode,enum nvkmd_mem_flags flags,struct nvkmd_mem ** mem_out)430 nvkmd_dev_alloc_tiled_mem(struct nvkmd_dev *dev,
431 struct vk_object_base *log_obj,
432 uint64_t size_B, uint64_t align_B,
433 uint8_t pte_kind, uint16_t tile_mode,
434 enum nvkmd_mem_flags flags,
435 struct nvkmd_mem **mem_out)
436 {
437 return dev->ops->alloc_tiled_mem(dev, log_obj, size_B, align_B,
438 pte_kind, tile_mode, flags, mem_out);
439 }
440
441 /* Implies NVKMD_MEM_CAN_MAP */
442 VkResult MUST_CHECK
443 nvkmd_dev_alloc_mapped_mem(struct nvkmd_dev *dev,
444 struct vk_object_base *log_obj,
445 uint64_t size_B, uint64_t align_B,
446 enum nvkmd_mem_flags flags,
447 enum nvkmd_mem_map_flags map_flags,
448 struct nvkmd_mem **mem_out);
449
450 void
451 nvkmd_dev_track_mem(struct nvkmd_dev *dev,
452 struct nvkmd_mem *mem);
453
454 struct nvkmd_mem *
455 nvkmd_dev_lookup_mem_by_va(struct nvkmd_dev *dev,
456 uint64_t addr,
457 uint64_t *offset_out);
458
459 static inline VkResult MUST_CHECK
nvkmd_dev_import_dma_buf(struct nvkmd_dev * dev,struct vk_object_base * log_obj,int fd,struct nvkmd_mem ** mem_out)460 nvkmd_dev_import_dma_buf(struct nvkmd_dev *dev,
461 struct vk_object_base *log_obj,
462 int fd, struct nvkmd_mem **mem_out)
463 {
464 return dev->ops->import_dma_buf(dev, log_obj, fd, mem_out);
465 }
466
467 VkResult MUST_CHECK
468 nvkmd_dev_alloc_va(struct nvkmd_dev *dev,
469 struct vk_object_base *log_obj,
470 enum nvkmd_va_flags flags, uint8_t pte_kind,
471 uint64_t size_B, uint64_t align_B,
472 uint64_t fixed_addr, struct nvkmd_va **va_out);
473
474 static inline VkResult MUST_CHECK
nvkmd_dev_create_ctx(struct nvkmd_dev * dev,struct vk_object_base * log_obj,enum nvkmd_engines engines,struct nvkmd_ctx ** ctx_out)475 nvkmd_dev_create_ctx(struct nvkmd_dev *dev,
476 struct vk_object_base *log_obj,
477 enum nvkmd_engines engines,
478 struct nvkmd_ctx **ctx_out)
479 {
480 return dev->ops->create_ctx(dev, log_obj, engines, ctx_out);
481 }
482
483 static inline struct nvkmd_mem *
nvkmd_mem_ref(struct nvkmd_mem * mem)484 nvkmd_mem_ref(struct nvkmd_mem *mem)
485 {
486 p_atomic_inc(&mem->refcnt);
487 return mem;
488 }
489
490 void nvkmd_mem_unref(struct nvkmd_mem *mem);
491
492 VkResult MUST_CHECK
493 nvkmd_mem_map(struct nvkmd_mem *mem, struct vk_object_base *log_obj,
494 enum nvkmd_mem_map_flags flags, void *fixed_addr,
495 void **map_out);
496
497 void nvkmd_mem_unmap(struct nvkmd_mem *mem, enum nvkmd_mem_map_flags flags);
498
499 static inline VkResult MUST_CHECK
nvkmd_mem_overmap(struct nvkmd_mem * mem,struct vk_object_base * log_obj,enum nvkmd_mem_map_flags flags)500 nvkmd_mem_overmap(struct nvkmd_mem *mem, struct vk_object_base *log_obj,
501 enum nvkmd_mem_map_flags flags)
502 {
503 assert(flags & NVKMD_MEM_MAP_CLIENT);
504 assert(mem->client_map != NULL);
505
506 VkResult result = mem->ops->overmap(mem, log_obj, flags, mem->client_map);
507 if (result == VK_SUCCESS)
508 mem->client_map = NULL;
509
510 return result;
511 }
512
513 static inline VkResult MUST_CHECK
nvkmd_mem_export_dma_buf(struct nvkmd_mem * mem,struct vk_object_base * log_obj,int * fd_out)514 nvkmd_mem_export_dma_buf(struct nvkmd_mem *mem,
515 struct vk_object_base *log_obj,
516 int *fd_out)
517 {
518 assert(mem->flags & NVKMD_MEM_SHARED);
519
520 return mem->ops->export_dma_buf(mem, log_obj, fd_out);
521 }
522
523 void
524 nvkmd_va_free(struct nvkmd_va *va);
525
526 VkResult MUST_CHECK
527 nvkmd_va_bind_mem(struct nvkmd_va *va,
528 struct vk_object_base *log_obj,
529 uint64_t va_offset_B,
530 struct nvkmd_mem *mem,
531 uint64_t mem_offset_B,
532 uint64_t range_B);
533
534 VkResult MUST_CHECK
535 nvkmd_va_unbind(struct nvkmd_va *va,
536 struct vk_object_base *log_obj,
537 uint64_t va_offset_B,
538 uint64_t range_B);
539
540 static inline void
nvkmd_ctx_destroy(struct nvkmd_ctx * ctx)541 nvkmd_ctx_destroy(struct nvkmd_ctx *ctx)
542 {
543 ctx->ops->destroy(ctx);
544 }
545
546 static inline VkResult MUST_CHECK
nvkmd_ctx_wait(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj,uint32_t wait_count,const struct vk_sync_wait * waits)547 nvkmd_ctx_wait(struct nvkmd_ctx *ctx,
548 struct vk_object_base *log_obj,
549 uint32_t wait_count,
550 const struct vk_sync_wait *waits)
551 {
552 return ctx->ops->wait(ctx, log_obj, wait_count, waits);
553 }
554
555 static inline VkResult MUST_CHECK
nvkmd_ctx_exec(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj,uint32_t exec_count,const struct nvkmd_ctx_exec * execs)556 nvkmd_ctx_exec(struct nvkmd_ctx *ctx,
557 struct vk_object_base *log_obj,
558 uint32_t exec_count,
559 const struct nvkmd_ctx_exec *execs)
560 {
561 return ctx->ops->exec(ctx, log_obj, exec_count, execs);
562 }
563
564 VkResult MUST_CHECK
565 nvkmd_ctx_bind(struct nvkmd_ctx *ctx,
566 struct vk_object_base *log_obj,
567 uint32_t bind_count,
568 const struct nvkmd_ctx_bind *binds);
569
570 static inline VkResult MUST_CHECK
nvkmd_ctx_signal(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj,uint32_t signal_count,const struct vk_sync_signal * signals)571 nvkmd_ctx_signal(struct nvkmd_ctx *ctx,
572 struct vk_object_base *log_obj,
573 uint32_t signal_count,
574 const struct vk_sync_signal *signals)
575 {
576 return ctx->ops->signal(ctx, log_obj, signal_count, signals);
577 }
578
579 static inline VkResult MUST_CHECK
nvkmd_ctx_flush(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj)580 nvkmd_ctx_flush(struct nvkmd_ctx *ctx,
581 struct vk_object_base *log_obj)
582 {
583 return ctx->ops->flush(ctx, log_obj);
584 }
585
586 static inline VkResult MUST_CHECK
nvkmd_ctx_sync(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj)587 nvkmd_ctx_sync(struct nvkmd_ctx *ctx,
588 struct vk_object_base *log_obj)
589 {
590 return ctx->ops->sync(ctx, log_obj);
591 }
592
593 #endif /* NVKMD_H */
594