1 /*
2 * Copyright 2024 Advanced Micro Devices, Inc.
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "drm-uapi/amdgpu_drm.h"
8
9 #include "amdgpu_virtio_private.h"
10 #include "ac_linux_drm.h"
11 #include "util/list.h"
12 #include "util/log.h"
13 #include "util/os_mman.h"
14 #include "util/os_time.h"
15 #include "util/u_math.h"
16 #include "sid.h"
17
18 #include <xf86drm.h>
19 #include <string.h>
20 #include <fcntl.h>
21
22 struct amdvgpu_host_blob {
23 /* virtgpu properties */
24 uint32_t handle;
25 uint32_t res_id;
26 uint64_t alloc_size;
27
28 /* CPU mapping handling. */
29 uint64_t offset;
30 int map_count;
31 void *cpu_addr;
32 simple_mtx_t cpu_access_mutex;
33
34 /* Allocation parameters. */
35 uint32_t vm_flags;
36 uint32_t preferred_heap;
37 uint64_t phys_alignment;
38 uint64_t flags;
39 };
40
41 static
42 void destroy_host_blob(amdvgpu_device_handle dev, struct amdvgpu_host_blob *hb);
43
44 static
create_host_blob(uint32_t kms_handle,uint32_t res_id,uint64_t size,struct amdgpu_ccmd_gem_new_req * req)45 struct amdvgpu_host_blob *create_host_blob(uint32_t kms_handle,
46 uint32_t res_id,
47 uint64_t size,
48 struct amdgpu_ccmd_gem_new_req *req)
49 {
50 struct amdvgpu_host_blob *hb = calloc(1, sizeof(*hb));
51 hb->handle = kms_handle;
52 hb->res_id = res_id;
53 hb->alloc_size = size;
54
55 if (req) {
56 hb->phys_alignment = req->r.phys_alignment;
57 hb->preferred_heap = req->r.preferred_heap;
58 hb->flags = req->r.flags;
59 }
60
61 simple_mtx_init(&hb->cpu_access_mutex, mtx_plain);
62 return hb;
63 }
64
65 static
destroy_host_blob(amdvgpu_device_handle dev,struct amdvgpu_host_blob * hb)66 void destroy_host_blob(amdvgpu_device_handle dev, struct amdvgpu_host_blob *hb) {
67 simple_mtx_destroy(&hb->cpu_access_mutex);
68
69 struct drm_gem_close req = {
70 .handle = hb->handle,
71 };
72 int r = drmIoctl(dev->fd, DRM_IOCTL_GEM_CLOSE, &req);
73 if (r != 0) {
74 mesa_loge("DRM_IOCTL_GEM_CLOSE failed for res_id: %d\n", hb->res_id);
75 }
76 free(hb);
77 }
78
79 static int
alloc_host_blob(amdvgpu_bo_handle bo,struct amdgpu_ccmd_gem_new_req * req,uint32_t blob_flags)80 alloc_host_blob(amdvgpu_bo_handle bo,
81 struct amdgpu_ccmd_gem_new_req *req,
82 uint32_t blob_flags)
83 {
84 uint32_t kms_handle, res_id;
85
86 /* Create the host blob requires 2 steps. First create the host blob... */
87 kms_handle = vdrm_bo_create(bo->dev->vdev, req->r.alloc_size, blob_flags,
88 req->blob_id, &req->hdr);
89
90 /* 0 is an invalid handle and is used by vdrm_bo_create to signal an error. */
91 if (kms_handle == 0)
92 return -1;
93
94 /* ... and then retrieve its resource id (global id). */
95 res_id = vdrm_handle_to_res_id(bo->dev->vdev, kms_handle);
96
97 bo->host_blob = create_host_blob(kms_handle, res_id, req->r.alloc_size, req);
98
99 simple_mtx_lock(&bo->dev->handle_to_vbo_mutex);
100 _mesa_hash_table_insert(bo->dev->handle_to_vbo, (void*)(intptr_t)bo->host_blob->handle, bo);
101 simple_mtx_unlock(&bo->dev->handle_to_vbo_mutex);
102
103 return 0;
104 }
105
amdvgpu_bo_export(amdvgpu_device_handle dev,amdvgpu_bo_handle bo,enum amdgpu_bo_handle_type type,uint32_t * shared_handle)106 int amdvgpu_bo_export(amdvgpu_device_handle dev, amdvgpu_bo_handle bo,
107 enum amdgpu_bo_handle_type type,
108 uint32_t *shared_handle)
109 {
110 switch (type) {
111 case amdgpu_bo_handle_type_kms:
112 /* Return the resource id as this handle is only going to be used
113 * internally (AMDGPU_CHUNK_ID_BO_HANDLES mostly).
114 */
115 *shared_handle = amdvgpu_get_resource_id(bo);
116 return 0;
117
118 case amdgpu_bo_handle_type_dma_buf_fd:
119 return drmPrimeHandleToFD(dev->fd, bo->host_blob->handle, DRM_CLOEXEC | DRM_RDWR,
120 (int*)shared_handle);
121
122 case amdgpu_bo_handle_type_kms_noimport:
123 /* Treat this deprecated type as _type_kms and return the GEM handle. */
124 *shared_handle = bo->host_blob->handle;
125 return 0;
126
127 case amdgpu_bo_handle_type_gem_flink_name:
128 break;
129 }
130 return -EINVAL;
131 }
132
amdvgpu_bo_free(amdvgpu_device_handle dev,struct amdvgpu_bo * bo)133 int amdvgpu_bo_free(amdvgpu_device_handle dev, struct amdvgpu_bo *bo) {
134 int refcnt = p_atomic_dec_return(&bo->refcount);
135
136 if (refcnt == 0) {
137 /* Flush pending ops. */
138 vdrm_flush(dev->vdev);
139
140 /* Remove it from the bo table. */
141 if (bo->host_blob->handle > 0) {
142 simple_mtx_lock(&dev->handle_to_vbo_mutex);
143 struct hash_entry *entry = _mesa_hash_table_search(dev->handle_to_vbo,
144 (void*)(intptr_t)bo->host_blob->handle);
145 if (entry) {
146 /* entry can be NULL for the shmem buffer. */
147 _mesa_hash_table_remove(dev->handle_to_vbo, entry);
148 }
149 simple_mtx_unlock(&dev->handle_to_vbo_mutex);
150 }
151
152 if (bo->host_blob)
153 destroy_host_blob(dev, bo->host_blob);
154
155 free(bo);
156 }
157
158 return 0;
159 }
160
amdvgpu_bo_alloc(amdvgpu_device_handle dev,struct amdgpu_bo_alloc_request * request,amdvgpu_bo_handle * bo)161 int amdvgpu_bo_alloc(amdvgpu_device_handle dev,
162 struct amdgpu_bo_alloc_request *request,
163 amdvgpu_bo_handle *bo)
164 {
165 int r;
166 uint32_t blob_flags = 0;
167
168 struct amdgpu_ccmd_gem_new_req req = {
169 .hdr = AMDGPU_CCMD(GEM_NEW, sizeof(req)),
170 .blob_id = p_atomic_inc_return(&dev->next_blob_id),
171 };
172 req.r.alloc_size = request->alloc_size;
173 req.r.phys_alignment = request->phys_alignment;
174 req.r.preferred_heap = request->preferred_heap;
175 req.r.__pad = 0;
176 req.r.flags = request->flags;
177
178 if (!(request->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
179 blob_flags |= VIRTGPU_BLOB_FLAG_USE_MAPPABLE;
180
181 if (request->flags & AMDGPU_GEM_CREATE_VIRTIO_SHARED) {
182 blob_flags |= VIRTGPU_BLOB_FLAG_USE_SHAREABLE;
183 req.r.flags &= ~AMDGPU_GEM_CREATE_VIRTIO_SHARED;
184 }
185
186 /* blob_id 0 is reserved for the shared memory buffer. */
187 assert(req.blob_id > 0);
188
189 amdvgpu_bo_handle out = calloc(1, sizeof(struct amdvgpu_bo));
190 out->dev = dev;
191 out->size = request->alloc_size;
192
193 r = alloc_host_blob(out, &req, blob_flags);
194
195 if (r < 0) {
196 free(out);
197 return r;
198 }
199
200 p_atomic_set(&out->refcount, 1);
201 *bo = out;
202
203 return 0;
204 }
205
amdvgpu_bo_va_op_raw(amdvgpu_device_handle dev,uint32_t res_id,uint64_t offset,uint64_t size,uint64_t addr,uint64_t flags,uint32_t ops)206 int amdvgpu_bo_va_op_raw(amdvgpu_device_handle dev,
207 uint32_t res_id,
208 uint64_t offset,
209 uint64_t size,
210 uint64_t addr,
211 uint64_t flags,
212 uint32_t ops)
213 {
214 int r;
215
216 /* Fill base structure fields. */
217 struct amdgpu_ccmd_bo_va_op_req req = {
218 .hdr = AMDGPU_CCMD(BO_VA_OP, sizeof(req)),
219 .va = addr,
220 .res_id = res_id,
221 .offset = offset,
222 .vm_map_size = size,
223 .flags = flags,
224 .op = ops,
225 .flags2 = res_id == 0 ? AMDGPU_CCMD_BO_VA_OP_SPARSE_BO : 0,
226 };
227 struct amdgpu_ccmd_rsp *rsp =
228 vdrm_alloc_rsp(dev->vdev, &req.hdr, sizeof(*rsp));
229
230 r = vdrm_send_req_wrapper(dev, &req.hdr, rsp, false);
231
232 return r;
233 }
234
amdvgpu_bo_import(amdvgpu_device_handle dev,enum amdgpu_bo_handle_type type,uint32_t handle,struct amdvgpu_bo_import_result * result)235 int amdvgpu_bo_import(amdvgpu_device_handle dev, enum amdgpu_bo_handle_type type,
236 uint32_t handle, struct amdvgpu_bo_import_result *result)
237 {
238 if (type != amdgpu_bo_handle_type_dma_buf_fd)
239 return -1;
240
241 uint32_t kms_handle;
242 int r = drmPrimeFDToHandle(dev->fd, handle, &kms_handle);
243 if (r) {
244 mesa_loge("drmPrimeFDToHandle failed for dmabuf fd: %u\n", handle);
245 return r;
246 }
247
248 /* Look up existing bo. */
249 simple_mtx_lock(&dev->handle_to_vbo_mutex);
250 struct hash_entry *entry = _mesa_hash_table_search(dev->handle_to_vbo, (void*)(intptr_t)kms_handle);
251
252 if (entry) {
253 struct amdvgpu_bo *bo = entry->data;
254 p_atomic_inc(&bo->refcount);
255 simple_mtx_unlock(&dev->handle_to_vbo_mutex);
256 result->buf_handle = (void*)bo;
257 result->alloc_size = bo->size;
258 assert(bo->host_blob);
259 return 0;
260 }
261 simple_mtx_unlock(&dev->handle_to_vbo_mutex);
262
263 struct drm_virtgpu_resource_info args = {
264 .bo_handle = kms_handle,
265 };
266 r = virtio_ioctl(dev->fd, VIRTGPU_RESOURCE_INFO, &args);
267
268 if (r) {
269 mesa_loge("VIRTGPU_RESOURCE_INFO failed (%s)\n", strerror(errno));
270 return r;
271 }
272
273 off_t size = lseek(handle, 0, SEEK_END);
274 if (size == (off_t) -1) {
275 mesa_loge("lseek failed (%s)\n", strerror(errno));
276 return -1;
277 }
278 lseek(handle, 0, SEEK_CUR);
279
280 struct amdvgpu_bo *bo = calloc(1, sizeof(struct amdvgpu_bo));
281 bo->dev = dev;
282 bo->size = size;
283 bo->host_blob = create_host_blob(kms_handle, args.res_handle, size, NULL);
284 p_atomic_set(&bo->refcount, 1);
285
286 result->buf_handle = bo;
287 result->alloc_size = bo->size;
288
289 simple_mtx_lock(&dev->handle_to_vbo_mutex);
290 _mesa_hash_table_insert(dev->handle_to_vbo, (void*)(intptr_t)bo->host_blob->handle, bo);
291 simple_mtx_unlock(&dev->handle_to_vbo_mutex);
292
293 return 0;
294 }
295
amdvgpu_get_offset(amdvgpu_bo_handle bo_handle)296 static int amdvgpu_get_offset(amdvgpu_bo_handle bo_handle)
297 {
298 if (bo_handle->host_blob->offset)
299 return 0;
300
301 struct drm_virtgpu_map req = {
302 .handle = bo_handle->host_blob->handle,
303 };
304 int ret = virtio_ioctl(bo_handle->dev->fd, VIRTGPU_MAP, &req);
305 if (ret) {
306 mesa_loge("amdvgpu_bo_map failed (%s) handle: %d\n",
307 strerror(errno), bo_handle->host_blob->handle);
308 return ret;
309 }
310 bo_handle->host_blob->offset = req.offset;
311
312 return 0;
313 }
314
amdvgpu_bo_cpu_map(amdvgpu_device_handle dev,amdvgpu_bo_handle bo_handle,void ** cpu)315 int amdvgpu_bo_cpu_map(amdvgpu_device_handle dev, amdvgpu_bo_handle bo_handle,
316 void **cpu) {
317 int r;
318
319 simple_mtx_lock(&bo_handle->host_blob->cpu_access_mutex);
320
321 if (bo_handle->host_blob->cpu_addr == NULL) {
322 assert(bo_handle->host_blob->cpu_addr == NULL);
323 r = amdvgpu_get_offset(bo_handle);
324 if (r) {
325 mesa_loge("get_offset failed\n");
326 simple_mtx_unlock(&bo_handle->host_blob->cpu_access_mutex);
327 return r;
328 }
329
330 /* Use *cpu as a fixed address hint from the caller. */
331 bo_handle->host_blob->cpu_addr = os_mmap(*cpu, bo_handle->host_blob->alloc_size,
332 PROT_READ | PROT_WRITE, MAP_SHARED,
333 dev->fd,
334 bo_handle->host_blob->offset);
335 }
336
337 assert(bo_handle->host_blob->cpu_addr != MAP_FAILED);
338 *cpu = bo_handle->host_blob->cpu_addr;
339 p_atomic_inc(&bo_handle->host_blob->map_count);
340
341 simple_mtx_unlock(&bo_handle->host_blob->cpu_access_mutex);
342
343 return *cpu == MAP_FAILED;
344 }
345
amdvgpu_bo_cpu_unmap(amdvgpu_device_handle dev,amdvgpu_bo_handle bo)346 int amdvgpu_bo_cpu_unmap(amdvgpu_device_handle dev, amdvgpu_bo_handle bo) {
347 int r = 0;
348
349 simple_mtx_lock(&bo->host_blob->cpu_access_mutex);
350 if (bo->host_blob->map_count == 0) {
351 simple_mtx_unlock(&bo->host_blob->cpu_access_mutex);
352 return 0;
353 }
354 assert(bo->host_blob->cpu_addr);
355 if (p_atomic_dec_zero(&bo->host_blob->map_count)) {
356 r = os_munmap(bo->host_blob->cpu_addr, bo->host_blob->alloc_size);
357 bo->host_blob->cpu_addr = NULL;
358 }
359 simple_mtx_unlock(&bo->host_blob->cpu_access_mutex);
360
361 return r;
362 }
363
amdvgpu_get_resource_id(amdvgpu_bo_handle bo)364 uint32_t amdvgpu_get_resource_id(amdvgpu_bo_handle bo) {
365 return bo->host_blob->res_id;
366 }
367
amdvgpu_bo_wait_for_idle(amdvgpu_device_handle dev,amdvgpu_bo_handle bo,uint64_t abs_timeout_ns)368 int amdvgpu_bo_wait_for_idle(amdvgpu_device_handle dev,
369 amdvgpu_bo_handle bo,
370 uint64_t abs_timeout_ns) {
371 /* TODO: add a wait for idle command? */
372 return vdrm_bo_wait(dev->vdev, bo->host_blob->handle);
373 }
374