1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * SPDX-License-Identifier: MIT
9 */
10
11 #include "radv_device_memory.h"
12 #include "radv_android.h"
13 #include "radv_buffer.h"
14 #include "radv_debug.h"
15 #include "radv_entrypoints.h"
16 #include "radv_image.h"
17 #include "radv_rmv.h"
18
19 #include "vk_log.h"
20
21 void
radv_device_memory_init(struct radv_device_memory * mem,struct radv_device * device,struct radeon_winsys_bo * bo)22 radv_device_memory_init(struct radv_device_memory *mem, struct radv_device *device, struct radeon_winsys_bo *bo)
23 {
24 memset(mem, 0, sizeof(*mem));
25 vk_object_base_init(&device->vk, &mem->base, VK_OBJECT_TYPE_DEVICE_MEMORY);
26
27 mem->bo = bo;
28 }
29
30 void
radv_device_memory_finish(struct radv_device_memory * mem)31 radv_device_memory_finish(struct radv_device_memory *mem)
32 {
33 vk_object_base_finish(&mem->base);
34 }
35
36 void
radv_free_memory(struct radv_device * device,const VkAllocationCallbacks * pAllocator,struct radv_device_memory * mem)37 radv_free_memory(struct radv_device *device, const VkAllocationCallbacks *pAllocator, struct radv_device_memory *mem)
38 {
39 if (mem == NULL)
40 return;
41
42 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
43 if (mem->android_hardware_buffer)
44 AHardwareBuffer_release(mem->android_hardware_buffer);
45 #endif
46
47 if (mem->bo) {
48 if (device->overallocation_disallowed) {
49 mtx_lock(&device->overallocation_mutex);
50 device->allocated_memory_size[mem->heap_index] -= mem->alloc_size;
51 mtx_unlock(&device->overallocation_mutex);
52 }
53
54 if (device->use_global_bo_list)
55 device->ws->buffer_make_resident(device->ws, mem->bo, false);
56 radv_bo_destroy(device, &mem->base, mem->bo);
57 mem->bo = NULL;
58 }
59
60 radv_rmv_log_resource_destroy(device, (uint64_t)radv_device_memory_to_handle(mem));
61 radv_device_memory_finish(mem);
62 vk_free2(&device->vk.alloc, pAllocator, mem);
63 }
64
65 VkResult
radv_alloc_memory(struct radv_device * device,const VkMemoryAllocateInfo * pAllocateInfo,const VkAllocationCallbacks * pAllocator,VkDeviceMemory * pMem,bool is_internal)66 radv_alloc_memory(struct radv_device *device, const VkMemoryAllocateInfo *pAllocateInfo,
67 const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMem, bool is_internal)
68 {
69 struct radv_device_memory *mem;
70 VkResult result;
71 enum radeon_bo_domain domain;
72 uint32_t flags = 0;
73
74 assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
75
76 const VkImportMemoryFdInfoKHR *import_info = vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_FD_INFO_KHR);
77 const VkMemoryDedicatedAllocateInfo *dedicate_info =
78 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_DEDICATED_ALLOCATE_INFO);
79 const VkExportMemoryAllocateInfo *export_info =
80 vk_find_struct_const(pAllocateInfo->pNext, EXPORT_MEMORY_ALLOCATE_INFO);
81 const struct VkImportAndroidHardwareBufferInfoANDROID *ahb_import_info =
82 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID);
83 const VkImportMemoryHostPointerInfoEXT *host_ptr_info =
84 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT);
85 const struct VkMemoryAllocateFlagsInfo *flags_info =
86 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_ALLOCATE_FLAGS_INFO);
87
88 const struct wsi_memory_allocate_info *wsi_info =
89 vk_find_struct_const(pAllocateInfo->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA);
90
91 if (pAllocateInfo->allocationSize == 0 && !ahb_import_info &&
92 !(export_info &&
93 (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID))) {
94 /* Apparently, this is allowed */
95 *pMem = VK_NULL_HANDLE;
96 return VK_SUCCESS;
97 }
98
99 mem = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*mem), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
100 if (mem == NULL)
101 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
102
103 radv_device_memory_init(mem, device, NULL);
104
105 if (dedicate_info) {
106 mem->image = radv_image_from_handle(dedicate_info->image);
107 mem->buffer = radv_buffer_from_handle(dedicate_info->buffer);
108 } else {
109 mem->image = NULL;
110 mem->buffer = NULL;
111 }
112
113 if (wsi_info && wsi_info->implicit_sync) {
114 flags |= RADEON_FLAG_IMPLICIT_SYNC;
115
116 /* Mark the linear prime buffer (aka the destination of the prime blit
117 * as uncached.
118 */
119 if (mem->buffer)
120 flags |= RADEON_FLAG_VA_UNCACHED;
121 }
122
123 float priority_float = 0.5;
124 const struct VkMemoryPriorityAllocateInfoEXT *priority_ext =
125 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_PRIORITY_ALLOCATE_INFO_EXT);
126 if (priority_ext)
127 priority_float = priority_ext->priority;
128
129 uint64_t replay_address = 0;
130 const VkMemoryOpaqueCaptureAddressAllocateInfo *replay_info =
131 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_OPAQUE_CAPTURE_ADDRESS_ALLOCATE_INFO);
132 if (replay_info && replay_info->opaqueCaptureAddress)
133 replay_address = replay_info->opaqueCaptureAddress;
134
135 unsigned priority =
136 MIN2(RADV_BO_PRIORITY_APPLICATION_MAX - 1, (int)(priority_float * RADV_BO_PRIORITY_APPLICATION_MAX));
137
138 mem->user_ptr = NULL;
139
140 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
141 mem->android_hardware_buffer = NULL;
142 #endif
143
144 if (ahb_import_info) {
145 result = radv_import_ahb_memory(device, mem, priority, ahb_import_info);
146 if (result != VK_SUCCESS)
147 goto fail;
148 } else if (export_info &&
149 (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID)) {
150 result = radv_create_ahb_memory(device, mem, priority, pAllocateInfo);
151 if (result != VK_SUCCESS)
152 goto fail;
153 } else if (import_info) {
154 assert(import_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
155 import_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
156 result = radv_bo_from_fd(device, import_info->fd, priority, mem, NULL);
157 if (result != VK_SUCCESS) {
158 goto fail;
159 } else {
160 close(import_info->fd);
161 }
162
163 if (mem->image && mem->image->plane_count == 1 && !vk_format_is_depth_or_stencil(mem->image->vk.format) &&
164 mem->image->vk.samples == 1 && mem->image->vk.tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
165 struct radeon_bo_metadata metadata;
166 device->ws->buffer_get_metadata(device->ws, mem->bo, &metadata);
167
168 struct radv_image_create_info create_info = {.no_metadata_planes = true, .bo_metadata = &metadata};
169
170 /* This gives a basic ability to import radeonsi images
171 * that don't have DCC. This is not guaranteed by any
172 * spec and can be removed after we support modifiers. */
173 result = radv_image_create_layout(device, create_info, NULL, NULL, mem->image);
174 if (result != VK_SUCCESS) {
175 radv_bo_destroy(device, &mem->base, mem->bo);
176 goto fail;
177 }
178 }
179 } else if (host_ptr_info) {
180 assert(host_ptr_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
181 result = radv_bo_from_ptr(device, host_ptr_info->pHostPointer, pAllocateInfo->allocationSize, priority, mem);
182 if (result != VK_SUCCESS) {
183 goto fail;
184 } else {
185 mem->user_ptr = host_ptr_info->pHostPointer;
186 }
187 } else {
188 const struct radv_physical_device *pdev = radv_device_physical(device);
189 const struct radv_instance *instance = radv_physical_device_instance(pdev);
190 uint64_t alloc_size = align64(pAllocateInfo->allocationSize, 4096);
191 uint32_t heap_index;
192
193 heap_index = pdev->memory_properties.memoryTypes[pAllocateInfo->memoryTypeIndex].heapIndex;
194 domain = pdev->memory_domains[pAllocateInfo->memoryTypeIndex];
195 flags |= pdev->memory_flags[pAllocateInfo->memoryTypeIndex];
196
197 if (export_info && export_info->handleTypes) {
198 /* Setting RADEON_FLAG_GTT_WC in case the bo is spilled to GTT. This is important when the
199 * foreign queue is the display engine of iGPU. The carveout of iGPU can be tiny and the
200 * kernel driver refuses to spill without the flag.
201 *
202 * This covers any external memory user, including WSI.
203 */
204 if (domain == RADEON_DOMAIN_VRAM)
205 flags |= RADEON_FLAG_GTT_WC;
206 } else if (!import_info) {
207 /* neither export nor import */
208 flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
209 if (device->use_global_bo_list) {
210 flags |= RADEON_FLAG_PREFER_LOCAL_BO;
211 }
212 }
213
214 if (flags_info && flags_info->flags & VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT)
215 flags |= RADEON_FLAG_REPLAYABLE;
216
217 if (instance->drirc.zero_vram)
218 flags |= RADEON_FLAG_ZERO_VRAM;
219
220 /* On GFX12, DCC is transparent to the userspace driver and PTE.DCC is
221 * set per buffer allocation. Only VRAM can have DCC. When the kernel
222 * moves a buffer from VRAM->GTT it decompresses. When the kernel moves
223 * it from GTT->VRAM it recompresses but only if WRITE_COMPRESS_DISABLE=0
224 * (see DCC tiling flags).
225 */
226 if (pdev->info.gfx_level >= GFX12 && pdev->info.gfx12_supports_dcc_write_compress_disable &&
227 domain == RADEON_DOMAIN_VRAM && !(instance->debug_flags & RADV_DEBUG_NO_DCC)) {
228 flags |= RADEON_FLAG_GFX12_ALLOW_DCC;
229 }
230
231 if (device->overallocation_disallowed) {
232 uint64_t total_size = pdev->memory_properties.memoryHeaps[heap_index].size;
233
234 mtx_lock(&device->overallocation_mutex);
235 if (device->allocated_memory_size[heap_index] + alloc_size > total_size) {
236 mtx_unlock(&device->overallocation_mutex);
237 result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
238 goto fail;
239 }
240 device->allocated_memory_size[heap_index] += alloc_size;
241 mtx_unlock(&device->overallocation_mutex);
242 }
243
244 result = radv_bo_create(device, &mem->base, alloc_size, pdev->info.max_alignment, domain, flags, priority,
245 replay_address, is_internal, &mem->bo);
246
247 if (result != VK_SUCCESS) {
248 if (device->overallocation_disallowed) {
249 mtx_lock(&device->overallocation_mutex);
250 device->allocated_memory_size[heap_index] -= alloc_size;
251 mtx_unlock(&device->overallocation_mutex);
252 }
253 goto fail;
254 }
255
256 if (flags & RADEON_FLAG_GFX12_ALLOW_DCC) {
257 if (mem->image) {
258 /* Set BO metadata (including DCC tiling flags) for dedicated
259 * allocations because compressed writes are enabled and the kernel
260 * requires a DCC view for recompression.
261 */
262 radv_image_bo_set_metadata(device, mem->image, mem->bo);
263 } else {
264 /* Otherwise, disable compressed writes to prevent recompression
265 * when the BO is moved back to VRAM because it's not yet possible
266 * to set DCC tiling flags per range for suballocations. The only
267 * problem is that we will loose DCC after migration but that
268 * should happen rarely.
269 */
270 struct radeon_bo_metadata md = {0};
271
272 md.u.gfx12.dcc_write_compress_disable = true;
273
274 device->ws->buffer_set_metadata(device->ws, mem->bo, &md);
275 }
276 }
277
278 mem->heap_index = heap_index;
279 mem->alloc_size = alloc_size;
280 }
281
282 if (!wsi_info) {
283 if (device->use_global_bo_list) {
284 result = device->ws->buffer_make_resident(device->ws, mem->bo, true);
285 if (result != VK_SUCCESS)
286 goto fail;
287 }
288 }
289
290 *pMem = radv_device_memory_to_handle(mem);
291 radv_rmv_log_heap_create(device, *pMem, is_internal, flags_info ? flags_info->flags : 0);
292
293 return VK_SUCCESS;
294
295 fail:
296 radv_free_memory(device, pAllocator, mem);
297
298 return result;
299 }
300
301 VKAPI_ATTR VkResult VKAPI_CALL
radv_AllocateMemory(VkDevice _device,const VkMemoryAllocateInfo * pAllocateInfo,const VkAllocationCallbacks * pAllocator,VkDeviceMemory * pMem)302 radv_AllocateMemory(VkDevice _device, const VkMemoryAllocateInfo *pAllocateInfo,
303 const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMem)
304 {
305 VK_FROM_HANDLE(radv_device, device, _device);
306 return radv_alloc_memory(device, pAllocateInfo, pAllocator, pMem, false);
307 }
308
309 VKAPI_ATTR void VKAPI_CALL
radv_FreeMemory(VkDevice _device,VkDeviceMemory _mem,const VkAllocationCallbacks * pAllocator)310 radv_FreeMemory(VkDevice _device, VkDeviceMemory _mem, const VkAllocationCallbacks *pAllocator)
311 {
312 VK_FROM_HANDLE(radv_device, device, _device);
313 VK_FROM_HANDLE(radv_device_memory, mem, _mem);
314
315 radv_free_memory(device, pAllocator, mem);
316 }
317
318 VKAPI_ATTR VkResult VKAPI_CALL
radv_MapMemory2(VkDevice _device,const VkMemoryMapInfo * pMemoryMapInfo,void ** ppData)319 radv_MapMemory2(VkDevice _device, const VkMemoryMapInfo *pMemoryMapInfo, void **ppData)
320 {
321 VK_FROM_HANDLE(radv_device, device, _device);
322 VK_FROM_HANDLE(radv_device_memory, mem, pMemoryMapInfo->memory);
323 void *fixed_address = NULL;
324 bool use_fixed_address = false;
325
326 if (pMemoryMapInfo->flags & VK_MEMORY_MAP_PLACED_BIT_EXT) {
327 const VkMemoryMapPlacedInfoEXT *placed_info =
328 vk_find_struct_const(pMemoryMapInfo->pNext, MEMORY_MAP_PLACED_INFO_EXT);
329 if (placed_info) {
330 fixed_address = placed_info->pPlacedAddress;
331 use_fixed_address = true;
332 }
333 }
334
335 if (mem->user_ptr)
336 *ppData = mem->user_ptr;
337 else
338 *ppData = device->ws->buffer_map(device->ws, mem->bo, use_fixed_address, fixed_address);
339
340 if (*ppData) {
341 vk_rmv_log_cpu_map(&device->vk, mem->bo->va, false);
342 *ppData = (uint8_t *)*ppData + pMemoryMapInfo->offset;
343 return VK_SUCCESS;
344 }
345
346 return vk_error(device, VK_ERROR_MEMORY_MAP_FAILED);
347 }
348
349 VKAPI_ATTR VkResult VKAPI_CALL
radv_UnmapMemory2(VkDevice _device,const VkMemoryUnmapInfo * pMemoryUnmapInfo)350 radv_UnmapMemory2(VkDevice _device, const VkMemoryUnmapInfo *pMemoryUnmapInfo)
351 {
352 VK_FROM_HANDLE(radv_device, device, _device);
353 VK_FROM_HANDLE(radv_device_memory, mem, pMemoryUnmapInfo->memory);
354
355 vk_rmv_log_cpu_map(&device->vk, mem->bo->va, true);
356 if (mem->user_ptr == NULL)
357 device->ws->buffer_unmap(device->ws, mem->bo, (pMemoryUnmapInfo->flags & VK_MEMORY_UNMAP_RESERVE_BIT_EXT));
358
359 return VK_SUCCESS;
360 }
361
362 VKAPI_ATTR VkResult VKAPI_CALL
radv_FlushMappedMemoryRanges(VkDevice _device,uint32_t memoryRangeCount,const VkMappedMemoryRange * pMemoryRanges)363 radv_FlushMappedMemoryRanges(VkDevice _device, uint32_t memoryRangeCount, const VkMappedMemoryRange *pMemoryRanges)
364 {
365 return VK_SUCCESS;
366 }
367
368 VKAPI_ATTR VkResult VKAPI_CALL
radv_InvalidateMappedMemoryRanges(VkDevice _device,uint32_t memoryRangeCount,const VkMappedMemoryRange * pMemoryRanges)369 radv_InvalidateMappedMemoryRanges(VkDevice _device, uint32_t memoryRangeCount, const VkMappedMemoryRange *pMemoryRanges)
370 {
371 return VK_SUCCESS;
372 }
373
374 VKAPI_ATTR uint64_t VKAPI_CALL
radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device,const VkDeviceMemoryOpaqueCaptureAddressInfo * pInfo)375 radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device, const VkDeviceMemoryOpaqueCaptureAddressInfo *pInfo)
376 {
377 VK_FROM_HANDLE(radv_device_memory, mem, pInfo->memory);
378 return radv_buffer_get_va(mem->bo);
379 }
380
381 VKAPI_ATTR void VKAPI_CALL
radv_GetDeviceMemoryCommitment(VkDevice device,VkDeviceMemory memory,VkDeviceSize * pCommittedMemoryInBytes)382 radv_GetDeviceMemoryCommitment(VkDevice device, VkDeviceMemory memory, VkDeviceSize *pCommittedMemoryInBytes)
383 {
384 *pCommittedMemoryInBytes = 0;
385 }
386