1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * SPDX-License-Identifier: MIT
9 */
10
11 #include "radv_device_memory.h"
12 #include "radv_android.h"
13 #include "radv_buffer.h"
14 #include "radv_entrypoints.h"
15 #include "radv_image.h"
16 #include "radv_rmv.h"
17
18 #include "vk_log.h"
19
20 void
radv_device_memory_init(struct radv_device_memory * mem,struct radv_device * device,struct radeon_winsys_bo * bo)21 radv_device_memory_init(struct radv_device_memory *mem, struct radv_device *device, struct radeon_winsys_bo *bo)
22 {
23 memset(mem, 0, sizeof(*mem));
24 vk_object_base_init(&device->vk, &mem->base, VK_OBJECT_TYPE_DEVICE_MEMORY);
25
26 mem->bo = bo;
27 }
28
29 void
radv_device_memory_finish(struct radv_device_memory * mem)30 radv_device_memory_finish(struct radv_device_memory *mem)
31 {
32 vk_object_base_finish(&mem->base);
33 }
34
35 void
radv_free_memory(struct radv_device * device,const VkAllocationCallbacks * pAllocator,struct radv_device_memory * mem)36 radv_free_memory(struct radv_device *device, const VkAllocationCallbacks *pAllocator, struct radv_device_memory *mem)
37 {
38 if (mem == NULL)
39 return;
40
41 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
42 if (mem->android_hardware_buffer)
43 AHardwareBuffer_release(mem->android_hardware_buffer);
44 #endif
45
46 if (mem->bo) {
47 if (device->overallocation_disallowed) {
48 mtx_lock(&device->overallocation_mutex);
49 device->allocated_memory_size[mem->heap_index] -= mem->alloc_size;
50 mtx_unlock(&device->overallocation_mutex);
51 }
52
53 if (device->use_global_bo_list)
54 device->ws->buffer_make_resident(device->ws, mem->bo, false);
55 radv_bo_destroy(device, &mem->base, mem->bo);
56 mem->bo = NULL;
57 }
58
59 radv_rmv_log_resource_destroy(device, (uint64_t)radv_device_memory_to_handle(mem));
60 radv_device_memory_finish(mem);
61 vk_free2(&device->vk.alloc, pAllocator, mem);
62 }
63
64 VkResult
radv_alloc_memory(struct radv_device * device,const VkMemoryAllocateInfo * pAllocateInfo,const VkAllocationCallbacks * pAllocator,VkDeviceMemory * pMem,bool is_internal)65 radv_alloc_memory(struct radv_device *device, const VkMemoryAllocateInfo *pAllocateInfo,
66 const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMem, bool is_internal)
67 {
68 struct radv_device_memory *mem;
69 VkResult result;
70 enum radeon_bo_domain domain;
71 uint32_t flags = 0;
72
73 assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
74
75 const VkImportMemoryFdInfoKHR *import_info = vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_FD_INFO_KHR);
76 const VkMemoryDedicatedAllocateInfo *dedicate_info =
77 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_DEDICATED_ALLOCATE_INFO);
78 const VkExportMemoryAllocateInfo *export_info =
79 vk_find_struct_const(pAllocateInfo->pNext, EXPORT_MEMORY_ALLOCATE_INFO);
80 const struct VkImportAndroidHardwareBufferInfoANDROID *ahb_import_info =
81 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID);
82 const VkImportMemoryHostPointerInfoEXT *host_ptr_info =
83 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT);
84 const struct VkMemoryAllocateFlagsInfo *flags_info =
85 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_ALLOCATE_FLAGS_INFO);
86
87 const struct wsi_memory_allocate_info *wsi_info =
88 vk_find_struct_const(pAllocateInfo->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA);
89
90 if (pAllocateInfo->allocationSize == 0 && !ahb_import_info &&
91 !(export_info &&
92 (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID))) {
93 /* Apparently, this is allowed */
94 *pMem = VK_NULL_HANDLE;
95 return VK_SUCCESS;
96 }
97
98 mem = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*mem), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
99 if (mem == NULL)
100 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
101
102 radv_device_memory_init(mem, device, NULL);
103
104 if (dedicate_info) {
105 mem->image = radv_image_from_handle(dedicate_info->image);
106 mem->buffer = radv_buffer_from_handle(dedicate_info->buffer);
107 } else {
108 mem->image = NULL;
109 mem->buffer = NULL;
110 }
111
112 if (wsi_info && wsi_info->implicit_sync) {
113 flags |= RADEON_FLAG_IMPLICIT_SYNC;
114
115 /* Mark the linear prime buffer (aka the destination of the prime blit
116 * as uncached.
117 */
118 if (mem->buffer)
119 flags |= RADEON_FLAG_VA_UNCACHED;
120 }
121
122 float priority_float = 0.5;
123 const struct VkMemoryPriorityAllocateInfoEXT *priority_ext =
124 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_PRIORITY_ALLOCATE_INFO_EXT);
125 if (priority_ext)
126 priority_float = priority_ext->priority;
127
128 uint64_t replay_address = 0;
129 const VkMemoryOpaqueCaptureAddressAllocateInfo *replay_info =
130 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_OPAQUE_CAPTURE_ADDRESS_ALLOCATE_INFO);
131 if (replay_info && replay_info->opaqueCaptureAddress)
132 replay_address = replay_info->opaqueCaptureAddress;
133
134 unsigned priority =
135 MIN2(RADV_BO_PRIORITY_APPLICATION_MAX - 1, (int)(priority_float * RADV_BO_PRIORITY_APPLICATION_MAX));
136
137 mem->user_ptr = NULL;
138
139 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
140 mem->android_hardware_buffer = NULL;
141 #endif
142
143 if (ahb_import_info) {
144 result = radv_import_ahb_memory(device, mem, priority, ahb_import_info);
145 if (result != VK_SUCCESS)
146 goto fail;
147 } else if (export_info &&
148 (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID)) {
149 result = radv_create_ahb_memory(device, mem, priority, pAllocateInfo);
150 if (result != VK_SUCCESS)
151 goto fail;
152 } else if (import_info) {
153 assert(import_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
154 import_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
155 result = radv_bo_from_fd(device, import_info->fd, priority, mem, NULL);
156 if (result != VK_SUCCESS) {
157 goto fail;
158 } else {
159 close(import_info->fd);
160 }
161
162 if (mem->image && mem->image->plane_count == 1 && !vk_format_is_depth_or_stencil(mem->image->vk.format) &&
163 mem->image->vk.samples == 1 && mem->image->vk.tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
164 struct radeon_bo_metadata metadata;
165 device->ws->buffer_get_metadata(device->ws, mem->bo, &metadata);
166
167 struct radv_image_create_info create_info = {.no_metadata_planes = true, .bo_metadata = &metadata};
168
169 /* This gives a basic ability to import radeonsi images
170 * that don't have DCC. This is not guaranteed by any
171 * spec and can be removed after we support modifiers. */
172 result = radv_image_create_layout(device, create_info, NULL, NULL, mem->image);
173 if (result != VK_SUCCESS) {
174 radv_bo_destroy(device, &mem->base, mem->bo);
175 goto fail;
176 }
177 }
178 } else if (host_ptr_info) {
179 assert(host_ptr_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
180 result = radv_bo_from_ptr(device, host_ptr_info->pHostPointer, pAllocateInfo->allocationSize, priority, mem);
181 if (result != VK_SUCCESS) {
182 goto fail;
183 } else {
184 mem->user_ptr = host_ptr_info->pHostPointer;
185 }
186 } else {
187 const struct radv_physical_device *pdev = radv_device_physical(device);
188 const struct radv_instance *instance = radv_physical_device_instance(pdev);
189 uint64_t alloc_size = align64(pAllocateInfo->allocationSize, 4096);
190 uint32_t heap_index;
191
192 heap_index = pdev->memory_properties.memoryTypes[pAllocateInfo->memoryTypeIndex].heapIndex;
193 domain = pdev->memory_domains[pAllocateInfo->memoryTypeIndex];
194 flags |= pdev->memory_flags[pAllocateInfo->memoryTypeIndex];
195
196 if (export_info && export_info->handleTypes) {
197 /* Setting RADEON_FLAG_GTT_WC in case the bo is spilled to GTT. This is important when the
198 * foreign queue is the display engine of iGPU. The carveout of iGPU can be tiny and the
199 * kernel driver refuses to spill without the flag.
200 *
201 * This covers any external memory user, including WSI.
202 */
203 if (domain == RADEON_DOMAIN_VRAM)
204 flags |= RADEON_FLAG_GTT_WC;
205 } else if (!import_info) {
206 /* neither export nor import */
207 flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
208 if (device->use_global_bo_list) {
209 flags |= RADEON_FLAG_PREFER_LOCAL_BO;
210 }
211 }
212
213 if (flags_info && flags_info->flags & VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT)
214 flags |= RADEON_FLAG_REPLAYABLE;
215
216 if (instance->drirc.zero_vram)
217 flags |= RADEON_FLAG_ZERO_VRAM;
218
219 if (device->overallocation_disallowed) {
220 uint64_t total_size = pdev->memory_properties.memoryHeaps[heap_index].size;
221
222 mtx_lock(&device->overallocation_mutex);
223 if (device->allocated_memory_size[heap_index] + alloc_size > total_size) {
224 mtx_unlock(&device->overallocation_mutex);
225 result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
226 goto fail;
227 }
228 device->allocated_memory_size[heap_index] += alloc_size;
229 mtx_unlock(&device->overallocation_mutex);
230 }
231
232 result = radv_bo_create(device, &mem->base, alloc_size, pdev->info.max_alignment, domain, flags, priority,
233 replay_address, is_internal, &mem->bo);
234
235 if (result != VK_SUCCESS) {
236 if (device->overallocation_disallowed) {
237 mtx_lock(&device->overallocation_mutex);
238 device->allocated_memory_size[heap_index] -= alloc_size;
239 mtx_unlock(&device->overallocation_mutex);
240 }
241 goto fail;
242 }
243
244 mem->heap_index = heap_index;
245 mem->alloc_size = alloc_size;
246 }
247
248 if (!wsi_info) {
249 if (device->use_global_bo_list) {
250 result = device->ws->buffer_make_resident(device->ws, mem->bo, true);
251 if (result != VK_SUCCESS)
252 goto fail;
253 }
254 }
255
256 *pMem = radv_device_memory_to_handle(mem);
257 radv_rmv_log_heap_create(device, *pMem, is_internal, flags_info ? flags_info->flags : 0);
258
259 return VK_SUCCESS;
260
261 fail:
262 radv_free_memory(device, pAllocator, mem);
263
264 return result;
265 }
266
267 VKAPI_ATTR VkResult VKAPI_CALL
radv_AllocateMemory(VkDevice _device,const VkMemoryAllocateInfo * pAllocateInfo,const VkAllocationCallbacks * pAllocator,VkDeviceMemory * pMem)268 radv_AllocateMemory(VkDevice _device, const VkMemoryAllocateInfo *pAllocateInfo,
269 const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMem)
270 {
271 VK_FROM_HANDLE(radv_device, device, _device);
272 return radv_alloc_memory(device, pAllocateInfo, pAllocator, pMem, false);
273 }
274
275 VKAPI_ATTR void VKAPI_CALL
radv_FreeMemory(VkDevice _device,VkDeviceMemory _mem,const VkAllocationCallbacks * pAllocator)276 radv_FreeMemory(VkDevice _device, VkDeviceMemory _mem, const VkAllocationCallbacks *pAllocator)
277 {
278 VK_FROM_HANDLE(radv_device, device, _device);
279 VK_FROM_HANDLE(radv_device_memory, mem, _mem);
280
281 radv_free_memory(device, pAllocator, mem);
282 }
283
284 VKAPI_ATTR VkResult VKAPI_CALL
radv_MapMemory2(VkDevice _device,const VkMemoryMapInfo * pMemoryMapInfo,void ** ppData)285 radv_MapMemory2(VkDevice _device, const VkMemoryMapInfo *pMemoryMapInfo, void **ppData)
286 {
287 VK_FROM_HANDLE(radv_device, device, _device);
288 VK_FROM_HANDLE(radv_device_memory, mem, pMemoryMapInfo->memory);
289 void *fixed_address = NULL;
290 bool use_fixed_address = false;
291
292 if (pMemoryMapInfo->flags & VK_MEMORY_MAP_PLACED_BIT_EXT) {
293 const VkMemoryMapPlacedInfoEXT *placed_info =
294 vk_find_struct_const(pMemoryMapInfo->pNext, MEMORY_MAP_PLACED_INFO_EXT);
295 if (placed_info) {
296 fixed_address = placed_info->pPlacedAddress;
297 use_fixed_address = true;
298 }
299 }
300
301 if (mem->user_ptr)
302 *ppData = mem->user_ptr;
303 else
304 *ppData = device->ws->buffer_map(device->ws, mem->bo, use_fixed_address, fixed_address);
305
306 if (*ppData) {
307 vk_rmv_log_cpu_map(&device->vk, mem->bo->va, false);
308 *ppData = (uint8_t *)*ppData + pMemoryMapInfo->offset;
309 return VK_SUCCESS;
310 }
311
312 return vk_error(device, VK_ERROR_MEMORY_MAP_FAILED);
313 }
314
315 VKAPI_ATTR VkResult VKAPI_CALL
radv_UnmapMemory2(VkDevice _device,const VkMemoryUnmapInfo * pMemoryUnmapInfo)316 radv_UnmapMemory2(VkDevice _device, const VkMemoryUnmapInfo *pMemoryUnmapInfo)
317 {
318 VK_FROM_HANDLE(radv_device, device, _device);
319 VK_FROM_HANDLE(radv_device_memory, mem, pMemoryUnmapInfo->memory);
320
321 vk_rmv_log_cpu_map(&device->vk, mem->bo->va, true);
322 if (mem->user_ptr == NULL)
323 device->ws->buffer_unmap(device->ws, mem->bo, (pMemoryUnmapInfo->flags & VK_MEMORY_UNMAP_RESERVE_BIT_EXT));
324
325 return VK_SUCCESS;
326 }
327
328 VKAPI_ATTR VkResult VKAPI_CALL
radv_FlushMappedMemoryRanges(VkDevice _device,uint32_t memoryRangeCount,const VkMappedMemoryRange * pMemoryRanges)329 radv_FlushMappedMemoryRanges(VkDevice _device, uint32_t memoryRangeCount, const VkMappedMemoryRange *pMemoryRanges)
330 {
331 return VK_SUCCESS;
332 }
333
334 VKAPI_ATTR VkResult VKAPI_CALL
radv_InvalidateMappedMemoryRanges(VkDevice _device,uint32_t memoryRangeCount,const VkMappedMemoryRange * pMemoryRanges)335 radv_InvalidateMappedMemoryRanges(VkDevice _device, uint32_t memoryRangeCount, const VkMappedMemoryRange *pMemoryRanges)
336 {
337 return VK_SUCCESS;
338 }
339
340 VKAPI_ATTR uint64_t VKAPI_CALL
radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device,const VkDeviceMemoryOpaqueCaptureAddressInfo * pInfo)341 radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device, const VkDeviceMemoryOpaqueCaptureAddressInfo *pInfo)
342 {
343 VK_FROM_HANDLE(radv_device_memory, mem, pInfo->memory);
344 return radv_buffer_get_va(mem->bo);
345 }
346
347 VKAPI_ATTR void VKAPI_CALL
radv_GetDeviceMemoryCommitment(VkDevice device,VkDeviceMemory memory,VkDeviceSize * pCommittedMemoryInBytes)348 radv_GetDeviceMemoryCommitment(VkDevice device, VkDeviceMemory memory, VkDeviceSize *pCommittedMemoryInBytes)
349 {
350 *pCommittedMemoryInBytes = 0;
351 }
352