1 /* Copyright © 2024 Intel Corporation
2 * SPDX-License-Identifier: MIT
3 */
4
5 #include "anv_private.h"
6
7 static void
anv_bind_buffer_memory(struct anv_device * device,const VkBindBufferMemoryInfo * pBindInfo)8 anv_bind_buffer_memory(struct anv_device *device,
9 const VkBindBufferMemoryInfo *pBindInfo)
10 {
11 ANV_FROM_HANDLE(anv_device_memory, mem, pBindInfo->memory);
12 ANV_FROM_HANDLE(anv_buffer, buffer, pBindInfo->buffer);
13
14 assert(pBindInfo->sType == VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO);
15 assert(!anv_buffer_is_sparse(buffer));
16
17 const VkBindMemoryStatusKHR *bind_status =
18 vk_find_struct_const(pBindInfo->pNext, BIND_MEMORY_STATUS_KHR);
19
20 if (mem) {
21 assert(pBindInfo->memoryOffset < mem->vk.size);
22 assert(mem->vk.size - pBindInfo->memoryOffset >= buffer->vk.size);
23 buffer->address = (struct anv_address) {
24 .bo = mem->bo,
25 .offset = pBindInfo->memoryOffset,
26 };
27 } else {
28 buffer->address = ANV_NULL_ADDRESS;
29 }
30
31 ANV_RMV(buffer_bind, device, buffer);
32
33 if (bind_status)
34 *bind_status->pResult = VK_SUCCESS;
35 }
36
anv_BindBufferMemory2(VkDevice _device,uint32_t bindInfoCount,const VkBindBufferMemoryInfo * pBindInfos)37 VkResult anv_BindBufferMemory2(
38 VkDevice _device,
39 uint32_t bindInfoCount,
40 const VkBindBufferMemoryInfo* pBindInfos)
41 {
42 ANV_FROM_HANDLE(anv_device, device, _device);
43
44 for (uint32_t i = 0; i < bindInfoCount; i++)
45 anv_bind_buffer_memory(device, &pBindInfos[i]);
46
47 return VK_SUCCESS;
48 }
49
50 // Buffer functions
51
52 static void
anv_get_buffer_memory_requirements(struct anv_device * device,VkBufferCreateFlags flags,VkDeviceSize size,VkBufferUsageFlags2KHR usage,bool is_sparse,VkMemoryRequirements2 * pMemoryRequirements)53 anv_get_buffer_memory_requirements(struct anv_device *device,
54 VkBufferCreateFlags flags,
55 VkDeviceSize size,
56 VkBufferUsageFlags2KHR usage,
57 bool is_sparse,
58 VkMemoryRequirements2* pMemoryRequirements)
59 {
60 /* The Vulkan spec (git aaed022) says:
61 *
62 * memoryTypeBits is a bitfield and contains one bit set for every
63 * supported memory type for the resource. The bit `1<<i` is set if and
64 * only if the memory type `i` in the VkPhysicalDeviceMemoryProperties
65 * structure for the physical device is supported.
66 *
67 * We have special memory types for descriptor buffers.
68 */
69 uint32_t memory_types;
70 if (flags & VK_BUFFER_CREATE_PROTECTED_BIT)
71 memory_types = device->physical->memory.protected_mem_types;
72 else if (usage & (VK_BUFFER_USAGE_2_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT |
73 VK_BUFFER_USAGE_2_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT))
74 memory_types = device->physical->memory.dynamic_visible_mem_types;
75 else if (device->physical->instance->enable_buffer_comp)
76 memory_types = device->physical->memory.default_buffer_mem_types |
77 device->physical->memory.compressed_mem_types;
78 else
79 memory_types = device->physical->memory.default_buffer_mem_types;
80
81 /* The GPU appears to write back to main memory in cachelines. Writes to a
82 * buffers should not clobber with writes to another buffers so make sure
83 * those are in different cachelines.
84 */
85 uint32_t alignment = 64;
86
87 /* From the spec, section "Sparse Buffer and Fully-Resident Image Block
88 * Size":
89 * "The sparse block size in bytes for sparse buffers and fully-resident
90 * images is reported as VkMemoryRequirements::alignment. alignment
91 * represents both the memory alignment requirement and the binding
92 * granularity (in bytes) for sparse resources."
93 */
94 if (is_sparse) {
95 alignment = ANV_SPARSE_BLOCK_SIZE;
96 size = align64(size, alignment);
97 }
98
99 pMemoryRequirements->memoryRequirements.size = size;
100 pMemoryRequirements->memoryRequirements.alignment = alignment;
101
102 /* Storage and Uniform buffers should have their size aligned to
103 * 32-bits to avoid boundary checks when last DWord is not complete.
104 * This would ensure that not internal padding would be needed for
105 * 16-bit types.
106 */
107 if (device->robust_buffer_access &&
108 (usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT ||
109 usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT))
110 pMemoryRequirements->memoryRequirements.size = align64(size, 4);
111
112 pMemoryRequirements->memoryRequirements.memoryTypeBits = memory_types;
113
114 vk_foreach_struct(ext, pMemoryRequirements->pNext) {
115 switch (ext->sType) {
116 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
117 VkMemoryDedicatedRequirements *requirements = (void *)ext;
118 requirements->prefersDedicatedAllocation = false;
119 requirements->requiresDedicatedAllocation = false;
120 break;
121 }
122
123 default:
124 vk_debug_ignored_stype(ext->sType);
125 break;
126 }
127 }
128 }
129
130 static VkBufferUsageFlags2KHR
get_buffer_usages(const VkBufferCreateInfo * create_info)131 get_buffer_usages(const VkBufferCreateInfo *create_info)
132 {
133 const VkBufferUsageFlags2CreateInfoKHR *usage2_info =
134 vk_find_struct_const(create_info->pNext,
135 BUFFER_USAGE_FLAGS_2_CREATE_INFO_KHR);
136 return usage2_info != NULL ? usage2_info->usage : create_info->usage;
137 }
138
anv_GetDeviceBufferMemoryRequirements(VkDevice _device,const VkDeviceBufferMemoryRequirements * pInfo,VkMemoryRequirements2 * pMemoryRequirements)139 void anv_GetDeviceBufferMemoryRequirements(
140 VkDevice _device,
141 const VkDeviceBufferMemoryRequirements* pInfo,
142 VkMemoryRequirements2* pMemoryRequirements)
143 {
144 ANV_FROM_HANDLE(anv_device, device, _device);
145 const bool is_sparse =
146 pInfo->pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT;
147 VkBufferUsageFlags2KHR usages = get_buffer_usages(pInfo->pCreateInfo);
148
149 if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
150 INTEL_DEBUG(DEBUG_SPARSE) &&
151 pInfo->pCreateInfo->flags & (VK_BUFFER_CREATE_SPARSE_BINDING_BIT |
152 VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT |
153 VK_BUFFER_CREATE_SPARSE_ALIASED_BIT))
154 fprintf(stderr, "=== %s %s:%d flags:0x%08x\n", __func__, __FILE__,
155 __LINE__, pInfo->pCreateInfo->flags);
156
157 anv_get_buffer_memory_requirements(device,
158 pInfo->pCreateInfo->flags,
159 pInfo->pCreateInfo->size,
160 usages,
161 is_sparse,
162 pMemoryRequirements);
163 }
164
anv_CreateBuffer(VkDevice _device,const VkBufferCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkBuffer * pBuffer)165 VkResult anv_CreateBuffer(
166 VkDevice _device,
167 const VkBufferCreateInfo* pCreateInfo,
168 const VkAllocationCallbacks* pAllocator,
169 VkBuffer* pBuffer)
170 {
171 ANV_FROM_HANDLE(anv_device, device, _device);
172 struct anv_buffer *buffer;
173
174 if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
175 INTEL_DEBUG(DEBUG_SPARSE) &&
176 pCreateInfo->flags & (VK_BUFFER_CREATE_SPARSE_BINDING_BIT |
177 VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT |
178 VK_BUFFER_CREATE_SPARSE_ALIASED_BIT))
179 fprintf(stderr, "=== %s %s:%d flags:0x%08x\n", __func__, __FILE__,
180 __LINE__, pCreateInfo->flags);
181
182 if ((pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) &&
183 device->physical->sparse_type == ANV_SPARSE_TYPE_TRTT) {
184 VkBufferUsageFlags2KHR usages = get_buffer_usages(pCreateInfo);
185 if (usages & (VK_BUFFER_USAGE_2_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT |
186 VK_BUFFER_USAGE_2_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT)) {
187 return vk_errorf(device, VK_ERROR_UNKNOWN,
188 "Cannot support sparse descriptor buffers with TRTT.");
189 }
190 }
191
192 /* Don't allow creating buffers bigger than our address space. The real
193 * issue here is that we may align up the buffer size and we don't want
194 * doing so to cause roll-over. However, no one has any business
195 * allocating a buffer larger than our GTT size.
196 */
197 if (pCreateInfo->size > device->physical->gtt_size)
198 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
199
200 buffer = vk_buffer_create(&device->vk, pCreateInfo,
201 pAllocator, sizeof(*buffer));
202 if (buffer == NULL)
203 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
204
205 buffer->address = ANV_NULL_ADDRESS;
206 if (anv_buffer_is_sparse(buffer)) {
207 enum anv_bo_alloc_flags alloc_flags = 0;
208 uint64_t client_address = 0;
209
210 if (buffer->vk.create_flags & VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT) {
211 alloc_flags = ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS;
212 const VkBufferOpaqueCaptureAddressCreateInfo *opaque_addr_info =
213 vk_find_struct_const(pCreateInfo->pNext,
214 BUFFER_OPAQUE_CAPTURE_ADDRESS_CREATE_INFO);
215 if (opaque_addr_info)
216 client_address = opaque_addr_info->opaqueCaptureAddress;
217 }
218
219 if (buffer->vk.create_flags & VK_BUFFER_CREATE_DESCRIPTOR_BUFFER_CAPTURE_REPLAY_BIT_EXT) {
220 alloc_flags = ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS;
221
222 const VkOpaqueCaptureDescriptorDataCreateInfoEXT *opaque_info =
223 vk_find_struct_const(pCreateInfo->pNext,
224 OPAQUE_CAPTURE_DESCRIPTOR_DATA_CREATE_INFO_EXT);
225 if (opaque_info)
226 client_address = *((const uint64_t *)opaque_info->opaqueCaptureDescriptorData);
227 }
228
229 /* If this buffer will be used as a descriptor buffer, make sure we
230 * allocate it on the correct heap.
231 */
232 if (buffer->vk.usage & (VK_BUFFER_USAGE_2_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT |
233 VK_BUFFER_USAGE_2_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT)) {
234 alloc_flags |= ANV_BO_ALLOC_DYNAMIC_VISIBLE_POOL;
235 }
236
237 VkResult result = anv_init_sparse_bindings(device, buffer->vk.size,
238 &buffer->sparse_data,
239 alloc_flags, client_address,
240 &buffer->address);
241 if (result != VK_SUCCESS) {
242 vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk);
243 return result;
244 }
245 }
246
247 ANV_RMV(buffer_create, device, false, buffer);
248
249 *pBuffer = anv_buffer_to_handle(buffer);
250
251 return VK_SUCCESS;
252 }
253
anv_DestroyBuffer(VkDevice _device,VkBuffer _buffer,const VkAllocationCallbacks * pAllocator)254 void anv_DestroyBuffer(
255 VkDevice _device,
256 VkBuffer _buffer,
257 const VkAllocationCallbacks* pAllocator)
258 {
259 ANV_FROM_HANDLE(anv_device, device, _device);
260 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
261
262 if (!buffer)
263 return;
264
265 ANV_RMV(buffer_destroy, device, buffer);
266
267 if (anv_buffer_is_sparse(buffer)) {
268 assert(buffer->address.offset == buffer->sparse_data.address);
269 anv_free_sparse_bindings(device, &buffer->sparse_data);
270 }
271
272 vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk);
273 }
274
anv_GetBufferDeviceAddress(VkDevice device,const VkBufferDeviceAddressInfo * pInfo)275 VkDeviceAddress anv_GetBufferDeviceAddress(
276 VkDevice device,
277 const VkBufferDeviceAddressInfo* pInfo)
278 {
279 ANV_FROM_HANDLE(anv_buffer, buffer, pInfo->buffer);
280
281 assert(!anv_address_is_null(buffer->address));
282
283 return anv_address_physical(buffer->address);
284 }
285
anv_GetBufferOpaqueCaptureAddress(VkDevice device,const VkBufferDeviceAddressInfo * pInfo)286 uint64_t anv_GetBufferOpaqueCaptureAddress(
287 VkDevice device,
288 const VkBufferDeviceAddressInfo* pInfo)
289 {
290 ANV_FROM_HANDLE(anv_buffer, buffer, pInfo->buffer);
291
292 return anv_address_physical(buffer->address);
293 }
294
anv_GetBufferOpaqueCaptureDescriptorDataEXT(VkDevice device,const VkBufferCaptureDescriptorDataInfoEXT * pInfo,void * pData)295 VkResult anv_GetBufferOpaqueCaptureDescriptorDataEXT(
296 VkDevice device,
297 const VkBufferCaptureDescriptorDataInfoEXT* pInfo,
298 void* pData)
299 {
300 ANV_FROM_HANDLE(anv_buffer, buffer, pInfo->buffer);
301
302 *((uint64_t *)pData) = anv_address_physical(buffer->address);
303
304 return VK_SUCCESS;
305 }
306
anv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device,const VkDeviceMemoryOpaqueCaptureAddressInfo * pInfo)307 uint64_t anv_GetDeviceMemoryOpaqueCaptureAddress(
308 VkDevice device,
309 const VkDeviceMemoryOpaqueCaptureAddressInfo* pInfo)
310 {
311 ANV_FROM_HANDLE(anv_device_memory, memory, pInfo->memory);
312
313 assert(memory->bo->alloc_flags & ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS);
314
315 return intel_48b_address(memory->bo->offset);
316 }
317
318 void
anv_fill_buffer_surface_state(struct anv_device * device,void * surface_state_ptr,enum isl_format format,struct isl_swizzle swizzle,isl_surf_usage_flags_t usage,struct anv_address address,uint32_t range,uint32_t stride)319 anv_fill_buffer_surface_state(struct anv_device *device,
320 void *surface_state_ptr,
321 enum isl_format format,
322 struct isl_swizzle swizzle,
323 isl_surf_usage_flags_t usage,
324 struct anv_address address,
325 uint32_t range, uint32_t stride)
326 {
327 if (address.bo && address.bo->alloc_flags & ANV_BO_ALLOC_PROTECTED)
328 usage |= ISL_SURF_USAGE_PROTECTED_BIT;
329 isl_buffer_fill_state(&device->isl_dev, surface_state_ptr,
330 .address = anv_address_physical(address),
331 .mocs = isl_mocs(&device->isl_dev, usage,
332 address.bo && anv_bo_is_external(address.bo)),
333 .size_B = range,
334 .format = format,
335 .swizzle = swizzle,
336 .stride_B = stride);
337 }
338