1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24 #include <assert.h>
25 #include <fcntl.h>
26 #include <stdbool.h>
27 #include <string.h>
28
29 #include "util/mesa-sha1.h"
30 #include "radv_private.h"
31 #include "sid.h"
32 #include "vk_descriptors.h"
33 #include "vk_format.h"
34 #include "vk_util.h"
35
36 static unsigned
radv_descriptor_type_buffer_count(VkDescriptorType type)37 radv_descriptor_type_buffer_count(VkDescriptorType type)
38 {
39 switch (type) {
40 case VK_DESCRIPTOR_TYPE_SAMPLER:
41 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
42 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
43 return 0;
44 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
45 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
46 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
47 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
48 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE:
49 return 3;
50 default:
51 return 1;
52 }
53 }
54
55 static bool
has_equal_immutable_samplers(const VkSampler * samplers,uint32_t count)56 has_equal_immutable_samplers(const VkSampler *samplers, uint32_t count)
57 {
58 if (!samplers)
59 return false;
60 for (uint32_t i = 1; i < count; ++i) {
61 if (memcmp(radv_sampler_from_handle(samplers[0])->state,
62 radv_sampler_from_handle(samplers[i])->state, 16)) {
63 return false;
64 }
65 }
66 return true;
67 }
68
69 static bool
radv_mutable_descriptor_type_size_alignment(const VkMutableDescriptorTypeListVALVE * list,uint64_t * out_size,uint64_t * out_align)70 radv_mutable_descriptor_type_size_alignment(const VkMutableDescriptorTypeListVALVE *list,
71 uint64_t *out_size, uint64_t *out_align)
72 {
73 uint32_t max_size = 0;
74 uint32_t max_align = 0;
75
76 for (uint32_t i = 0; i < list->descriptorTypeCount; i++) {
77 uint32_t size = 0;
78 uint32_t align = 0;
79
80 switch (list->pDescriptorTypes[i]) {
81 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
82 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
83 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
84 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
85 case VK_DESCRIPTOR_TYPE_SAMPLER:
86 size = 16;
87 align = 16;
88 break;
89 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
90 size = 32;
91 align = 32;
92 break;
93 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
94 size = 64;
95 align = 32;
96 break;
97 default:
98 return false;
99 }
100
101 max_size = MAX2(max_size, size);
102 max_align = MAX2(max_align, align);
103 }
104
105 *out_size = max_size;
106 *out_align = max_align;
107 return true;
108 }
109
110 VKAPI_ATTR VkResult VKAPI_CALL
radv_CreateDescriptorSetLayout(VkDevice _device,const VkDescriptorSetLayoutCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorSetLayout * pSetLayout)111 radv_CreateDescriptorSetLayout(VkDevice _device, const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
112 const VkAllocationCallbacks *pAllocator,
113 VkDescriptorSetLayout *pSetLayout)
114 {
115 RADV_FROM_HANDLE(radv_device, device, _device);
116 struct radv_descriptor_set_layout *set_layout;
117
118 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO);
119 const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags =
120 vk_find_struct_const(pCreateInfo->pNext, DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO);
121 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
122 vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
123
124 uint32_t num_bindings = 0;
125 uint32_t immutable_sampler_count = 0;
126 uint32_t ycbcr_sampler_count = 0;
127 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
128 num_bindings = MAX2(num_bindings, pCreateInfo->pBindings[j].binding + 1);
129 if ((pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
130 pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
131 pCreateInfo->pBindings[j].pImmutableSamplers) {
132 immutable_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
133
134 bool has_ycbcr_sampler = false;
135 for (unsigned i = 0; i < pCreateInfo->pBindings[j].descriptorCount; ++i) {
136 if (radv_sampler_from_handle(pCreateInfo->pBindings[j].pImmutableSamplers[i])
137 ->ycbcr_sampler)
138 has_ycbcr_sampler = true;
139 }
140
141 if (has_ycbcr_sampler)
142 ycbcr_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
143 }
144 }
145
146 uint32_t samplers_offset = offsetof(struct radv_descriptor_set_layout, binding[num_bindings]);
147 size_t size = samplers_offset + immutable_sampler_count * 4 * sizeof(uint32_t);
148 if (ycbcr_sampler_count > 0) {
149 /* Store block of offsets first, followed by the conversion descriptors (padded to the struct
150 * alignment) */
151 size += num_bindings * sizeof(uint32_t);
152 size = ALIGN(size, alignof(struct radv_sampler_ycbcr_conversion_state));
153 size += ycbcr_sampler_count * sizeof(struct radv_sampler_ycbcr_conversion_state);
154 }
155
156 /* We need to allocate decriptor set layouts off the device allocator with DEVICE scope because
157 * they are reference counted and may not be destroyed when vkDestroyDescriptorSetLayout is
158 * called.
159 */
160 set_layout = vk_descriptor_set_layout_zalloc(&device->vk, size);
161 if (!set_layout)
162 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
163
164 set_layout->flags = pCreateInfo->flags;
165 set_layout->layout_size = size;
166
167 /* We just allocate all the samplers at the end of the struct */
168 uint32_t *samplers = (uint32_t *)&set_layout->binding[num_bindings];
169 struct radv_sampler_ycbcr_conversion_state *ycbcr_samplers = NULL;
170 uint32_t *ycbcr_sampler_offsets = NULL;
171
172 if (ycbcr_sampler_count > 0) {
173 ycbcr_sampler_offsets = samplers + 4 * immutable_sampler_count;
174 set_layout->ycbcr_sampler_offsets_offset = (char *)ycbcr_sampler_offsets - (char *)set_layout;
175
176 uintptr_t first_ycbcr_sampler_offset =
177 (uintptr_t)ycbcr_sampler_offsets + sizeof(uint32_t) * num_bindings;
178 first_ycbcr_sampler_offset =
179 ALIGN(first_ycbcr_sampler_offset, alignof(struct radv_sampler_ycbcr_conversion_state));
180 ycbcr_samplers = (struct radv_sampler_ycbcr_conversion_state *)first_ycbcr_sampler_offset;
181 } else
182 set_layout->ycbcr_sampler_offsets_offset = 0;
183
184 VkDescriptorSetLayoutBinding *bindings = NULL;
185 VkResult result =
186 vk_create_sorted_bindings(pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
187 if (result != VK_SUCCESS) {
188 vk_descriptor_set_layout_unref(&device->vk, &set_layout->vk);
189 return vk_error(device, result);
190 }
191
192 set_layout->binding_count = num_bindings;
193 set_layout->shader_stages = 0;
194 set_layout->dynamic_shader_stages = 0;
195 set_layout->has_immutable_samplers = false;
196 set_layout->size = 0;
197
198 uint32_t buffer_count = 0;
199 uint32_t dynamic_offset_count = 0;
200
201 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
202 const VkDescriptorSetLayoutBinding *binding = bindings + j;
203 uint32_t b = binding->binding;
204 uint32_t alignment = 0;
205 unsigned binding_buffer_count =
206 radv_descriptor_type_buffer_count(binding->descriptorType);
207 uint32_t descriptor_count = binding->descriptorCount;
208 bool has_ycbcr_sampler = false;
209
210 /* main image + fmask */
211 uint32_t max_sampled_image_descriptors = 2;
212
213 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER &&
214 binding->pImmutableSamplers) {
215 for (unsigned i = 0; i < binding->descriptorCount; ++i) {
216 struct radv_sampler_ycbcr_conversion *conversion =
217 radv_sampler_from_handle(binding->pImmutableSamplers[i])->ycbcr_sampler;
218
219 if (conversion) {
220 has_ycbcr_sampler = true;
221 max_sampled_image_descriptors = MAX2(max_sampled_image_descriptors,
222 vk_format_get_plane_count(conversion->state.format));
223 }
224 }
225 }
226
227 switch (binding->descriptorType) {
228 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
229 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
230 assert(!(pCreateInfo->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
231 set_layout->binding[b].dynamic_offset_count = 1;
232 set_layout->dynamic_shader_stages |= binding->stageFlags;
233 if (binding->stageFlags & RADV_RT_STAGE_BITS)
234 set_layout->dynamic_shader_stages |= VK_SHADER_STAGE_COMPUTE_BIT;
235 set_layout->binding[b].size = 0;
236 alignment = 1;
237 break;
238 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
239 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
240 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
241 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
242 set_layout->binding[b].size = 16;
243 alignment = 16;
244 break;
245 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
246 set_layout->binding[b].size = 32;
247 alignment = 32;
248 break;
249 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
250 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
251 /* main descriptor + fmask descriptor */
252 set_layout->binding[b].size = 64;
253 alignment = 32;
254 break;
255 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
256 /* main descriptor + fmask descriptor + sampler */
257 set_layout->binding[b].size = 96;
258 alignment = 32;
259 break;
260 case VK_DESCRIPTOR_TYPE_SAMPLER:
261 set_layout->binding[b].size = 16;
262 alignment = 16;
263 break;
264 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE: {
265 uint64_t mutable_size = 0, mutable_align = 0;
266 radv_mutable_descriptor_type_size_alignment(&mutable_info->pMutableDescriptorTypeLists[j],
267 &mutable_size, &mutable_align);
268 assert(mutable_size && mutable_align);
269 set_layout->binding[b].size = mutable_size;
270 alignment = mutable_align;
271 break;
272 }
273 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
274 alignment = 16;
275 set_layout->binding[b].size = descriptor_count;
276 descriptor_count = 1;
277 break;
278 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
279 set_layout->binding[b].size = 16;
280 alignment = 16;
281 break;
282 default:
283 break;
284 }
285
286 set_layout->size = align(set_layout->size, alignment);
287 set_layout->binding[b].type = binding->descriptorType;
288 set_layout->binding[b].array_size = descriptor_count;
289 set_layout->binding[b].offset = set_layout->size;
290 set_layout->binding[b].buffer_offset = buffer_count;
291 set_layout->binding[b].dynamic_offset_offset = dynamic_offset_count;
292
293 if (variable_flags && binding->binding < variable_flags->bindingCount &&
294 (variable_flags->pBindingFlags[binding->binding] &
295 VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) {
296 assert(
297 !binding->pImmutableSamplers); /* Terribly ill defined how many samplers are valid */
298 assert(binding->binding == num_bindings - 1);
299
300 set_layout->has_variable_descriptors = true;
301 }
302
303 if ((binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
304 binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
305 binding->pImmutableSamplers) {
306 set_layout->binding[b].immutable_samplers_offset = samplers_offset;
307 set_layout->binding[b].immutable_samplers_equal =
308 has_equal_immutable_samplers(binding->pImmutableSamplers, binding->descriptorCount);
309 set_layout->has_immutable_samplers = true;
310
311 for (uint32_t i = 0; i < binding->descriptorCount; i++)
312 memcpy(samplers + 4 * i,
313 &radv_sampler_from_handle(binding->pImmutableSamplers[i])->state, 16);
314
315 /* Don't reserve space for the samplers if they're not accessed. */
316 if (set_layout->binding[b].immutable_samplers_equal) {
317 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER &&
318 max_sampled_image_descriptors <= 2)
319 set_layout->binding[b].size -= 32;
320 else if (binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER)
321 set_layout->binding[b].size -= 16;
322 }
323 samplers += 4 * binding->descriptorCount;
324 samplers_offset += 4 * sizeof(uint32_t) * binding->descriptorCount;
325
326 if (has_ycbcr_sampler) {
327 ycbcr_sampler_offsets[b] = (const char *)ycbcr_samplers - (const char *)set_layout;
328 for (uint32_t i = 0; i < binding->descriptorCount; i++) {
329 if (radv_sampler_from_handle(binding->pImmutableSamplers[i])->ycbcr_sampler)
330 ycbcr_samplers[i] =
331 radv_sampler_from_handle(binding->pImmutableSamplers[i])->ycbcr_sampler->state;
332 else
333 ycbcr_samplers[i].format = VK_FORMAT_UNDEFINED;
334 }
335 ycbcr_samplers += binding->descriptorCount;
336 }
337 }
338
339 set_layout->size += descriptor_count * set_layout->binding[b].size;
340 buffer_count += descriptor_count * binding_buffer_count;
341 dynamic_offset_count += descriptor_count * set_layout->binding[b].dynamic_offset_count;
342 set_layout->shader_stages |= binding->stageFlags;
343 }
344
345 free(bindings);
346
347 set_layout->buffer_count = buffer_count;
348 set_layout->dynamic_offset_count = dynamic_offset_count;
349
350 *pSetLayout = radv_descriptor_set_layout_to_handle(set_layout);
351
352 return VK_SUCCESS;
353 }
354
355 VKAPI_ATTR void VKAPI_CALL
radv_GetDescriptorSetLayoutSupport(VkDevice device,const VkDescriptorSetLayoutCreateInfo * pCreateInfo,VkDescriptorSetLayoutSupport * pSupport)356 radv_GetDescriptorSetLayoutSupport(VkDevice device,
357 const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
358 VkDescriptorSetLayoutSupport *pSupport)
359 {
360 VkDescriptorSetLayoutBinding *bindings = NULL;
361 VkResult result =
362 vk_create_sorted_bindings(pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
363 if (result != VK_SUCCESS) {
364 pSupport->supported = false;
365 return;
366 }
367
368 const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags =
369 vk_find_struct_const(pCreateInfo->pNext, DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO);
370 VkDescriptorSetVariableDescriptorCountLayoutSupport *variable_count = vk_find_struct(
371 (void *)pCreateInfo->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_LAYOUT_SUPPORT);
372 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
373 vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
374 if (variable_count) {
375 variable_count->maxVariableDescriptorCount = 0;
376 }
377
378 bool supported = true;
379 uint64_t size = 0;
380 for (uint32_t i = 0; i < pCreateInfo->bindingCount; i++) {
381 const VkDescriptorSetLayoutBinding *binding = bindings + i;
382
383 uint64_t descriptor_size = 0;
384 uint64_t descriptor_alignment = 1;
385 uint32_t descriptor_count = binding->descriptorCount;
386 switch (binding->descriptorType) {
387 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
388 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
389 break;
390 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
391 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
392 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
393 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
394 descriptor_size = 16;
395 descriptor_alignment = 16;
396 break;
397 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
398 descriptor_size = 32;
399 descriptor_alignment = 32;
400 break;
401 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
402 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
403 descriptor_size = 64;
404 descriptor_alignment = 32;
405 break;
406 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
407 if (!has_equal_immutable_samplers(binding->pImmutableSamplers, descriptor_count)) {
408 descriptor_size = 64;
409 } else {
410 descriptor_size = 96;
411 }
412 descriptor_alignment = 32;
413 break;
414 case VK_DESCRIPTOR_TYPE_SAMPLER:
415 if (!has_equal_immutable_samplers(binding->pImmutableSamplers, descriptor_count)) {
416 descriptor_size = 16;
417 descriptor_alignment = 16;
418 }
419 break;
420 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
421 descriptor_alignment = 16;
422 descriptor_size = descriptor_count;
423 descriptor_count = 1;
424 break;
425 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE:
426 if (!radv_mutable_descriptor_type_size_alignment(
427 &mutable_info->pMutableDescriptorTypeLists[i], &descriptor_size,
428 &descriptor_alignment)) {
429 supported = false;
430 }
431 break;
432 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
433 descriptor_size = 16;
434 descriptor_alignment = 16;
435 break;
436 default:
437 break;
438 }
439
440 if (size && !align_u64(size, descriptor_alignment)) {
441 supported = false;
442 }
443 size = align_u64(size, descriptor_alignment);
444
445 uint64_t max_count = INT32_MAX;
446 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK)
447 max_count = INT32_MAX - size;
448 else if (descriptor_size)
449 max_count = (INT32_MAX - size) / descriptor_size;
450
451 if (max_count < descriptor_count) {
452 supported = false;
453 }
454 if (variable_flags && binding->binding < variable_flags->bindingCount && variable_count &&
455 (variable_flags->pBindingFlags[binding->binding] &
456 VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) {
457 variable_count->maxVariableDescriptorCount = MIN2(UINT32_MAX, max_count);
458 }
459 size += descriptor_count * descriptor_size;
460 }
461
462 free(bindings);
463
464 pSupport->supported = supported;
465 }
466
467 /*
468 * Pipeline layouts. These have nothing to do with the pipeline. They are
469 * just multiple descriptor set layouts pasted together.
470 */
471 void
radv_pipeline_layout_init(struct radv_device * device,struct radv_pipeline_layout * layout,bool independent_sets)472 radv_pipeline_layout_init(struct radv_device *device, struct radv_pipeline_layout *layout,
473 bool independent_sets)
474 {
475 memset(layout, 0, sizeof(*layout));
476
477 vk_object_base_init(&device->vk, &layout->base, VK_OBJECT_TYPE_PIPELINE_LAYOUT);
478
479 layout->independent_sets = independent_sets;
480 }
481
482 void
radv_pipeline_layout_add_set(struct radv_pipeline_layout * layout,uint32_t set_idx,struct radv_descriptor_set_layout * set_layout)483 radv_pipeline_layout_add_set(struct radv_pipeline_layout *layout, uint32_t set_idx,
484 struct radv_descriptor_set_layout *set_layout)
485 {
486 unsigned dynamic_offset_count = 0;
487
488 if (layout->set[set_idx].layout)
489 return;
490
491 layout->num_sets = MAX2(set_idx + 1, layout->num_sets);
492
493 layout->set[set_idx].layout = set_layout;
494 vk_descriptor_set_layout_ref(&set_layout->vk);
495
496 for (uint32_t b = 0; b < set_layout->binding_count; b++) {
497 dynamic_offset_count += set_layout->binding[b].array_size * set_layout->binding[b].dynamic_offset_count;
498 }
499
500 layout->set[set_idx].dynamic_offset_start = layout->dynamic_offset_count;
501
502 layout->dynamic_offset_count += dynamic_offset_count;
503 layout->dynamic_shader_stages |= set_layout->dynamic_shader_stages;
504 }
505
506 void
radv_pipeline_layout_hash(struct radv_pipeline_layout * layout)507 radv_pipeline_layout_hash(struct radv_pipeline_layout *layout)
508 {
509 struct mesa_sha1 ctx;
510
511 _mesa_sha1_init(&ctx);
512 for (uint32_t i = 0; i < layout->num_sets; i++) {
513 struct radv_descriptor_set_layout *set_layout = layout->set[i].layout;
514
515 if (!set_layout)
516 continue;
517
518 /* Hash the entire set layout except for the vk_object_base and the reference counter. The
519 * rest of the set layout is carefully constructed to not have pointers so a full hash instead
520 * of a per-field hash should be ok.
521 */
522 uint32_t hash_offset = sizeof(struct vk_object_base) + sizeof(uint32_t);
523 _mesa_sha1_update(&ctx, (const char *)set_layout + hash_offset,
524 set_layout->layout_size - hash_offset);
525 }
526 _mesa_sha1_update(&ctx, &layout->push_constant_size, sizeof(layout->push_constant_size));
527 _mesa_sha1_final(&ctx, layout->sha1);
528 }
529
530 void
radv_pipeline_layout_finish(struct radv_device * device,struct radv_pipeline_layout * layout)531 radv_pipeline_layout_finish(struct radv_device *device, struct radv_pipeline_layout *layout)
532 {
533 for (uint32_t i = 0; i < layout->num_sets; i++) {
534 if (!layout->set[i].layout)
535 continue;
536
537 vk_descriptor_set_layout_unref(&device->vk, &layout->set[i].layout->vk);
538 }
539
540 vk_object_base_finish(&layout->base);
541 }
542
543 VKAPI_ATTR VkResult VKAPI_CALL
radv_CreatePipelineLayout(VkDevice _device,const VkPipelineLayoutCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkPipelineLayout * pPipelineLayout)544 radv_CreatePipelineLayout(VkDevice _device, const VkPipelineLayoutCreateInfo *pCreateInfo,
545 const VkAllocationCallbacks *pAllocator,
546 VkPipelineLayout *pPipelineLayout)
547 {
548 RADV_FROM_HANDLE(radv_device, device, _device);
549 struct radv_pipeline_layout *layout;
550
551 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO);
552
553 layout = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*layout), 8,
554 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
555 if (layout == NULL)
556 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
557
558 radv_pipeline_layout_init(device, layout,
559 pCreateInfo->flags & VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT);
560
561 layout->num_sets = pCreateInfo->setLayoutCount;
562
563 for (uint32_t set = 0; set < pCreateInfo->setLayoutCount; set++) {
564 RADV_FROM_HANDLE(radv_descriptor_set_layout, set_layout, pCreateInfo->pSetLayouts[set]);
565
566 /* From the Vulkan spec 1.3.211:
567 *
568 * "VUID-VkPipelineLayoutCreateInfo-flags-06562
569 * If flags: does not include VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, elements of
570 * pSetLayouts must be valid VkDescriptorSetLayout objects"
571 */
572 if (set_layout == NULL) {
573 assert(layout->independent_sets);
574 continue;
575 }
576
577 radv_pipeline_layout_add_set(layout, set, set_layout);
578 }
579
580 layout->push_constant_size = 0;
581
582 for (unsigned i = 0; i < pCreateInfo->pushConstantRangeCount; ++i) {
583 const VkPushConstantRange *range = pCreateInfo->pPushConstantRanges + i;
584 layout->push_constant_size = MAX2(layout->push_constant_size, range->offset + range->size);
585 }
586
587 layout->push_constant_size = align(layout->push_constant_size, 16);
588
589 radv_pipeline_layout_hash(layout);
590
591 *pPipelineLayout = radv_pipeline_layout_to_handle(layout);
592
593 return VK_SUCCESS;
594 }
595
596 VKAPI_ATTR void VKAPI_CALL
radv_DestroyPipelineLayout(VkDevice _device,VkPipelineLayout _pipelineLayout,const VkAllocationCallbacks * pAllocator)597 radv_DestroyPipelineLayout(VkDevice _device, VkPipelineLayout _pipelineLayout,
598 const VkAllocationCallbacks *pAllocator)
599 {
600 RADV_FROM_HANDLE(radv_device, device, _device);
601 RADV_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, _pipelineLayout);
602
603 if (!pipeline_layout)
604 return;
605
606 radv_pipeline_layout_finish(device, pipeline_layout);
607
608 vk_free2(&device->vk.alloc, pAllocator, pipeline_layout);
609 }
610
611 static VkResult
radv_descriptor_set_create(struct radv_device * device,struct radv_descriptor_pool * pool,struct radv_descriptor_set_layout * layout,const uint32_t * variable_count,struct radv_descriptor_set ** out_set)612 radv_descriptor_set_create(struct radv_device *device, struct radv_descriptor_pool *pool,
613 struct radv_descriptor_set_layout *layout, const uint32_t *variable_count,
614 struct radv_descriptor_set **out_set)
615 {
616 if (pool->entry_count == pool->max_entry_count)
617 return VK_ERROR_OUT_OF_POOL_MEMORY;
618
619 struct radv_descriptor_set *set;
620 uint32_t buffer_count = layout->buffer_count;
621 if (variable_count) {
622 unsigned stride =
623 radv_descriptor_type_buffer_count(layout->binding[layout->binding_count - 1].type);
624 buffer_count =
625 layout->binding[layout->binding_count - 1].buffer_offset + *variable_count * stride;
626 }
627 unsigned range_offset =
628 sizeof(struct radv_descriptor_set_header) + sizeof(struct radeon_winsys_bo *) * buffer_count;
629 const unsigned dynamic_offset_count = layout->dynamic_offset_count;
630 unsigned mem_size =
631 range_offset + sizeof(struct radv_descriptor_range) * dynamic_offset_count;
632
633 if (pool->host_memory_base) {
634 if (pool->host_memory_end - pool->host_memory_ptr < mem_size)
635 return VK_ERROR_OUT_OF_POOL_MEMORY;
636
637 set = (struct radv_descriptor_set *)pool->host_memory_ptr;
638 pool->host_memory_ptr += mem_size;
639 } else {
640 set = vk_alloc2(&device->vk.alloc, NULL, mem_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
641
642 if (!set)
643 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
644 }
645
646 memset(set, 0, mem_size);
647
648 vk_object_base_init(&device->vk, &set->header.base, VK_OBJECT_TYPE_DESCRIPTOR_SET);
649
650 if (dynamic_offset_count) {
651 set->header.dynamic_descriptors =
652 (struct radv_descriptor_range *)((uint8_t *)set + range_offset);
653 }
654
655 set->header.layout = layout;
656 set->header.buffer_count = buffer_count;
657 uint32_t layout_size = layout->size;
658 if (variable_count) {
659 uint32_t stride = layout->binding[layout->binding_count - 1].size;
660 if (layout->binding[layout->binding_count - 1].type ==
661 VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK)
662 stride = 1;
663
664 layout_size = layout->binding[layout->binding_count - 1].offset + *variable_count * stride;
665 }
666 layout_size = align_u32(layout_size, 32);
667 set->header.size = layout_size;
668
669 /* try to allocate linearly first, so that we don't spend
670 * time looking for gaps if the app only allocates &
671 * resets via the pool. */
672 if (pool->current_offset + layout_size <= pool->size) {
673 set->header.bo = pool->bo;
674 set->header.mapped_ptr = (uint32_t *)(pool->mapped_ptr + pool->current_offset);
675 set->header.va = pool->bo ? (radv_buffer_get_va(set->header.bo) + pool->current_offset) : 0;
676 if (!pool->host_memory_base) {
677 pool->entries[pool->entry_count].offset = pool->current_offset;
678 pool->entries[pool->entry_count].size = layout_size;
679 }
680 pool->entries[pool->entry_count].set = set;
681 pool->current_offset += layout_size;
682 } else if (!pool->host_memory_base) {
683 uint64_t offset = 0;
684 int index;
685
686 for (index = 0; index < pool->entry_count; ++index) {
687 if (pool->entries[index].offset - offset >= layout_size)
688 break;
689 offset = pool->entries[index].offset + pool->entries[index].size;
690 }
691
692 if (pool->size - offset < layout_size) {
693 vk_free2(&device->vk.alloc, NULL, set);
694 return VK_ERROR_OUT_OF_POOL_MEMORY;
695 }
696 set->header.bo = pool->bo;
697 set->header.mapped_ptr = (uint32_t *)(pool->mapped_ptr + offset);
698 set->header.va = pool->bo ? (radv_buffer_get_va(set->header.bo) + offset) : 0;
699 memmove(&pool->entries[index + 1], &pool->entries[index],
700 sizeof(pool->entries[0]) * (pool->entry_count - index));
701 pool->entries[index].offset = offset;
702 pool->entries[index].size = layout_size;
703 pool->entries[index].set = set;
704 } else
705 return VK_ERROR_OUT_OF_POOL_MEMORY;
706
707 if (layout->has_immutable_samplers) {
708 for (unsigned i = 0; i < layout->binding_count; ++i) {
709 if (!layout->binding[i].immutable_samplers_offset ||
710 layout->binding[i].immutable_samplers_equal)
711 continue;
712
713 unsigned offset = layout->binding[i].offset / 4;
714 if (layout->binding[i].type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
715 offset += radv_combined_image_descriptor_sampler_offset(layout->binding + i) / 4;
716
717 const uint32_t *samplers =
718 (const uint32_t *)((const char *)layout + layout->binding[i].immutable_samplers_offset);
719 for (unsigned j = 0; j < layout->binding[i].array_size; ++j) {
720 memcpy(set->header.mapped_ptr + offset, samplers + 4 * j, 16);
721 offset += layout->binding[i].size / 4;
722 }
723 }
724 }
725
726 pool->entry_count++;
727 vk_descriptor_set_layout_ref(&layout->vk);
728 *out_set = set;
729 return VK_SUCCESS;
730 }
731
732 static void
radv_descriptor_set_destroy(struct radv_device * device,struct radv_descriptor_pool * pool,struct radv_descriptor_set * set,bool free_bo)733 radv_descriptor_set_destroy(struct radv_device *device, struct radv_descriptor_pool *pool,
734 struct radv_descriptor_set *set, bool free_bo)
735 {
736 vk_descriptor_set_layout_unref(&device->vk, &set->header.layout->vk);
737
738 if (pool->host_memory_base)
739 return;
740
741 if (free_bo && !pool->host_memory_base) {
742 for (int i = 0; i < pool->entry_count; ++i) {
743 if (pool->entries[i].set == set) {
744 memmove(&pool->entries[i], &pool->entries[i + 1],
745 sizeof(pool->entries[i]) * (pool->entry_count - i - 1));
746 --pool->entry_count;
747 break;
748 }
749 }
750 }
751 vk_object_base_finish(&set->header.base);
752 vk_free2(&device->vk.alloc, NULL, set);
753 }
754
755 static void
radv_destroy_descriptor_pool(struct radv_device * device,const VkAllocationCallbacks * pAllocator,struct radv_descriptor_pool * pool)756 radv_destroy_descriptor_pool(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
757 struct radv_descriptor_pool *pool)
758 {
759 for (int i = 0; i < pool->entry_count; ++i) {
760 radv_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
761 }
762
763 if (pool->bo)
764 device->ws->buffer_destroy(device->ws, pool->bo);
765 if (pool->host_bo)
766 vk_free2(&device->vk.alloc, pAllocator, pool->host_bo);
767
768 vk_object_base_finish(&pool->base);
769 vk_free2(&device->vk.alloc, pAllocator, pool);
770 }
771
772 VKAPI_ATTR VkResult VKAPI_CALL
radv_CreateDescriptorPool(VkDevice _device,const VkDescriptorPoolCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorPool * pDescriptorPool)773 radv_CreateDescriptorPool(VkDevice _device, const VkDescriptorPoolCreateInfo *pCreateInfo,
774 const VkAllocationCallbacks *pAllocator,
775 VkDescriptorPool *pDescriptorPool)
776 {
777 RADV_FROM_HANDLE(radv_device, device, _device);
778 struct radv_descriptor_pool *pool;
779 uint64_t size = sizeof(struct radv_descriptor_pool);
780 uint64_t bo_size = 0, bo_count = 0, range_count = 0;
781
782 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
783 vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
784
785 vk_foreach_struct_const(ext, pCreateInfo->pNext)
786 {
787 switch (ext->sType) {
788 case VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_INLINE_UNIFORM_BLOCK_CREATE_INFO: {
789 const VkDescriptorPoolInlineUniformBlockCreateInfo *info =
790 (const VkDescriptorPoolInlineUniformBlockCreateInfo *)ext;
791 /* the sizes are 4 aligned, and we need to align to at
792 * most 32, which needs at most 28 bytes extra per
793 * binding. */
794 bo_size += 28llu * info->maxInlineUniformBlockBindings;
795 break;
796 }
797 default:
798 break;
799 }
800 }
801
802 for (unsigned i = 0; i < pCreateInfo->poolSizeCount; ++i) {
803 bo_count += radv_descriptor_type_buffer_count(pCreateInfo->pPoolSizes[i].type) *
804 pCreateInfo->pPoolSizes[i].descriptorCount;
805
806 switch (pCreateInfo->pPoolSizes[i].type) {
807 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
808 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
809 range_count += pCreateInfo->pPoolSizes[i].descriptorCount;
810 break;
811 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
812 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
813 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
814 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
815 case VK_DESCRIPTOR_TYPE_SAMPLER:
816 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
817 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
818 /* 32 as we may need to align for images */
819 bo_size += 32 * pCreateInfo->pPoolSizes[i].descriptorCount;
820 break;
821 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
822 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
823 bo_size += 64 * pCreateInfo->pPoolSizes[i].descriptorCount;
824 break;
825 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE:
826 /* Per spec, if a mutable descriptor type list is provided for the pool entry, we
827 * allocate enough memory to hold any subset of that list.
828 * If there is no mutable descriptor type list available,
829 * we must allocate enough for any supported mutable descriptor type, i.e. 64 bytes. */
830 if (mutable_info && i < mutable_info->mutableDescriptorTypeListCount) {
831 uint64_t mutable_size, mutable_alignment;
832 if (radv_mutable_descriptor_type_size_alignment(
833 &mutable_info->pMutableDescriptorTypeLists[i], &mutable_size,
834 &mutable_alignment)) {
835 /* 32 as we may need to align for images */
836 mutable_size = align(mutable_size, 32);
837 bo_size += mutable_size * pCreateInfo->pPoolSizes[i].descriptorCount;
838 }
839 } else {
840 bo_size += 64 * pCreateInfo->pPoolSizes[i].descriptorCount;
841 }
842 break;
843 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
844 bo_size += 96 * pCreateInfo->pPoolSizes[i].descriptorCount;
845 break;
846 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
847 bo_size += pCreateInfo->pPoolSizes[i].descriptorCount;
848 break;
849 default:
850 break;
851 }
852 }
853
854 uint64_t entries_size = sizeof(struct radv_descriptor_pool_entry) * pCreateInfo->maxSets;
855 size += entries_size;
856
857 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
858 uint64_t host_size = pCreateInfo->maxSets * sizeof(struct radv_descriptor_set);
859 host_size += sizeof(struct radeon_winsys_bo *) * bo_count;
860 host_size += sizeof(struct radv_descriptor_range) * range_count;
861 size += host_size;
862 }
863
864 pool = vk_alloc2(&device->vk.alloc, pAllocator, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
865 if (!pool)
866 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
867
868 memset(pool, 0, sizeof(*pool));
869
870 vk_object_base_init(&device->vk, &pool->base, VK_OBJECT_TYPE_DESCRIPTOR_POOL);
871
872 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
873 pool->host_memory_base = (uint8_t *)pool + sizeof(struct radv_descriptor_pool) + entries_size;
874 pool->host_memory_ptr = pool->host_memory_base;
875 pool->host_memory_end = (uint8_t *)pool + size;
876 }
877
878 if (bo_size) {
879 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_HOST_ONLY_BIT_VALVE)) {
880 enum radeon_bo_flag flags = RADEON_FLAG_NO_INTERPROCESS_SHARING | RADEON_FLAG_READ_ONLY |
881 RADEON_FLAG_32BIT;
882
883 if (device->instance->zero_vram)
884 flags |= RADEON_FLAG_ZERO_VRAM;
885
886 VkResult result = device->ws->buffer_create(
887 device->ws, bo_size, 32, RADEON_DOMAIN_VRAM, flags, RADV_BO_PRIORITY_DESCRIPTOR, 0,
888 &pool->bo);
889 if (result != VK_SUCCESS) {
890 radv_destroy_descriptor_pool(device, pAllocator, pool);
891 return vk_error(device, result);
892 }
893 pool->mapped_ptr = (uint8_t *)device->ws->buffer_map(pool->bo);
894 if (!pool->mapped_ptr) {
895 radv_destroy_descriptor_pool(device, pAllocator, pool);
896 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
897 }
898 } else {
899 pool->host_bo =
900 vk_alloc2(&device->vk.alloc, pAllocator, bo_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
901 if (!pool->host_bo) {
902 radv_destroy_descriptor_pool(device, pAllocator, pool);
903 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
904 }
905 pool->mapped_ptr = pool->host_bo;
906 }
907 }
908 pool->size = bo_size;
909 pool->max_entry_count = pCreateInfo->maxSets;
910
911 *pDescriptorPool = radv_descriptor_pool_to_handle(pool);
912 return VK_SUCCESS;
913 }
914
915 VKAPI_ATTR void VKAPI_CALL
radv_DestroyDescriptorPool(VkDevice _device,VkDescriptorPool _pool,const VkAllocationCallbacks * pAllocator)916 radv_DestroyDescriptorPool(VkDevice _device, VkDescriptorPool _pool,
917 const VkAllocationCallbacks *pAllocator)
918 {
919 RADV_FROM_HANDLE(radv_device, device, _device);
920 RADV_FROM_HANDLE(radv_descriptor_pool, pool, _pool);
921
922 if (!pool)
923 return;
924
925 radv_destroy_descriptor_pool(device, pAllocator, pool);
926 }
927
928 VKAPI_ATTR VkResult VKAPI_CALL
radv_ResetDescriptorPool(VkDevice _device,VkDescriptorPool descriptorPool,VkDescriptorPoolResetFlags flags)929 radv_ResetDescriptorPool(VkDevice _device, VkDescriptorPool descriptorPool,
930 VkDescriptorPoolResetFlags flags)
931 {
932 RADV_FROM_HANDLE(radv_device, device, _device);
933 RADV_FROM_HANDLE(radv_descriptor_pool, pool, descriptorPool);
934
935 for (int i = 0; i < pool->entry_count; ++i) {
936 radv_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
937 }
938 pool->entry_count = 0;
939
940 pool->current_offset = 0;
941 pool->host_memory_ptr = pool->host_memory_base;
942
943 return VK_SUCCESS;
944 }
945
946 VKAPI_ATTR VkResult VKAPI_CALL
radv_AllocateDescriptorSets(VkDevice _device,const VkDescriptorSetAllocateInfo * pAllocateInfo,VkDescriptorSet * pDescriptorSets)947 radv_AllocateDescriptorSets(VkDevice _device, const VkDescriptorSetAllocateInfo *pAllocateInfo,
948 VkDescriptorSet *pDescriptorSets)
949 {
950 RADV_FROM_HANDLE(radv_device, device, _device);
951 RADV_FROM_HANDLE(radv_descriptor_pool, pool, pAllocateInfo->descriptorPool);
952
953 VkResult result = VK_SUCCESS;
954 uint32_t i;
955 struct radv_descriptor_set *set = NULL;
956
957 const VkDescriptorSetVariableDescriptorCountAllocateInfo *variable_counts = vk_find_struct_const(
958 pAllocateInfo->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_ALLOCATE_INFO);
959 const uint32_t zero = 0;
960
961 /* allocate a set of buffers for each shader to contain descriptors */
962 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
963 RADV_FROM_HANDLE(radv_descriptor_set_layout, layout, pAllocateInfo->pSetLayouts[i]);
964
965 const uint32_t *variable_count = NULL;
966 if (layout->has_variable_descriptors && variable_counts) {
967 if (i < variable_counts->descriptorSetCount)
968 variable_count = variable_counts->pDescriptorCounts + i;
969 else
970 variable_count = &zero;
971 }
972
973 assert(!(layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
974
975 result = radv_descriptor_set_create(device, pool, layout, variable_count, &set);
976 if (result != VK_SUCCESS)
977 break;
978
979 pDescriptorSets[i] = radv_descriptor_set_to_handle(set);
980 }
981
982 if (result != VK_SUCCESS) {
983 radv_FreeDescriptorSets(_device, pAllocateInfo->descriptorPool, i, pDescriptorSets);
984 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
985 pDescriptorSets[i] = VK_NULL_HANDLE;
986 }
987 }
988 return result;
989 }
990
991 VKAPI_ATTR VkResult VKAPI_CALL
radv_FreeDescriptorSets(VkDevice _device,VkDescriptorPool descriptorPool,uint32_t count,const VkDescriptorSet * pDescriptorSets)992 radv_FreeDescriptorSets(VkDevice _device, VkDescriptorPool descriptorPool, uint32_t count,
993 const VkDescriptorSet *pDescriptorSets)
994 {
995 RADV_FROM_HANDLE(radv_device, device, _device);
996 RADV_FROM_HANDLE(radv_descriptor_pool, pool, descriptorPool);
997
998 for (uint32_t i = 0; i < count; i++) {
999 RADV_FROM_HANDLE(radv_descriptor_set, set, pDescriptorSets[i]);
1000
1001 if (set && !pool->host_memory_base)
1002 radv_descriptor_set_destroy(device, pool, set, true);
1003 }
1004 return VK_SUCCESS;
1005 }
1006
1007 static ALWAYS_INLINE void
write_texel_buffer_descriptor(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,unsigned * dst,struct radeon_winsys_bo ** buffer_list,const VkBufferView _buffer_view)1008 write_texel_buffer_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1009 unsigned *dst, struct radeon_winsys_bo **buffer_list,
1010 const VkBufferView _buffer_view)
1011 {
1012 RADV_FROM_HANDLE(radv_buffer_view, buffer_view, _buffer_view);
1013
1014 if (!buffer_view) {
1015 memset(dst, 0, 4 * 4);
1016 if (!cmd_buffer)
1017 *buffer_list = NULL;
1018 return;
1019 }
1020
1021 memcpy(dst, buffer_view->state, 4 * 4);
1022
1023 if (cmd_buffer)
1024 radv_cs_add_buffer(device->ws, cmd_buffer->cs, buffer_view->bo);
1025 else
1026 *buffer_list = buffer_view->bo;
1027 }
1028
1029 static ALWAYS_INLINE void
write_buffer_descriptor(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,unsigned * dst,struct radeon_winsys_bo ** buffer_list,const VkDescriptorBufferInfo * buffer_info)1030 write_buffer_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1031 unsigned *dst, struct radeon_winsys_bo **buffer_list,
1032 const VkDescriptorBufferInfo *buffer_info)
1033 {
1034 RADV_FROM_HANDLE(radv_buffer, buffer, buffer_info->buffer);
1035
1036 if (!buffer) {
1037 memset(dst, 0, 4 * 4);
1038 if (!cmd_buffer)
1039 *buffer_list = NULL;
1040 return;
1041 }
1042
1043 uint64_t va = radv_buffer_get_va(buffer->bo);
1044
1045 uint32_t range = vk_buffer_range(&buffer->vk, buffer_info->offset, buffer_info->range);
1046 assert(buffer->vk.size > 0 && range > 0);
1047
1048 /* robustBufferAccess is relaxed enough to allow this (in combination
1049 * with the alignment/size we return from vkGetBufferMemoryRequirements)
1050 * and this allows the shader compiler to create more efficient 8/16-bit
1051 * buffer accesses. */
1052 range = align(range, 4);
1053
1054 va += buffer_info->offset + buffer->offset;
1055
1056 uint32_t rsrc_word3 =
1057 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1058 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
1059
1060 if (device->physical_device->rad_info.gfx_level >= GFX11) {
1061 rsrc_word3 |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_FLOAT) |
1062 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW);
1063 } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
1064 rsrc_word3 |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_FLOAT) |
1065 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) | S_008F0C_RESOURCE_LEVEL(1);
1066 } else {
1067 rsrc_word3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1068 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
1069 }
1070
1071 dst[0] = va;
1072 dst[1] = S_008F04_BASE_ADDRESS_HI(va >> 32);
1073 dst[2] = range;
1074 dst[3] = rsrc_word3;
1075
1076 if (cmd_buffer)
1077 radv_cs_add_buffer(device->ws, cmd_buffer->cs, buffer->bo);
1078 else
1079 *buffer_list = buffer->bo;
1080 }
1081
1082 static ALWAYS_INLINE void
write_block_descriptor(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,void * dst,const VkWriteDescriptorSet * writeset)1083 write_block_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, void *dst,
1084 const VkWriteDescriptorSet *writeset)
1085 {
1086 const VkWriteDescriptorSetInlineUniformBlock *inline_ub =
1087 vk_find_struct_const(writeset->pNext, WRITE_DESCRIPTOR_SET_INLINE_UNIFORM_BLOCK);
1088
1089 memcpy(dst, inline_ub->pData, inline_ub->dataSize);
1090 }
1091
1092 static ALWAYS_INLINE void
write_dynamic_buffer_descriptor(struct radv_device * device,struct radv_descriptor_range * range,struct radeon_winsys_bo ** buffer_list,const VkDescriptorBufferInfo * buffer_info)1093 write_dynamic_buffer_descriptor(struct radv_device *device, struct radv_descriptor_range *range,
1094 struct radeon_winsys_bo **buffer_list,
1095 const VkDescriptorBufferInfo *buffer_info)
1096 {
1097 RADV_FROM_HANDLE(radv_buffer, buffer, buffer_info->buffer);
1098 uint64_t va;
1099 unsigned size;
1100
1101 if (!buffer) {
1102 range->va = 0;
1103 *buffer_list = NULL;
1104 return;
1105 }
1106
1107 va = radv_buffer_get_va(buffer->bo);
1108
1109 size = vk_buffer_range(&buffer->vk, buffer_info->offset, buffer_info->range);
1110 assert(buffer->vk.size > 0 && size > 0);
1111
1112 /* robustBufferAccess is relaxed enough to allow this (in combination
1113 * with the alignment/size we return from vkGetBufferMemoryRequirements)
1114 * and this allows the shader compiler to create more efficient 8/16-bit
1115 * buffer accesses. */
1116 size = align(size, 4);
1117
1118 va += buffer_info->offset + buffer->offset;
1119 range->va = va;
1120 range->size = size;
1121
1122 *buffer_list = buffer->bo;
1123 }
1124
1125 static ALWAYS_INLINE void
write_image_descriptor(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,unsigned size,unsigned * dst,struct radeon_winsys_bo ** buffer_list,VkDescriptorType descriptor_type,const VkDescriptorImageInfo * image_info)1126 write_image_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1127 unsigned size, unsigned *dst, struct radeon_winsys_bo **buffer_list,
1128 VkDescriptorType descriptor_type, const VkDescriptorImageInfo *image_info)
1129 {
1130 RADV_FROM_HANDLE(radv_image_view, iview, image_info->imageView);
1131 union radv_descriptor *descriptor;
1132
1133 if (!iview) {
1134 memset(dst, 0, size);
1135 if (!cmd_buffer)
1136 *buffer_list = NULL;
1137 return;
1138 }
1139
1140 if (descriptor_type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) {
1141 descriptor = &iview->storage_descriptor;
1142 } else {
1143 descriptor = &iview->descriptor;
1144 }
1145 assert(size > 0);
1146
1147 memcpy(dst, descriptor, size);
1148
1149 const uint32_t max_bindings = sizeof(iview->image->bindings) /
1150 sizeof(iview->image->bindings[0]);
1151 for (uint32_t b = 0; b < max_bindings; b++) {
1152 if (cmd_buffer) {
1153 if (iview->image->bindings[b].bo)
1154 radv_cs_add_buffer(device->ws, cmd_buffer->cs, iview->image->bindings[b].bo);
1155 } else {
1156 *buffer_list = iview->image->bindings[b].bo;
1157 buffer_list++;
1158 }
1159 }
1160 }
1161
1162 static ALWAYS_INLINE void
write_combined_image_sampler_descriptor(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,unsigned sampler_offset,unsigned * dst,struct radeon_winsys_bo ** buffer_list,VkDescriptorType descriptor_type,const VkDescriptorImageInfo * image_info,bool has_sampler)1163 write_combined_image_sampler_descriptor(struct radv_device *device,
1164 struct radv_cmd_buffer *cmd_buffer, unsigned sampler_offset,
1165 unsigned *dst, struct radeon_winsys_bo **buffer_list,
1166 VkDescriptorType descriptor_type,
1167 const VkDescriptorImageInfo *image_info, bool has_sampler)
1168 {
1169 write_image_descriptor(device, cmd_buffer, sampler_offset, dst, buffer_list, descriptor_type,
1170 image_info);
1171 /* copy over sampler state */
1172 if (has_sampler) {
1173 RADV_FROM_HANDLE(radv_sampler, sampler, image_info->sampler);
1174 memcpy(dst + sampler_offset / sizeof(*dst), sampler->state, 16);
1175 }
1176 }
1177
1178 static ALWAYS_INLINE void
write_sampler_descriptor(struct radv_device * device,unsigned * dst,const VkDescriptorImageInfo * image_info)1179 write_sampler_descriptor(struct radv_device *device, unsigned *dst,
1180 const VkDescriptorImageInfo *image_info)
1181 {
1182 RADV_FROM_HANDLE(radv_sampler, sampler, image_info->sampler);
1183
1184 memcpy(dst, sampler->state, 16);
1185 }
1186
1187 static ALWAYS_INLINE void
write_accel_struct(void * ptr,VkAccelerationStructureKHR _accel_struct)1188 write_accel_struct(void *ptr, VkAccelerationStructureKHR _accel_struct)
1189 {
1190 RADV_FROM_HANDLE(radv_acceleration_structure, accel_struct, _accel_struct);
1191 uint64_t va = accel_struct ? radv_accel_struct_get_va(accel_struct) : 0;
1192 memcpy(ptr, &va, sizeof(va));
1193 }
1194
1195 static ALWAYS_INLINE void
radv_update_descriptor_sets_impl(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,VkDescriptorSet dstSetOverride,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)1196 radv_update_descriptor_sets_impl(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1197 VkDescriptorSet dstSetOverride, uint32_t descriptorWriteCount,
1198 const VkWriteDescriptorSet *pDescriptorWrites,
1199 uint32_t descriptorCopyCount,
1200 const VkCopyDescriptorSet *pDescriptorCopies)
1201 {
1202 uint32_t i, j;
1203 for (i = 0; i < descriptorWriteCount; i++) {
1204 const VkWriteDescriptorSet *writeset = &pDescriptorWrites[i];
1205 RADV_FROM_HANDLE(radv_descriptor_set, set,
1206 dstSetOverride ? dstSetOverride : writeset->dstSet);
1207 const struct radv_descriptor_set_binding_layout *binding_layout =
1208 set->header.layout->binding + writeset->dstBinding;
1209 uint32_t *ptr = set->header.mapped_ptr;
1210 struct radeon_winsys_bo **buffer_list = set->descriptors;
1211 /* Immutable samplers are not copied into push descriptors when they are
1212 * allocated, so if we are writing push descriptors we have to copy the
1213 * immutable samplers into them now.
1214 */
1215 const bool copy_immutable_samplers = cmd_buffer &&
1216 binding_layout->immutable_samplers_offset &&
1217 !binding_layout->immutable_samplers_equal;
1218 const uint32_t *samplers = radv_immutable_samplers(set->header.layout, binding_layout);
1219 const VkWriteDescriptorSetAccelerationStructureKHR *accel_structs = NULL;
1220
1221 ptr += binding_layout->offset / 4;
1222
1223 if (writeset->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) {
1224 write_block_descriptor(device, cmd_buffer, (uint8_t *)ptr + writeset->dstArrayElement,
1225 writeset);
1226 continue;
1227 } else if (writeset->descriptorType == VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR) {
1228 accel_structs =
1229 vk_find_struct_const(writeset->pNext, WRITE_DESCRIPTOR_SET_ACCELERATION_STRUCTURE_KHR);
1230 }
1231
1232 ptr += binding_layout->size * writeset->dstArrayElement / 4;
1233 buffer_list += binding_layout->buffer_offset;
1234 buffer_list += writeset->dstArrayElement;
1235 for (j = 0; j < writeset->descriptorCount; ++j) {
1236 switch (writeset->descriptorType) {
1237 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1238 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1239 unsigned idx = writeset->dstArrayElement + j;
1240 idx += binding_layout->dynamic_offset_offset;
1241 assert(!(set->header.layout->flags &
1242 VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1243 write_dynamic_buffer_descriptor(device, set->header.dynamic_descriptors + idx,
1244 buffer_list, writeset->pBufferInfo + j);
1245 break;
1246 }
1247 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1248 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1249 write_buffer_descriptor(device, cmd_buffer, ptr, buffer_list,
1250 writeset->pBufferInfo + j);
1251 break;
1252 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1253 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1254 write_texel_buffer_descriptor(device, cmd_buffer, ptr, buffer_list,
1255 writeset->pTexelBufferView[j]);
1256 break;
1257 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
1258 write_image_descriptor(device, cmd_buffer, 32, ptr, buffer_list,
1259 writeset->descriptorType, writeset->pImageInfo + j);
1260 break;
1261 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1262 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1263 write_image_descriptor(device, cmd_buffer, 64, ptr, buffer_list,
1264 writeset->descriptorType, writeset->pImageInfo + j);
1265 break;
1266 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: {
1267 unsigned sampler_offset = radv_combined_image_descriptor_sampler_offset(binding_layout);
1268 write_combined_image_sampler_descriptor(
1269 device, cmd_buffer, sampler_offset, ptr, buffer_list, writeset->descriptorType,
1270 writeset->pImageInfo + j, !binding_layout->immutable_samplers_offset);
1271 if (copy_immutable_samplers) {
1272 const unsigned idx = writeset->dstArrayElement + j;
1273 memcpy((char *)ptr + sampler_offset, samplers + 4 * idx, 16);
1274 }
1275 break;
1276 }
1277 case VK_DESCRIPTOR_TYPE_SAMPLER:
1278 if (!binding_layout->immutable_samplers_offset) {
1279 write_sampler_descriptor(device, ptr, writeset->pImageInfo + j);
1280 } else if (copy_immutable_samplers) {
1281 unsigned idx = writeset->dstArrayElement + j;
1282 memcpy(ptr, samplers + 4 * idx, 16);
1283 }
1284 break;
1285 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
1286 write_accel_struct(ptr, accel_structs->pAccelerationStructures[j]);
1287 break;
1288 default:
1289 break;
1290 }
1291 ptr += binding_layout->size / 4;
1292 ++buffer_list;
1293 }
1294 }
1295
1296 for (i = 0; i < descriptorCopyCount; i++) {
1297 const VkCopyDescriptorSet *copyset = &pDescriptorCopies[i];
1298 RADV_FROM_HANDLE(radv_descriptor_set, src_set, copyset->srcSet);
1299 RADV_FROM_HANDLE(radv_descriptor_set, dst_set, copyset->dstSet);
1300 const struct radv_descriptor_set_binding_layout *src_binding_layout =
1301 src_set->header.layout->binding + copyset->srcBinding;
1302 const struct radv_descriptor_set_binding_layout *dst_binding_layout =
1303 dst_set->header.layout->binding + copyset->dstBinding;
1304 uint32_t *src_ptr = src_set->header.mapped_ptr;
1305 uint32_t *dst_ptr = dst_set->header.mapped_ptr;
1306 struct radeon_winsys_bo **src_buffer_list = src_set->descriptors;
1307 struct radeon_winsys_bo **dst_buffer_list = dst_set->descriptors;
1308
1309 src_ptr += src_binding_layout->offset / 4;
1310 dst_ptr += dst_binding_layout->offset / 4;
1311
1312 if (src_binding_layout->type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) {
1313 src_ptr += copyset->srcArrayElement / 4;
1314 dst_ptr += copyset->dstArrayElement / 4;
1315
1316 memcpy(dst_ptr, src_ptr, copyset->descriptorCount);
1317 continue;
1318 }
1319
1320 src_ptr += src_binding_layout->size * copyset->srcArrayElement / 4;
1321 dst_ptr += dst_binding_layout->size * copyset->dstArrayElement / 4;
1322
1323 src_buffer_list += src_binding_layout->buffer_offset;
1324 src_buffer_list += copyset->srcArrayElement;
1325
1326 dst_buffer_list += dst_binding_layout->buffer_offset;
1327 dst_buffer_list += copyset->dstArrayElement;
1328
1329 /* In case of copies between mutable descriptor types
1330 * and non-mutable descriptor types. */
1331 size_t copy_size = MIN2(src_binding_layout->size, dst_binding_layout->size);
1332
1333 for (j = 0; j < copyset->descriptorCount; ++j) {
1334 switch (src_binding_layout->type) {
1335 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1336 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1337 unsigned src_idx = copyset->srcArrayElement + j;
1338 unsigned dst_idx = copyset->dstArrayElement + j;
1339 struct radv_descriptor_range *src_range, *dst_range;
1340 src_idx += src_binding_layout->dynamic_offset_offset;
1341 dst_idx += dst_binding_layout->dynamic_offset_offset;
1342
1343 src_range = src_set->header.dynamic_descriptors + src_idx;
1344 dst_range = dst_set->header.dynamic_descriptors + dst_idx;
1345 *dst_range = *src_range;
1346 break;
1347 }
1348 default:
1349 memcpy(dst_ptr, src_ptr, copy_size);
1350 }
1351 src_ptr += src_binding_layout->size / 4;
1352 dst_ptr += dst_binding_layout->size / 4;
1353
1354 unsigned src_buffer_count = radv_descriptor_type_buffer_count(src_binding_layout->type);
1355 unsigned dst_buffer_count = radv_descriptor_type_buffer_count(dst_binding_layout->type);
1356 for (unsigned k = 0; k < dst_buffer_count; k++) {
1357 if (k < src_buffer_count)
1358 dst_buffer_list[k] = src_buffer_list[k];
1359 else
1360 dst_buffer_list[k] = NULL;
1361 }
1362
1363 dst_buffer_list += dst_buffer_count;
1364 src_buffer_list += src_buffer_count;
1365 }
1366 }
1367 }
1368
1369 VKAPI_ATTR void VKAPI_CALL
radv_UpdateDescriptorSets(VkDevice _device,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)1370 radv_UpdateDescriptorSets(VkDevice _device, uint32_t descriptorWriteCount,
1371 const VkWriteDescriptorSet *pDescriptorWrites,
1372 uint32_t descriptorCopyCount,
1373 const VkCopyDescriptorSet *pDescriptorCopies)
1374 {
1375 RADV_FROM_HANDLE(radv_device, device, _device);
1376
1377 radv_update_descriptor_sets_impl(device, NULL, VK_NULL_HANDLE, descriptorWriteCount,
1378 pDescriptorWrites, descriptorCopyCount, pDescriptorCopies);
1379 }
1380
1381 void
radv_cmd_update_descriptor_sets(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,VkDescriptorSet dstSetOverride,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)1382 radv_cmd_update_descriptor_sets(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1383 VkDescriptorSet dstSetOverride, uint32_t descriptorWriteCount,
1384 const VkWriteDescriptorSet *pDescriptorWrites,
1385 uint32_t descriptorCopyCount,
1386 const VkCopyDescriptorSet *pDescriptorCopies)
1387 {
1388 /* Assume cmd_buffer != NULL to optimize out cmd_buffer checks in generic code above. */
1389 assume(cmd_buffer != NULL);
1390 radv_update_descriptor_sets_impl(device, cmd_buffer, dstSetOverride, descriptorWriteCount,
1391 pDescriptorWrites, descriptorCopyCount, pDescriptorCopies);
1392 }
1393
1394 VKAPI_ATTR VkResult VKAPI_CALL
radv_CreateDescriptorUpdateTemplate(VkDevice _device,const VkDescriptorUpdateTemplateCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorUpdateTemplate * pDescriptorUpdateTemplate)1395 radv_CreateDescriptorUpdateTemplate(VkDevice _device,
1396 const VkDescriptorUpdateTemplateCreateInfo *pCreateInfo,
1397 const VkAllocationCallbacks *pAllocator,
1398 VkDescriptorUpdateTemplate *pDescriptorUpdateTemplate)
1399 {
1400 RADV_FROM_HANDLE(radv_device, device, _device);
1401 const uint32_t entry_count = pCreateInfo->descriptorUpdateEntryCount;
1402 const size_t size = sizeof(struct radv_descriptor_update_template) +
1403 sizeof(struct radv_descriptor_update_template_entry) * entry_count;
1404 struct radv_descriptor_set_layout *set_layout = NULL;
1405 struct radv_descriptor_update_template *templ;
1406 uint32_t i;
1407
1408 templ = vk_alloc2(&device->vk.alloc, pAllocator, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1409 if (!templ)
1410 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1411
1412 vk_object_base_init(&device->vk, &templ->base, VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE);
1413
1414 templ->entry_count = entry_count;
1415
1416 if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR) {
1417 RADV_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, pCreateInfo->pipelineLayout);
1418
1419 /* descriptorSetLayout should be ignored for push descriptors
1420 * and instead it refers to pipelineLayout and set.
1421 */
1422 assert(pCreateInfo->set < MAX_SETS);
1423 set_layout = pipeline_layout->set[pCreateInfo->set].layout;
1424
1425 templ->bind_point = pCreateInfo->pipelineBindPoint;
1426 } else {
1427 assert(pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET);
1428 set_layout = radv_descriptor_set_layout_from_handle(pCreateInfo->descriptorSetLayout);
1429 }
1430
1431 for (i = 0; i < entry_count; i++) {
1432 const VkDescriptorUpdateTemplateEntry *entry = &pCreateInfo->pDescriptorUpdateEntries[i];
1433 const struct radv_descriptor_set_binding_layout *binding_layout =
1434 set_layout->binding + entry->dstBinding;
1435 const uint32_t buffer_offset = binding_layout->buffer_offset + entry->dstArrayElement;
1436 const uint32_t *immutable_samplers = NULL;
1437 uint32_t dst_offset;
1438 uint32_t dst_stride;
1439
1440 /* dst_offset is an offset into dynamic_descriptors when the descriptor
1441 is dynamic, and an offset into mapped_ptr otherwise */
1442 switch (entry->descriptorType) {
1443 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1444 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
1445 assert(pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET);
1446 dst_offset = binding_layout->dynamic_offset_offset + entry->dstArrayElement;
1447 dst_stride = 0; /* Not used */
1448 break;
1449 default:
1450 switch (entry->descriptorType) {
1451 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1452 case VK_DESCRIPTOR_TYPE_SAMPLER:
1453 /* Immutable samplers are copied into push descriptors when they are pushed */
1454 if (pCreateInfo->templateType ==
1455 VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR &&
1456 binding_layout->immutable_samplers_offset &&
1457 !binding_layout->immutable_samplers_equal) {
1458 immutable_samplers =
1459 radv_immutable_samplers(set_layout, binding_layout) + entry->dstArrayElement * 4;
1460 }
1461 break;
1462 default:
1463 break;
1464 }
1465 dst_offset = binding_layout->offset / 4;
1466 if (entry->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK)
1467 dst_offset += entry->dstArrayElement / 4;
1468 else
1469 dst_offset += binding_layout->size * entry->dstArrayElement / 4;
1470
1471 dst_stride = binding_layout->size / 4;
1472 break;
1473 }
1474
1475 templ->entry[i] = (struct radv_descriptor_update_template_entry){
1476 .descriptor_type = entry->descriptorType,
1477 .descriptor_count = entry->descriptorCount,
1478 .src_offset = entry->offset,
1479 .src_stride = entry->stride,
1480 .dst_offset = dst_offset,
1481 .dst_stride = dst_stride,
1482 .buffer_offset = buffer_offset,
1483 .has_sampler = !binding_layout->immutable_samplers_offset,
1484 .sampler_offset = radv_combined_image_descriptor_sampler_offset(binding_layout),
1485 .immutable_samplers = immutable_samplers};
1486 }
1487
1488 *pDescriptorUpdateTemplate = radv_descriptor_update_template_to_handle(templ);
1489 return VK_SUCCESS;
1490 }
1491
1492 VKAPI_ATTR void VKAPI_CALL
radv_DestroyDescriptorUpdateTemplate(VkDevice _device,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const VkAllocationCallbacks * pAllocator)1493 radv_DestroyDescriptorUpdateTemplate(VkDevice _device,
1494 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1495 const VkAllocationCallbacks *pAllocator)
1496 {
1497 RADV_FROM_HANDLE(radv_device, device, _device);
1498 RADV_FROM_HANDLE(radv_descriptor_update_template, templ, descriptorUpdateTemplate);
1499
1500 if (!templ)
1501 return;
1502
1503 vk_object_base_finish(&templ->base);
1504 vk_free2(&device->vk.alloc, pAllocator, templ);
1505 }
1506
1507 static ALWAYS_INLINE void
radv_update_descriptor_set_with_template_impl(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,struct radv_descriptor_set * set,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1508 radv_update_descriptor_set_with_template_impl(struct radv_device *device,
1509 struct radv_cmd_buffer *cmd_buffer,
1510 struct radv_descriptor_set *set,
1511 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1512 const void *pData)
1513 {
1514 RADV_FROM_HANDLE(radv_descriptor_update_template, templ, descriptorUpdateTemplate);
1515 uint32_t i;
1516
1517 for (i = 0; i < templ->entry_count; ++i) {
1518 struct radeon_winsys_bo **buffer_list = set->descriptors + templ->entry[i].buffer_offset;
1519 uint32_t *pDst = set->header.mapped_ptr + templ->entry[i].dst_offset;
1520 const uint8_t *pSrc = ((const uint8_t *)pData) + templ->entry[i].src_offset;
1521 uint32_t j;
1522
1523 if (templ->entry[i].descriptor_type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) {
1524 memcpy((uint8_t *)pDst, pSrc, templ->entry[i].descriptor_count);
1525 continue;
1526 }
1527
1528 for (j = 0; j < templ->entry[i].descriptor_count; ++j) {
1529 switch (templ->entry[i].descriptor_type) {
1530 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1531 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1532 const unsigned idx = templ->entry[i].dst_offset + j;
1533 assert(!(set->header.layout->flags &
1534 VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1535 write_dynamic_buffer_descriptor(device, set->header.dynamic_descriptors + idx,
1536 buffer_list, (struct VkDescriptorBufferInfo *)pSrc);
1537 break;
1538 }
1539 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1540 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1541 write_buffer_descriptor(device, cmd_buffer, pDst, buffer_list,
1542 (struct VkDescriptorBufferInfo *)pSrc);
1543 break;
1544 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1545 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1546 write_texel_buffer_descriptor(device, cmd_buffer, pDst, buffer_list,
1547 *(VkBufferView *)pSrc);
1548 break;
1549 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
1550 write_image_descriptor(device, cmd_buffer, 32, pDst, buffer_list,
1551 templ->entry[i].descriptor_type,
1552 (struct VkDescriptorImageInfo *)pSrc);
1553 break;
1554 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1555 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1556 write_image_descriptor(device, cmd_buffer, 64, pDst, buffer_list,
1557 templ->entry[i].descriptor_type,
1558 (struct VkDescriptorImageInfo *)pSrc);
1559 break;
1560 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1561 write_combined_image_sampler_descriptor(
1562 device, cmd_buffer, templ->entry[i].sampler_offset, pDst, buffer_list,
1563 templ->entry[i].descriptor_type, (struct VkDescriptorImageInfo *)pSrc,
1564 templ->entry[i].has_sampler);
1565 if (cmd_buffer && templ->entry[i].immutable_samplers) {
1566 memcpy((char *)pDst + templ->entry[i].sampler_offset,
1567 templ->entry[i].immutable_samplers + 4 * j, 16);
1568 }
1569 break;
1570 case VK_DESCRIPTOR_TYPE_SAMPLER:
1571 if (templ->entry[i].has_sampler)
1572 write_sampler_descriptor(device, pDst, (struct VkDescriptorImageInfo *)pSrc);
1573 else if (cmd_buffer && templ->entry[i].immutable_samplers)
1574 memcpy(pDst, templ->entry[i].immutable_samplers + 4 * j, 16);
1575 break;
1576 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
1577 write_accel_struct(pDst, *(const VkAccelerationStructureKHR *)pSrc);
1578 break;
1579 default:
1580 break;
1581 }
1582 pSrc += templ->entry[i].src_stride;
1583 pDst += templ->entry[i].dst_stride;
1584 ++buffer_list;
1585 }
1586 }
1587 }
1588
1589 void
radv_cmd_update_descriptor_set_with_template(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,struct radv_descriptor_set * set,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1590 radv_cmd_update_descriptor_set_with_template(struct radv_device *device,
1591 struct radv_cmd_buffer *cmd_buffer,
1592 struct radv_descriptor_set *set,
1593 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1594 const void *pData)
1595 {
1596 /* Assume cmd_buffer != NULL to optimize out cmd_buffer checks in generic code above. */
1597 assume(cmd_buffer != NULL);
1598 radv_update_descriptor_set_with_template_impl(device, cmd_buffer, set, descriptorUpdateTemplate, pData);
1599 }
1600
1601 VKAPI_ATTR void VKAPI_CALL
radv_UpdateDescriptorSetWithTemplate(VkDevice _device,VkDescriptorSet descriptorSet,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1602 radv_UpdateDescriptorSetWithTemplate(VkDevice _device, VkDescriptorSet descriptorSet,
1603 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1604 const void *pData)
1605 {
1606 RADV_FROM_HANDLE(radv_device, device, _device);
1607 RADV_FROM_HANDLE(radv_descriptor_set, set, descriptorSet);
1608
1609 radv_update_descriptor_set_with_template_impl(device, NULL, set, descriptorUpdateTemplate, pData);
1610 }
1611
1612 VKAPI_ATTR void VKAPI_CALL
radv_GetDescriptorSetLayoutHostMappingInfoVALVE(VkDevice _device,const VkDescriptorSetBindingReferenceVALVE * pBindingReference,VkDescriptorSetLayoutHostMappingInfoVALVE * pHostMapping)1613 radv_GetDescriptorSetLayoutHostMappingInfoVALVE(
1614 VkDevice _device, const VkDescriptorSetBindingReferenceVALVE *pBindingReference,
1615 VkDescriptorSetLayoutHostMappingInfoVALVE *pHostMapping)
1616 {
1617 struct radv_descriptor_set_layout *set_layout =
1618 radv_descriptor_set_layout_from_handle(pBindingReference->descriptorSetLayout);
1619
1620 const struct radv_descriptor_set_binding_layout *binding_layout =
1621 set_layout->binding + pBindingReference->binding;
1622
1623 pHostMapping->descriptorOffset = binding_layout->offset;
1624 pHostMapping->descriptorSize = binding_layout->size;
1625 }
1626
1627 VKAPI_ATTR void VKAPI_CALL
radv_GetDescriptorSetHostMappingVALVE(VkDevice _device,VkDescriptorSet descriptorSet,void ** ppData)1628 radv_GetDescriptorSetHostMappingVALVE(VkDevice _device, VkDescriptorSet descriptorSet,
1629 void **ppData)
1630 {
1631 RADV_FROM_HANDLE(radv_descriptor_set, set, descriptorSet);
1632 *ppData = set->header.mapped_ptr;
1633 }
1634
1635 VKAPI_ATTR VkResult VKAPI_CALL
radv_CreateSamplerYcbcrConversion(VkDevice _device,const VkSamplerYcbcrConversionCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkSamplerYcbcrConversion * pYcbcrConversion)1636 radv_CreateSamplerYcbcrConversion(VkDevice _device,
1637 const VkSamplerYcbcrConversionCreateInfo *pCreateInfo,
1638 const VkAllocationCallbacks *pAllocator,
1639 VkSamplerYcbcrConversion *pYcbcrConversion)
1640 {
1641 RADV_FROM_HANDLE(radv_device, device, _device);
1642 struct radv_sampler_ycbcr_conversion *conversion = NULL;
1643
1644 conversion = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*conversion), 8,
1645 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1646
1647 if (conversion == NULL)
1648 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1649
1650 vk_object_base_init(&device->vk, &conversion->base, VK_OBJECT_TYPE_SAMPLER_YCBCR_CONVERSION);
1651
1652 conversion->state.format = pCreateInfo->format;
1653 conversion->state.ycbcr_model = pCreateInfo->ycbcrModel;
1654 conversion->state.ycbcr_range = pCreateInfo->ycbcrRange;
1655 conversion->state.components = pCreateInfo->components;
1656 conversion->state.chroma_offsets[0] = pCreateInfo->xChromaOffset;
1657 conversion->state.chroma_offsets[1] = pCreateInfo->yChromaOffset;
1658 conversion->state.chroma_filter = pCreateInfo->chromaFilter;
1659
1660 *pYcbcrConversion = radv_sampler_ycbcr_conversion_to_handle(conversion);
1661 return VK_SUCCESS;
1662 }
1663
1664 VKAPI_ATTR void VKAPI_CALL
radv_DestroySamplerYcbcrConversion(VkDevice _device,VkSamplerYcbcrConversion ycbcrConversion,const VkAllocationCallbacks * pAllocator)1665 radv_DestroySamplerYcbcrConversion(VkDevice _device, VkSamplerYcbcrConversion ycbcrConversion,
1666 const VkAllocationCallbacks *pAllocator)
1667 {
1668 RADV_FROM_HANDLE(radv_device, device, _device);
1669 RADV_FROM_HANDLE(radv_sampler_ycbcr_conversion, ycbcr_conversion, ycbcrConversion);
1670
1671 if (!ycbcr_conversion)
1672 return;
1673
1674 vk_object_base_finish(&ycbcr_conversion->base);
1675 vk_free2(&device->vk.alloc, pAllocator, ycbcr_conversion);
1676 }
1677