1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 * SPDX-License-Identifier: MIT
5 */
6
7 /**
8 * @file
9 *
10 * We use the bindless descriptor model, which maps fairly closely to how
11 * Vulkan descriptor sets work. The two exceptions are input attachments and
12 * dynamic descriptors, which have to be patched when recording command
13 * buffers. We reserve an extra descriptor set for these. This descriptor set
14 * contains all the input attachments in the pipeline, in order, and then all
15 * the dynamic descriptors. The dynamic descriptors are stored in the CPU-side
16 * datastructure for each tu_descriptor_set, and then combined into one big
17 * descriptor set at CmdBindDescriptors time/draw time.
18 */
19
20 #include "tu_descriptor_set.h"
21
22 #include <fcntl.h>
23
24 #include "util/mesa-sha1.h"
25 #include "vk_descriptors.h"
26 #include "vk_util.h"
27
28 #include "tu_device.h"
29 #include "tu_image.h"
30
31 static inline uint8_t *
pool_base(struct tu_descriptor_pool * pool)32 pool_base(struct tu_descriptor_pool *pool)
33 {
34 return pool->host_bo ?: pool->bo->map;
35 }
36
37 static uint32_t
descriptor_size(struct tu_device * dev,VkDescriptorType type)38 descriptor_size(struct tu_device *dev, VkDescriptorType type)
39 {
40 switch (type) {
41 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
42 if (unlikely(dev->instance->debug_flags & TU_DEBUG_DYNAMIC))
43 return A6XX_TEX_CONST_DWORDS * 4;
44
45 /* Input attachment doesn't use descriptor sets at all */
46 return 0;
47 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
48 /* We make offsets and sizes all 16 dwords, to match how the hardware
49 * interprets indices passed to sample/load/store instructions in
50 * multiples of 16 dwords. This means that "normal" descriptors are all
51 * of size 16, with padding for smaller descriptors like uniform storage
52 * descriptors which are less than 16 dwords. However combined images
53 * and samplers are actually two descriptors, so they have size 2.
54 */
55 return A6XX_TEX_CONST_DWORDS * 4 * 2;
56 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
57 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
58 /* When we support 16-bit storage, we need an extra descriptor setup as
59 * a 32-bit array for isam to work.
60 */
61 if (dev->physical_device->info->a6xx.storage_16bit) {
62 return A6XX_TEX_CONST_DWORDS * 4 * 2;
63 } else {
64 return A6XX_TEX_CONST_DWORDS * 4;
65 }
66 default:
67 return A6XX_TEX_CONST_DWORDS * 4;
68 }
69 }
70
71 static bool
is_dynamic(VkDescriptorType type)72 is_dynamic(VkDescriptorType type)
73 {
74 return type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC ||
75 type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC;
76 }
77
78 static uint32_t
mutable_descriptor_size(struct tu_device * dev,const VkMutableDescriptorTypeListVALVE * list)79 mutable_descriptor_size(struct tu_device *dev, const VkMutableDescriptorTypeListVALVE *list)
80 {
81 uint32_t max_size = 0;
82
83 for (uint32_t i = 0; i < list->descriptorTypeCount; i++) {
84 uint32_t size = descriptor_size(dev, list->pDescriptorTypes[i]);
85 max_size = MAX2(max_size, size);
86 }
87
88 return max_size;
89 }
90
91 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateDescriptorSetLayout(VkDevice _device,const VkDescriptorSetLayoutCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorSetLayout * pSetLayout)92 tu_CreateDescriptorSetLayout(
93 VkDevice _device,
94 const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
95 const VkAllocationCallbacks *pAllocator,
96 VkDescriptorSetLayout *pSetLayout)
97 {
98 TU_FROM_HANDLE(tu_device, device, _device);
99 struct tu_descriptor_set_layout *set_layout;
100
101 assert(pCreateInfo->sType ==
102 VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO);
103 const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags =
104 vk_find_struct_const(
105 pCreateInfo->pNext,
106 DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO);
107 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
108 vk_find_struct_const(
109 pCreateInfo->pNext,
110 MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
111
112 uint32_t num_bindings = 0;
113 uint32_t immutable_sampler_count = 0;
114 uint32_t ycbcr_sampler_count = 0;
115 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
116 num_bindings = MAX2(num_bindings, pCreateInfo->pBindings[j].binding + 1);
117 if ((pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
118 pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
119 pCreateInfo->pBindings[j].pImmutableSamplers) {
120 immutable_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
121
122 bool has_ycbcr_sampler = false;
123 for (unsigned i = 0; i < pCreateInfo->pBindings[j].descriptorCount; ++i) {
124 if (tu_sampler_from_handle(pCreateInfo->pBindings[j].pImmutableSamplers[i])->ycbcr_sampler)
125 has_ycbcr_sampler = true;
126 }
127
128 if (has_ycbcr_sampler)
129 ycbcr_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
130 }
131 }
132
133 uint32_t samplers_offset =
134 offsetof(struct tu_descriptor_set_layout, binding[num_bindings]);
135
136 /* note: only need to store TEX_SAMP_DWORDS for immutable samples,
137 * but using struct tu_sampler makes things simpler */
138 uint32_t size = samplers_offset +
139 immutable_sampler_count * sizeof(struct tu_sampler) +
140 ycbcr_sampler_count * sizeof(struct tu_sampler_ycbcr_conversion);
141
142 set_layout = vk_object_zalloc(&device->vk, NULL, size,
143 VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT);
144 if (!set_layout)
145 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
146
147 set_layout->flags = pCreateInfo->flags;
148
149 /* We just allocate all the immutable samplers at the end of the struct */
150 struct tu_sampler *samplers = (void*) &set_layout->binding[num_bindings];
151 struct tu_sampler_ycbcr_conversion *ycbcr_samplers =
152 (void*) &samplers[immutable_sampler_count];
153
154 VkDescriptorSetLayoutBinding *bindings = NULL;
155 VkResult result = vk_create_sorted_bindings(
156 pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
157 if (result != VK_SUCCESS) {
158 vk_object_free(&device->vk, pAllocator, set_layout);
159 return vk_error(device, result);
160 }
161
162 set_layout->ref_cnt = 1;
163 set_layout->binding_count = num_bindings;
164 set_layout->shader_stages = 0;
165 set_layout->has_immutable_samplers = false;
166 set_layout->size = 0;
167
168 uint32_t dynamic_offset_size = 0;
169
170 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
171 const VkDescriptorSetLayoutBinding *binding = bindings + j;
172 uint32_t b = binding->binding;
173
174 set_layout->binding[b].type = binding->descriptorType;
175 set_layout->binding[b].array_size = binding->descriptorCount;
176 set_layout->binding[b].offset = set_layout->size;
177 set_layout->binding[b].dynamic_offset_offset = dynamic_offset_size;
178 set_layout->binding[b].shader_stages = binding->stageFlags;
179
180 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_VALVE) {
181 /* For mutable descriptor types we must allocate a size that fits the
182 * largest descriptor type that the binding can mutate to.
183 */
184 set_layout->binding[b].size =
185 mutable_descriptor_size(device, &mutable_info->pMutableDescriptorTypeLists[j]);
186 } else {
187 set_layout->binding[b].size = descriptor_size(device, binding->descriptorType);
188 }
189
190 if (variable_flags && binding->binding < variable_flags->bindingCount &&
191 (variable_flags->pBindingFlags[binding->binding] &
192 VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) {
193 assert(!binding->pImmutableSamplers); /* Terribly ill defined how
194 many samplers are valid */
195 assert(binding->binding == num_bindings - 1);
196
197 set_layout->has_variable_descriptors = true;
198 }
199
200 if ((binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
201 binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
202 binding->pImmutableSamplers) {
203 set_layout->binding[b].immutable_samplers_offset = samplers_offset;
204 set_layout->has_immutable_samplers = true;
205
206 for (uint32_t i = 0; i < binding->descriptorCount; i++)
207 samplers[i] = *tu_sampler_from_handle(binding->pImmutableSamplers[i]);
208
209 samplers += binding->descriptorCount;
210 samplers_offset += sizeof(struct tu_sampler) * binding->descriptorCount;
211
212 bool has_ycbcr_sampler = false;
213 for (unsigned i = 0; i < pCreateInfo->pBindings[j].descriptorCount; ++i) {
214 if (tu_sampler_from_handle(binding->pImmutableSamplers[i])->ycbcr_sampler)
215 has_ycbcr_sampler = true;
216 }
217
218 if (has_ycbcr_sampler) {
219 set_layout->binding[b].ycbcr_samplers_offset =
220 (const char*)ycbcr_samplers - (const char*)set_layout;
221 for (uint32_t i = 0; i < binding->descriptorCount; i++) {
222 struct tu_sampler *sampler = tu_sampler_from_handle(binding->pImmutableSamplers[i]);
223 if (sampler->ycbcr_sampler)
224 ycbcr_samplers[i] = *sampler->ycbcr_sampler;
225 else
226 ycbcr_samplers[i].ycbcr_model = VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY;
227 }
228 ycbcr_samplers += binding->descriptorCount;
229 } else {
230 set_layout->binding[b].ycbcr_samplers_offset = 0;
231 }
232 }
233
234 if (is_dynamic(binding->descriptorType)) {
235 dynamic_offset_size +=
236 binding->descriptorCount * set_layout->binding[b].size;
237 } else {
238 set_layout->size +=
239 binding->descriptorCount * set_layout->binding[b].size;
240 }
241
242 set_layout->shader_stages |= binding->stageFlags;
243 }
244
245 free(bindings);
246
247 set_layout->dynamic_offset_size = dynamic_offset_size;
248
249 *pSetLayout = tu_descriptor_set_layout_to_handle(set_layout);
250
251 return VK_SUCCESS;
252 }
253
254 VKAPI_ATTR void VKAPI_CALL
tu_DestroyDescriptorSetLayout(VkDevice _device,VkDescriptorSetLayout _set_layout,const VkAllocationCallbacks * pAllocator)255 tu_DestroyDescriptorSetLayout(VkDevice _device,
256 VkDescriptorSetLayout _set_layout,
257 const VkAllocationCallbacks *pAllocator)
258 {
259 TU_FROM_HANDLE(tu_device, device, _device);
260 TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout, _set_layout);
261
262 if (!set_layout)
263 return;
264
265 tu_descriptor_set_layout_unref(device, set_layout);
266 }
267
268 void
tu_descriptor_set_layout_destroy(struct tu_device * device,struct tu_descriptor_set_layout * layout)269 tu_descriptor_set_layout_destroy(struct tu_device *device,
270 struct tu_descriptor_set_layout *layout)
271 {
272 assert(layout->ref_cnt == 0);
273 vk_object_free(&device->vk, NULL, layout);
274 }
275
276 VKAPI_ATTR void VKAPI_CALL
tu_GetDescriptorSetLayoutSupport(VkDevice _device,const VkDescriptorSetLayoutCreateInfo * pCreateInfo,VkDescriptorSetLayoutSupport * pSupport)277 tu_GetDescriptorSetLayoutSupport(
278 VkDevice _device,
279 const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
280 VkDescriptorSetLayoutSupport *pSupport)
281 {
282 TU_FROM_HANDLE(tu_device, device, _device);
283
284 VkDescriptorSetLayoutBinding *bindings = NULL;
285 VkResult result = vk_create_sorted_bindings(
286 pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
287 if (result != VK_SUCCESS) {
288 pSupport->supported = false;
289 return;
290 }
291
292 const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags =
293 vk_find_struct_const(
294 pCreateInfo->pNext,
295 DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO);
296 VkDescriptorSetVariableDescriptorCountLayoutSupport *variable_count =
297 vk_find_struct(
298 (void *) pCreateInfo->pNext,
299 DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_LAYOUT_SUPPORT);
300 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
301 vk_find_struct_const(
302 pCreateInfo->pNext,
303 MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
304
305 if (variable_count) {
306 variable_count->maxVariableDescriptorCount = 0;
307 }
308
309 bool supported = true;
310 uint64_t size = 0;
311 for (uint32_t i = 0; i < pCreateInfo->bindingCount; i++) {
312 const VkDescriptorSetLayoutBinding *binding = bindings + i;
313
314 uint64_t descriptor_sz;
315
316 if (is_dynamic(binding->descriptorType)) {
317 descriptor_sz = 0;
318 } else if (binding->descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_VALVE) {
319 const VkMutableDescriptorTypeListVALVE *list =
320 &mutable_info->pMutableDescriptorTypeLists[i];
321
322 for (uint32_t j = 0; j < list->descriptorTypeCount; j++) {
323 /* Don't support the input attachement and combined image sampler type
324 * for mutable descriptors */
325 if (list->pDescriptorTypes[j] == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT ||
326 list->pDescriptorTypes[j] == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) {
327 supported = false;
328 goto out;
329 }
330 }
331
332 descriptor_sz =
333 mutable_descriptor_size(device, &mutable_info->pMutableDescriptorTypeLists[i]);
334 } else {
335 descriptor_sz = descriptor_size(device, binding->descriptorType);
336 }
337 uint64_t descriptor_alignment = 8;
338
339 if (size && !ALIGN_POT(size, descriptor_alignment)) {
340 supported = false;
341 }
342 size = ALIGN_POT(size, descriptor_alignment);
343
344 uint64_t max_count = UINT64_MAX;
345 if (descriptor_sz)
346 max_count = (UINT64_MAX - size) / descriptor_sz;
347
348 if (max_count < binding->descriptorCount) {
349 supported = false;
350 }
351
352 if (variable_flags && binding->binding < variable_flags->bindingCount &&
353 variable_count &&
354 (variable_flags->pBindingFlags[binding->binding] &
355 VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) {
356 variable_count->maxVariableDescriptorCount =
357 MIN2(UINT32_MAX, max_count);
358 }
359 size += binding->descriptorCount * descriptor_sz;
360 }
361
362 out:
363 free(bindings);
364
365 pSupport->supported = supported;
366 }
367
368 /* Note: we must hash any values used in tu_lower_io(). */
369
370 #define SHA1_UPDATE_VALUE(ctx, x) _mesa_sha1_update(ctx, &(x), sizeof(x));
371
372 static void
sha1_update_ycbcr_sampler(struct mesa_sha1 * ctx,const struct tu_sampler_ycbcr_conversion * sampler)373 sha1_update_ycbcr_sampler(struct mesa_sha1 *ctx,
374 const struct tu_sampler_ycbcr_conversion *sampler)
375 {
376 SHA1_UPDATE_VALUE(ctx, sampler->ycbcr_model);
377 SHA1_UPDATE_VALUE(ctx, sampler->ycbcr_range);
378 SHA1_UPDATE_VALUE(ctx, sampler->format);
379 }
380
381 static void
sha1_update_descriptor_set_binding_layout(struct mesa_sha1 * ctx,const struct tu_descriptor_set_binding_layout * layout,const struct tu_descriptor_set_layout * set_layout)382 sha1_update_descriptor_set_binding_layout(struct mesa_sha1 *ctx,
383 const struct tu_descriptor_set_binding_layout *layout,
384 const struct tu_descriptor_set_layout *set_layout)
385 {
386 SHA1_UPDATE_VALUE(ctx, layout->type);
387 SHA1_UPDATE_VALUE(ctx, layout->offset);
388 SHA1_UPDATE_VALUE(ctx, layout->size);
389 SHA1_UPDATE_VALUE(ctx, layout->array_size);
390 SHA1_UPDATE_VALUE(ctx, layout->dynamic_offset_offset);
391 SHA1_UPDATE_VALUE(ctx, layout->immutable_samplers_offset);
392
393 const struct tu_sampler_ycbcr_conversion *ycbcr_samplers =
394 tu_immutable_ycbcr_samplers(set_layout, layout);
395
396 if (ycbcr_samplers) {
397 for (unsigned i = 0; i < layout->array_size; i++)
398 sha1_update_ycbcr_sampler(ctx, ycbcr_samplers + i);
399 }
400 }
401
402
403 static void
sha1_update_descriptor_set_layout(struct mesa_sha1 * ctx,const struct tu_descriptor_set_layout * layout)404 sha1_update_descriptor_set_layout(struct mesa_sha1 *ctx,
405 const struct tu_descriptor_set_layout *layout)
406 {
407 for (uint16_t i = 0; i < layout->binding_count; i++)
408 sha1_update_descriptor_set_binding_layout(ctx, &layout->binding[i],
409 layout);
410 }
411
412 /*
413 * Pipeline layouts. These have nothing to do with the pipeline. They are
414 * just multiple descriptor set layouts pasted together.
415 */
416
417 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreatePipelineLayout(VkDevice _device,const VkPipelineLayoutCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkPipelineLayout * pPipelineLayout)418 tu_CreatePipelineLayout(VkDevice _device,
419 const VkPipelineLayoutCreateInfo *pCreateInfo,
420 const VkAllocationCallbacks *pAllocator,
421 VkPipelineLayout *pPipelineLayout)
422 {
423 TU_FROM_HANDLE(tu_device, device, _device);
424 struct tu_pipeline_layout *layout;
425
426 assert(pCreateInfo->sType ==
427 VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO);
428
429 layout = vk_object_alloc(&device->vk, pAllocator, sizeof(*layout),
430 VK_OBJECT_TYPE_PIPELINE_LAYOUT);
431 if (layout == NULL)
432 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
433
434 layout->num_sets = pCreateInfo->setLayoutCount;
435 layout->dynamic_offset_size = 0;
436
437 unsigned dynamic_offset_size = 0;
438
439 for (uint32_t set = 0; set < pCreateInfo->setLayoutCount; set++) {
440 TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout,
441 pCreateInfo->pSetLayouts[set]);
442
443 assert(set < MAX_SETS);
444 layout->set[set].layout = set_layout;
445 layout->set[set].dynamic_offset_start = dynamic_offset_size;
446 tu_descriptor_set_layout_ref(set_layout);
447
448 dynamic_offset_size += set_layout->dynamic_offset_size;
449 }
450
451 layout->dynamic_offset_size = dynamic_offset_size;
452 layout->push_constant_size = 0;
453
454 for (unsigned i = 0; i < pCreateInfo->pushConstantRangeCount; ++i) {
455 const VkPushConstantRange *range = pCreateInfo->pPushConstantRanges + i;
456 layout->push_constant_size =
457 MAX2(layout->push_constant_size, range->offset + range->size);
458 }
459
460 layout->push_constant_size = align(layout->push_constant_size, 16);
461
462 struct mesa_sha1 ctx;
463 _mesa_sha1_init(&ctx);
464 for (unsigned s = 0; s < layout->num_sets; s++) {
465 sha1_update_descriptor_set_layout(&ctx, layout->set[s].layout);
466 _mesa_sha1_update(&ctx, &layout->set[s].dynamic_offset_start,
467 sizeof(layout->set[s].dynamic_offset_start));
468 }
469 _mesa_sha1_update(&ctx, &layout->num_sets, sizeof(layout->num_sets));
470 _mesa_sha1_update(&ctx, &layout->push_constant_size,
471 sizeof(layout->push_constant_size));
472 _mesa_sha1_final(&ctx, layout->sha1);
473
474 *pPipelineLayout = tu_pipeline_layout_to_handle(layout);
475
476 return VK_SUCCESS;
477 }
478
479 VKAPI_ATTR void VKAPI_CALL
tu_DestroyPipelineLayout(VkDevice _device,VkPipelineLayout _pipelineLayout,const VkAllocationCallbacks * pAllocator)480 tu_DestroyPipelineLayout(VkDevice _device,
481 VkPipelineLayout _pipelineLayout,
482 const VkAllocationCallbacks *pAllocator)
483 {
484 TU_FROM_HANDLE(tu_device, device, _device);
485 TU_FROM_HANDLE(tu_pipeline_layout, pipeline_layout, _pipelineLayout);
486
487 if (!pipeline_layout)
488 return;
489
490 for (uint32_t i = 0; i < pipeline_layout->num_sets; i++)
491 tu_descriptor_set_layout_unref(device, pipeline_layout->set[i].layout);
492
493 vk_object_free(&device->vk, pAllocator, pipeline_layout);
494 }
495
496 #define EMPTY 1
497
498 static VkResult
tu_descriptor_set_create(struct tu_device * device,struct tu_descriptor_pool * pool,struct tu_descriptor_set_layout * layout,const uint32_t * variable_count,struct tu_descriptor_set ** out_set)499 tu_descriptor_set_create(struct tu_device *device,
500 struct tu_descriptor_pool *pool,
501 struct tu_descriptor_set_layout *layout,
502 const uint32_t *variable_count,
503 struct tu_descriptor_set **out_set)
504 {
505 struct tu_descriptor_set *set;
506 unsigned dynamic_offset = sizeof(struct tu_descriptor_set);
507 unsigned mem_size = dynamic_offset + layout->dynamic_offset_size;
508
509 if (pool->host_memory_base) {
510 if (pool->host_memory_end - pool->host_memory_ptr < mem_size)
511 return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
512
513 set = (struct tu_descriptor_set*)pool->host_memory_ptr;
514 pool->host_memory_ptr += mem_size;
515 } else {
516 set = vk_alloc2(&device->vk.alloc, NULL, mem_size, 8,
517 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
518
519 if (!set)
520 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
521 }
522
523 memset(set, 0, mem_size);
524 vk_object_base_init(&device->vk, &set->base, VK_OBJECT_TYPE_DESCRIPTOR_SET);
525
526 if (layout->dynamic_offset_size) {
527 set->dynamic_descriptors = (uint32_t *)((uint8_t*)set + dynamic_offset);
528 }
529
530 set->layout = layout;
531 set->pool = pool;
532 uint32_t layout_size = layout->size;
533 if (variable_count) {
534 assert(layout->has_variable_descriptors);
535 uint32_t stride = layout->binding[layout->binding_count - 1].size;
536 layout_size = layout->binding[layout->binding_count - 1].offset +
537 *variable_count * stride;
538 }
539
540 if (layout_size) {
541 set->size = layout_size;
542
543 if (!pool->host_memory_base && pool->entry_count == pool->max_entry_count) {
544 vk_object_free(&device->vk, NULL, set);
545 return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
546 }
547
548 /* try to allocate linearly first, so that we don't spend
549 * time looking for gaps if the app only allocates &
550 * resets via the pool. */
551 if (pool->current_offset + layout_size <= pool->size) {
552 set->mapped_ptr = (uint32_t*)(pool_base(pool) + pool->current_offset);
553 set->va = pool->host_bo ? 0 : pool->bo->iova + pool->current_offset;
554
555 if (!pool->host_memory_base) {
556 pool->entries[pool->entry_count].offset = pool->current_offset;
557 pool->entries[pool->entry_count].size = layout_size;
558 pool->entries[pool->entry_count].set = set;
559 pool->entry_count++;
560 }
561 pool->current_offset += layout_size;
562 } else if (!pool->host_memory_base) {
563 uint64_t offset = 0;
564 int index;
565
566 for (index = 0; index < pool->entry_count; ++index) {
567 if (pool->entries[index].offset - offset >= layout_size)
568 break;
569 offset = pool->entries[index].offset + pool->entries[index].size;
570 }
571
572 if (pool->size - offset < layout_size) {
573 vk_object_free(&device->vk, NULL, set);
574 return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
575 }
576
577 set->mapped_ptr = (uint32_t*)(pool_base(pool) + offset);
578 set->va = pool->host_bo ? 0 : pool->bo->iova + offset;
579
580 memmove(&pool->entries[index + 1], &pool->entries[index],
581 sizeof(pool->entries[0]) * (pool->entry_count - index));
582 pool->entries[index].offset = offset;
583 pool->entries[index].size = layout_size;
584 pool->entries[index].set = set;
585 pool->entry_count++;
586 } else
587 return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
588 }
589
590 if (layout->has_immutable_samplers) {
591 for (unsigned i = 0; i < layout->binding_count; ++i) {
592 if (!layout->binding[i].immutable_samplers_offset)
593 continue;
594
595 unsigned offset = layout->binding[i].offset / 4;
596 if (layout->binding[i].type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
597 offset += A6XX_TEX_CONST_DWORDS;
598
599 const struct tu_sampler *samplers =
600 (const struct tu_sampler *)((const char *)layout +
601 layout->binding[i].immutable_samplers_offset);
602 for (unsigned j = 0; j < layout->binding[i].array_size; ++j) {
603 memcpy(set->mapped_ptr + offset, samplers[j].descriptor,
604 sizeof(samplers[j].descriptor));
605 offset += layout->binding[i].size / 4;
606 }
607 }
608 }
609
610 tu_descriptor_set_layout_ref(layout);
611 list_addtail(&set->pool_link, &pool->desc_sets);
612
613 *out_set = set;
614 return VK_SUCCESS;
615 }
616
617 static void
tu_descriptor_set_destroy(struct tu_device * device,struct tu_descriptor_pool * pool,struct tu_descriptor_set * set,bool free_bo)618 tu_descriptor_set_destroy(struct tu_device *device,
619 struct tu_descriptor_pool *pool,
620 struct tu_descriptor_set *set,
621 bool free_bo)
622 {
623 assert(!pool->host_memory_base);
624
625 if (free_bo && set->size && !pool->host_memory_base) {
626 uint32_t offset = (uint8_t*)set->mapped_ptr - pool_base(pool);
627
628 for (int i = 0; i < pool->entry_count; ++i) {
629 if (pool->entries[i].offset == offset) {
630 memmove(&pool->entries[i], &pool->entries[i+1],
631 sizeof(pool->entries[i]) * (pool->entry_count - i - 1));
632 --pool->entry_count;
633 break;
634 }
635 }
636 }
637
638 vk_object_free(&device->vk, NULL, set);
639 }
640
641 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateDescriptorPool(VkDevice _device,const VkDescriptorPoolCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorPool * pDescriptorPool)642 tu_CreateDescriptorPool(VkDevice _device,
643 const VkDescriptorPoolCreateInfo *pCreateInfo,
644 const VkAllocationCallbacks *pAllocator,
645 VkDescriptorPool *pDescriptorPool)
646 {
647 TU_FROM_HANDLE(tu_device, device, _device);
648 struct tu_descriptor_pool *pool;
649 uint64_t size = sizeof(struct tu_descriptor_pool);
650 uint64_t bo_size = 0, bo_count = 0, dynamic_size = 0;
651 VkResult ret;
652
653 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
654 vk_find_struct_const( pCreateInfo->pNext,
655 MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
656
657 for (unsigned i = 0; i < pCreateInfo->poolSizeCount; ++i) {
658 if (pCreateInfo->pPoolSizes[i].type != VK_DESCRIPTOR_TYPE_SAMPLER)
659 bo_count += pCreateInfo->pPoolSizes[i].descriptorCount;
660
661 switch(pCreateInfo->pPoolSizes[i].type) {
662 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
663 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
664 dynamic_size += descriptor_size(device, pCreateInfo->pPoolSizes[i].type) *
665 pCreateInfo->pPoolSizes[i].descriptorCount;
666 break;
667 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE:
668 if (mutable_info && i < mutable_info->mutableDescriptorTypeListCount &&
669 mutable_info->pMutableDescriptorTypeLists[i].descriptorTypeCount > 0) {
670 bo_size +=
671 mutable_descriptor_size(device, &mutable_info->pMutableDescriptorTypeLists[i]) *
672 pCreateInfo->pPoolSizes[i].descriptorCount;
673 } else {
674 /* Allocate the maximum size possible. */
675 bo_size += 2 * A6XX_TEX_CONST_DWORDS * 4 *
676 pCreateInfo->pPoolSizes[i].descriptorCount;
677 }
678 continue;
679 default:
680 break;
681 }
682
683 bo_size += descriptor_size(device, pCreateInfo->pPoolSizes[i].type) *
684 pCreateInfo->pPoolSizes[i].descriptorCount;
685 }
686
687 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
688 uint64_t host_size = pCreateInfo->maxSets * sizeof(struct tu_descriptor_set);
689 host_size += sizeof(struct tu_bo*) * bo_count;
690 host_size += dynamic_size;
691 size += host_size;
692 } else {
693 size += sizeof(struct tu_descriptor_pool_entry) * pCreateInfo->maxSets;
694 }
695
696 pool = vk_object_zalloc(&device->vk, pAllocator, size,
697 VK_OBJECT_TYPE_DESCRIPTOR_POOL);
698 if (!pool)
699 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
700
701 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
702 pool->host_memory_base = (uint8_t*)pool + sizeof(struct tu_descriptor_pool);
703 pool->host_memory_ptr = pool->host_memory_base;
704 pool->host_memory_end = (uint8_t*)pool + size;
705 }
706
707 if (bo_size) {
708 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_HOST_ONLY_BIT_VALVE)) {
709 ret = tu_bo_init_new(device, &pool->bo, bo_size, TU_BO_ALLOC_ALLOW_DUMP);
710 if (ret)
711 goto fail_alloc;
712
713 ret = tu_bo_map(device, pool->bo);
714 if (ret)
715 goto fail_map;
716 } else {
717 pool->host_bo = vk_alloc2(&device->vk.alloc, pAllocator, bo_size, 8,
718 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
719 if (!pool->host_bo) {
720 ret = VK_ERROR_OUT_OF_HOST_MEMORY;
721 goto fail_alloc;
722 }
723 }
724 }
725 pool->size = bo_size;
726 pool->max_entry_count = pCreateInfo->maxSets;
727
728 list_inithead(&pool->desc_sets);
729
730 *pDescriptorPool = tu_descriptor_pool_to_handle(pool);
731 return VK_SUCCESS;
732
733 fail_map:
734 tu_bo_finish(device, pool->bo);
735 fail_alloc:
736 vk_object_free(&device->vk, pAllocator, pool);
737 return ret;
738 }
739
740 VKAPI_ATTR void VKAPI_CALL
tu_DestroyDescriptorPool(VkDevice _device,VkDescriptorPool _pool,const VkAllocationCallbacks * pAllocator)741 tu_DestroyDescriptorPool(VkDevice _device,
742 VkDescriptorPool _pool,
743 const VkAllocationCallbacks *pAllocator)
744 {
745 TU_FROM_HANDLE(tu_device, device, _device);
746 TU_FROM_HANDLE(tu_descriptor_pool, pool, _pool);
747
748 if (!pool)
749 return;
750
751 list_for_each_entry_safe(struct tu_descriptor_set, set,
752 &pool->desc_sets, pool_link) {
753 tu_descriptor_set_layout_unref(device, set->layout);
754 }
755
756 if (!pool->host_memory_base) {
757 for(int i = 0; i < pool->entry_count; ++i) {
758 tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
759 }
760 }
761
762 if (pool->size) {
763 if (pool->host_bo)
764 vk_free2(&device->vk.alloc, pAllocator, pool->host_bo);
765 else
766 tu_bo_finish(device, pool->bo);
767 }
768
769 vk_object_free(&device->vk, pAllocator, pool);
770 }
771
772 VKAPI_ATTR VkResult VKAPI_CALL
tu_ResetDescriptorPool(VkDevice _device,VkDescriptorPool descriptorPool,VkDescriptorPoolResetFlags flags)773 tu_ResetDescriptorPool(VkDevice _device,
774 VkDescriptorPool descriptorPool,
775 VkDescriptorPoolResetFlags flags)
776 {
777 TU_FROM_HANDLE(tu_device, device, _device);
778 TU_FROM_HANDLE(tu_descriptor_pool, pool, descriptorPool);
779
780 list_for_each_entry_safe(struct tu_descriptor_set, set,
781 &pool->desc_sets, pool_link) {
782 tu_descriptor_set_layout_unref(device, set->layout);
783 }
784 list_inithead(&pool->desc_sets);
785
786 if (!pool->host_memory_base) {
787 for(int i = 0; i < pool->entry_count; ++i) {
788 tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
789 }
790 pool->entry_count = 0;
791 }
792
793 pool->current_offset = 0;
794 pool->host_memory_ptr = pool->host_memory_base;
795
796 return VK_SUCCESS;
797 }
798
799 VKAPI_ATTR VkResult VKAPI_CALL
tu_AllocateDescriptorSets(VkDevice _device,const VkDescriptorSetAllocateInfo * pAllocateInfo,VkDescriptorSet * pDescriptorSets)800 tu_AllocateDescriptorSets(VkDevice _device,
801 const VkDescriptorSetAllocateInfo *pAllocateInfo,
802 VkDescriptorSet *pDescriptorSets)
803 {
804 TU_FROM_HANDLE(tu_device, device, _device);
805 TU_FROM_HANDLE(tu_descriptor_pool, pool, pAllocateInfo->descriptorPool);
806
807 VkResult result = VK_SUCCESS;
808 uint32_t i;
809 struct tu_descriptor_set *set = NULL;
810
811 const VkDescriptorSetVariableDescriptorCountAllocateInfo *variable_counts =
812 vk_find_struct_const(pAllocateInfo->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_ALLOCATE_INFO);
813 const uint32_t zero = 0;
814
815 /* allocate a set of buffers for each shader to contain descriptors */
816 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
817 TU_FROM_HANDLE(tu_descriptor_set_layout, layout,
818 pAllocateInfo->pSetLayouts[i]);
819
820 const uint32_t *variable_count = NULL;
821 if (variable_counts) {
822 if (i < variable_counts->descriptorSetCount)
823 variable_count = variable_counts->pDescriptorCounts + i;
824 else
825 variable_count = &zero;
826 }
827
828 assert(!(layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
829
830 result = tu_descriptor_set_create(device, pool, layout, variable_count, &set);
831 if (result != VK_SUCCESS)
832 break;
833
834 pDescriptorSets[i] = tu_descriptor_set_to_handle(set);
835 }
836
837 if (result != VK_SUCCESS) {
838 tu_FreeDescriptorSets(_device, pAllocateInfo->descriptorPool,
839 i, pDescriptorSets);
840 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
841 pDescriptorSets[i] = VK_NULL_HANDLE;
842 }
843 }
844 return result;
845 }
846
847 VKAPI_ATTR VkResult VKAPI_CALL
tu_FreeDescriptorSets(VkDevice _device,VkDescriptorPool descriptorPool,uint32_t count,const VkDescriptorSet * pDescriptorSets)848 tu_FreeDescriptorSets(VkDevice _device,
849 VkDescriptorPool descriptorPool,
850 uint32_t count,
851 const VkDescriptorSet *pDescriptorSets)
852 {
853 TU_FROM_HANDLE(tu_device, device, _device);
854 TU_FROM_HANDLE(tu_descriptor_pool, pool, descriptorPool);
855
856 for (uint32_t i = 0; i < count; i++) {
857 TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
858
859 if (set) {
860 tu_descriptor_set_layout_unref(device, set->layout);
861 list_del(&set->pool_link);
862 }
863
864 if (set && !pool->host_memory_base)
865 tu_descriptor_set_destroy(device, pool, set, true);
866 }
867 return VK_SUCCESS;
868 }
869
870 static void
write_texel_buffer_descriptor(uint32_t * dst,const VkBufferView buffer_view)871 write_texel_buffer_descriptor(uint32_t *dst, const VkBufferView buffer_view)
872 {
873 if (buffer_view == VK_NULL_HANDLE) {
874 memset(dst, 0, A6XX_TEX_CONST_DWORDS * sizeof(uint32_t));
875 } else {
876 TU_FROM_HANDLE(tu_buffer_view, view, buffer_view);
877
878 memcpy(dst, view->descriptor, sizeof(view->descriptor));
879 }
880 }
881
get_range(struct tu_buffer * buf,VkDeviceSize offset,VkDeviceSize range)882 static uint32_t get_range(struct tu_buffer *buf, VkDeviceSize offset,
883 VkDeviceSize range)
884 {
885 if (range == VK_WHOLE_SIZE) {
886 return buf->size - offset;
887 } else {
888 return range;
889 }
890 }
891
892 static void
write_buffer_descriptor(const struct tu_device * device,uint32_t * dst,const VkDescriptorBufferInfo * buffer_info)893 write_buffer_descriptor(const struct tu_device *device,
894 uint32_t *dst,
895 const VkDescriptorBufferInfo *buffer_info)
896 {
897 bool storage_16bit = device->physical_device->info->a6xx.storage_16bit;
898 /* newer a6xx allows using 16-bit descriptor for both 16-bit and 32-bit
899 * access, but we need to keep a 32-bit descriptor for readonly access via
900 * isam.
901 */
902 unsigned descriptors = storage_16bit ? 2 : 1;
903 if (buffer_info->buffer == VK_NULL_HANDLE) {
904 memset(dst, 0, descriptors * A6XX_TEX_CONST_DWORDS * sizeof(uint32_t));
905 return;
906 }
907
908 TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
909
910 assert((buffer_info->offset & 63) == 0); /* minStorageBufferOffsetAlignment */
911 uint64_t va = buffer->iova + buffer_info->offset;
912 uint32_t range = get_range(buffer, buffer_info->offset, buffer_info->range);
913
914 for (unsigned i = 0; i < descriptors; i++) {
915 if (storage_16bit && i == 0) {
916 dst[0] = A6XX_TEX_CONST_0_TILE_MODE(TILE6_LINEAR) | A6XX_TEX_CONST_0_FMT(FMT6_16_UINT);
917 dst[1] = DIV_ROUND_UP(range, 2);
918 } else {
919 dst[0] = A6XX_TEX_CONST_0_TILE_MODE(TILE6_LINEAR) | A6XX_TEX_CONST_0_FMT(FMT6_32_UINT);
920 dst[1] = DIV_ROUND_UP(range, 4);
921 }
922 dst[2] =
923 A6XX_TEX_CONST_2_BUFFER | A6XX_TEX_CONST_2_TYPE(A6XX_TEX_BUFFER);
924 dst[3] = 0;
925 dst[4] = A6XX_TEX_CONST_4_BASE_LO(va);
926 dst[5] = A6XX_TEX_CONST_5_BASE_HI(va >> 32);
927 for (int j = 6; j < A6XX_TEX_CONST_DWORDS; j++)
928 dst[j] = 0;
929 dst += A6XX_TEX_CONST_DWORDS;
930 }
931 }
932
933 static void
write_ubo_descriptor(uint32_t * dst,const VkDescriptorBufferInfo * buffer_info)934 write_ubo_descriptor(uint32_t *dst, const VkDescriptorBufferInfo *buffer_info)
935 {
936 if (buffer_info->buffer == VK_NULL_HANDLE) {
937 dst[0] = dst[1] = 0;
938 return;
939 }
940
941 TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
942
943 uint32_t range = get_range(buffer, buffer_info->offset, buffer_info->range);
944 /* The HW range is in vec4 units */
945 range = ALIGN_POT(range, 16) / 16;
946 uint64_t va = buffer->iova + buffer_info->offset;
947
948 dst[0] = A6XX_UBO_0_BASE_LO(va);
949 dst[1] = A6XX_UBO_1_BASE_HI(va >> 32) | A6XX_UBO_1_SIZE(range);
950 }
951
952 static void
write_image_descriptor(uint32_t * dst,VkDescriptorType descriptor_type,const VkDescriptorImageInfo * image_info)953 write_image_descriptor(uint32_t *dst,
954 VkDescriptorType descriptor_type,
955 const VkDescriptorImageInfo *image_info)
956 {
957 if (image_info->imageView == VK_NULL_HANDLE) {
958 memset(dst, 0, A6XX_TEX_CONST_DWORDS * sizeof(uint32_t));
959 return;
960 }
961
962 TU_FROM_HANDLE(tu_image_view, iview, image_info->imageView);
963
964 if (descriptor_type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) {
965 memcpy(dst, iview->view.storage_descriptor, sizeof(iview->view.storage_descriptor));
966 } else {
967 memcpy(dst, iview->view.descriptor, sizeof(iview->view.descriptor));
968 }
969 }
970
971 static void
write_combined_image_sampler_descriptor(uint32_t * dst,VkDescriptorType descriptor_type,const VkDescriptorImageInfo * image_info,bool has_sampler)972 write_combined_image_sampler_descriptor(uint32_t *dst,
973 VkDescriptorType descriptor_type,
974 const VkDescriptorImageInfo *image_info,
975 bool has_sampler)
976 {
977 write_image_descriptor(dst, descriptor_type, image_info);
978 /* copy over sampler state */
979 if (has_sampler) {
980 TU_FROM_HANDLE(tu_sampler, sampler, image_info->sampler);
981 memcpy(dst + A6XX_TEX_CONST_DWORDS, sampler->descriptor, sizeof(sampler->descriptor));
982 }
983 }
984
985 static void
write_sampler_descriptor(uint32_t * dst,const VkDescriptorImageInfo * image_info)986 write_sampler_descriptor(uint32_t *dst, const VkDescriptorImageInfo *image_info)
987 {
988 TU_FROM_HANDLE(tu_sampler, sampler, image_info->sampler);
989
990 memcpy(dst, sampler->descriptor, sizeof(sampler->descriptor));
991 }
992
993 /* note: this is used with immutable samplers in push descriptors */
994 static void
write_sampler_push(uint32_t * dst,const struct tu_sampler * sampler)995 write_sampler_push(uint32_t *dst, const struct tu_sampler *sampler)
996 {
997 memcpy(dst, sampler->descriptor, sizeof(sampler->descriptor));
998 }
999
1000 void
tu_update_descriptor_sets(const struct tu_device * device,VkDescriptorSet dstSetOverride,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)1001 tu_update_descriptor_sets(const struct tu_device *device,
1002 VkDescriptorSet dstSetOverride,
1003 uint32_t descriptorWriteCount,
1004 const VkWriteDescriptorSet *pDescriptorWrites,
1005 uint32_t descriptorCopyCount,
1006 const VkCopyDescriptorSet *pDescriptorCopies)
1007 {
1008 uint32_t i, j;
1009 for (i = 0; i < descriptorWriteCount; i++) {
1010 const VkWriteDescriptorSet *writeset = &pDescriptorWrites[i];
1011 TU_FROM_HANDLE(tu_descriptor_set, set, dstSetOverride ?: writeset->dstSet);
1012 const struct tu_descriptor_set_binding_layout *binding_layout =
1013 set->layout->binding + writeset->dstBinding;
1014 uint32_t *ptr = set->mapped_ptr;
1015 if (writeset->descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
1016 writeset->descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
1017 ptr = set->dynamic_descriptors;
1018 ptr += binding_layout->dynamic_offset_offset / 4;
1019 } else {
1020 ptr = set->mapped_ptr;
1021 ptr += binding_layout->offset / 4;
1022 }
1023
1024 /* for immutable samplers with push descriptors: */
1025 const bool copy_immutable_samplers =
1026 dstSetOverride && binding_layout->immutable_samplers_offset;
1027 const struct tu_sampler *samplers =
1028 tu_immutable_samplers(set->layout, binding_layout);
1029
1030 ptr += (binding_layout->size / 4) * writeset->dstArrayElement;
1031 for (j = 0; j < writeset->descriptorCount; ++j) {
1032 switch(writeset->descriptorType) {
1033 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1034 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1035 write_ubo_descriptor(ptr, writeset->pBufferInfo + j);
1036 break;
1037 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
1038 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1039 write_buffer_descriptor(device, ptr, writeset->pBufferInfo + j);
1040 break;
1041 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1042 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1043 write_texel_buffer_descriptor(ptr, writeset->pTexelBufferView[j]);
1044 break;
1045 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1046 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
1047 write_image_descriptor(ptr, writeset->descriptorType, writeset->pImageInfo + j);
1048 break;
1049 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1050 write_combined_image_sampler_descriptor(ptr,
1051 writeset->descriptorType,
1052 writeset->pImageInfo + j,
1053 !binding_layout->immutable_samplers_offset);
1054
1055 if (copy_immutable_samplers)
1056 write_sampler_push(ptr + A6XX_TEX_CONST_DWORDS, &samplers[writeset->dstArrayElement + j]);
1057 break;
1058 case VK_DESCRIPTOR_TYPE_SAMPLER:
1059 if (!binding_layout->immutable_samplers_offset)
1060 write_sampler_descriptor(ptr, writeset->pImageInfo + j);
1061 else if (copy_immutable_samplers)
1062 write_sampler_push(ptr, &samplers[writeset->dstArrayElement + j]);
1063 break;
1064 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1065 /* nothing in descriptor set - framebuffer state is used instead */
1066 if (unlikely(device->instance->debug_flags & TU_DEBUG_DYNAMIC))
1067 write_image_descriptor(ptr, writeset->descriptorType, writeset->pImageInfo + j);
1068 break;
1069 default:
1070 unreachable("unimplemented descriptor type");
1071 break;
1072 }
1073 ptr += binding_layout->size / 4;
1074 }
1075 }
1076
1077 for (i = 0; i < descriptorCopyCount; i++) {
1078 const VkCopyDescriptorSet *copyset = &pDescriptorCopies[i];
1079 TU_FROM_HANDLE(tu_descriptor_set, src_set,
1080 copyset->srcSet);
1081 TU_FROM_HANDLE(tu_descriptor_set, dst_set,
1082 copyset->dstSet);
1083 const struct tu_descriptor_set_binding_layout *src_binding_layout =
1084 src_set->layout->binding + copyset->srcBinding;
1085 const struct tu_descriptor_set_binding_layout *dst_binding_layout =
1086 dst_set->layout->binding + copyset->dstBinding;
1087 uint32_t *src_ptr = src_set->mapped_ptr;
1088 uint32_t *dst_ptr = dst_set->mapped_ptr;
1089 if (src_binding_layout->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
1090 src_binding_layout->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
1091 src_ptr = src_set->dynamic_descriptors;
1092 dst_ptr = dst_set->dynamic_descriptors;
1093 src_ptr += src_binding_layout->dynamic_offset_offset / 4;
1094 dst_ptr += dst_binding_layout->dynamic_offset_offset / 4;
1095 } else {
1096 src_ptr = src_set->mapped_ptr;
1097 dst_ptr = dst_set->mapped_ptr;
1098 src_ptr += src_binding_layout->offset / 4;
1099 dst_ptr += dst_binding_layout->offset / 4;
1100 }
1101
1102 src_ptr += src_binding_layout->size * copyset->srcArrayElement / 4;
1103 dst_ptr += dst_binding_layout->size * copyset->dstArrayElement / 4;
1104
1105 /* In case of copies between mutable descriptor types
1106 * and non-mutable descriptor types.
1107 */
1108 uint32_t copy_size = MIN2(src_binding_layout->size, dst_binding_layout->size);
1109
1110 for (j = 0; j < copyset->descriptorCount; ++j) {
1111 memcpy(dst_ptr, src_ptr, copy_size);
1112
1113 src_ptr += src_binding_layout->size / 4;
1114 dst_ptr += dst_binding_layout->size / 4;
1115 }
1116 }
1117 }
1118
1119 VKAPI_ATTR void VKAPI_CALL
tu_UpdateDescriptorSets(VkDevice _device,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)1120 tu_UpdateDescriptorSets(VkDevice _device,
1121 uint32_t descriptorWriteCount,
1122 const VkWriteDescriptorSet *pDescriptorWrites,
1123 uint32_t descriptorCopyCount,
1124 const VkCopyDescriptorSet *pDescriptorCopies)
1125 {
1126 TU_FROM_HANDLE(tu_device, device, _device);
1127 tu_update_descriptor_sets(device, VK_NULL_HANDLE,
1128 descriptorWriteCount, pDescriptorWrites,
1129 descriptorCopyCount, pDescriptorCopies);
1130 }
1131
1132 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateDescriptorUpdateTemplate(VkDevice _device,const VkDescriptorUpdateTemplateCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorUpdateTemplate * pDescriptorUpdateTemplate)1133 tu_CreateDescriptorUpdateTemplate(
1134 VkDevice _device,
1135 const VkDescriptorUpdateTemplateCreateInfo *pCreateInfo,
1136 const VkAllocationCallbacks *pAllocator,
1137 VkDescriptorUpdateTemplate *pDescriptorUpdateTemplate)
1138 {
1139 TU_FROM_HANDLE(tu_device, device, _device);
1140 struct tu_descriptor_set_layout *set_layout = NULL;
1141 const uint32_t entry_count = pCreateInfo->descriptorUpdateEntryCount;
1142 const size_t size =
1143 sizeof(struct tu_descriptor_update_template) +
1144 sizeof(struct tu_descriptor_update_template_entry) * entry_count;
1145 struct tu_descriptor_update_template *templ;
1146
1147 templ = vk_object_alloc(&device->vk, pAllocator, size,
1148 VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE);
1149 if (!templ)
1150 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1151
1152 templ->entry_count = entry_count;
1153
1154 if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR) {
1155 TU_FROM_HANDLE(tu_pipeline_layout, pipeline_layout, pCreateInfo->pipelineLayout);
1156
1157 /* descriptorSetLayout should be ignored for push descriptors
1158 * and instead it refers to pipelineLayout and set.
1159 */
1160 assert(pCreateInfo->set < MAX_SETS);
1161 set_layout = pipeline_layout->set[pCreateInfo->set].layout;
1162
1163 templ->bind_point = pCreateInfo->pipelineBindPoint;
1164 } else {
1165 TU_FROM_HANDLE(tu_descriptor_set_layout, _set_layout,
1166 pCreateInfo->descriptorSetLayout);
1167 set_layout = _set_layout;
1168 }
1169
1170 for (uint32_t i = 0; i < entry_count; i++) {
1171 const VkDescriptorUpdateTemplateEntry *entry = &pCreateInfo->pDescriptorUpdateEntries[i];
1172
1173 const struct tu_descriptor_set_binding_layout *binding_layout =
1174 set_layout->binding + entry->dstBinding;
1175 uint32_t dst_offset, dst_stride;
1176 const struct tu_sampler *immutable_samplers = NULL;
1177
1178 /* dst_offset is an offset into dynamic_descriptors when the descriptor
1179 * is dynamic, and an offset into mapped_ptr otherwise.
1180 */
1181 switch (entry->descriptorType) {
1182 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1183 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
1184 dst_offset = binding_layout->dynamic_offset_offset / 4;
1185 break;
1186 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1187 case VK_DESCRIPTOR_TYPE_SAMPLER:
1188 if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR &&
1189 binding_layout->immutable_samplers_offset) {
1190 immutable_samplers =
1191 tu_immutable_samplers(set_layout, binding_layout) + entry->dstArrayElement;
1192 }
1193 FALLTHROUGH;
1194 default:
1195 dst_offset = binding_layout->offset / 4;
1196 }
1197
1198 dst_offset += (binding_layout->size * entry->dstArrayElement) / 4;
1199 dst_stride = binding_layout->size / 4;
1200
1201 templ->entry[i] = (struct tu_descriptor_update_template_entry) {
1202 .descriptor_type = entry->descriptorType,
1203 .descriptor_count = entry->descriptorCount,
1204 .src_offset = entry->offset,
1205 .src_stride = entry->stride,
1206 .dst_offset = dst_offset,
1207 .dst_stride = dst_stride,
1208 .has_sampler = !binding_layout->immutable_samplers_offset,
1209 .immutable_samplers = immutable_samplers,
1210 };
1211 }
1212
1213 *pDescriptorUpdateTemplate =
1214 tu_descriptor_update_template_to_handle(templ);
1215
1216 return VK_SUCCESS;
1217 }
1218
1219 VKAPI_ATTR void VKAPI_CALL
tu_DestroyDescriptorUpdateTemplate(VkDevice _device,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const VkAllocationCallbacks * pAllocator)1220 tu_DestroyDescriptorUpdateTemplate(
1221 VkDevice _device,
1222 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1223 const VkAllocationCallbacks *pAllocator)
1224 {
1225 TU_FROM_HANDLE(tu_device, device, _device);
1226 TU_FROM_HANDLE(tu_descriptor_update_template, templ,
1227 descriptorUpdateTemplate);
1228
1229 if (!templ)
1230 return;
1231
1232 vk_object_free(&device->vk, pAllocator, templ);
1233 }
1234
1235 void
tu_update_descriptor_set_with_template(const struct tu_device * device,struct tu_descriptor_set * set,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1236 tu_update_descriptor_set_with_template(
1237 const struct tu_device *device,
1238 struct tu_descriptor_set *set,
1239 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1240 const void *pData)
1241 {
1242 TU_FROM_HANDLE(tu_descriptor_update_template, templ,
1243 descriptorUpdateTemplate);
1244
1245 for (uint32_t i = 0; i < templ->entry_count; i++) {
1246 uint32_t *ptr = set->mapped_ptr;
1247 const void *src = ((const char *) pData) + templ->entry[i].src_offset;
1248 const struct tu_sampler *samplers = templ->entry[i].immutable_samplers;
1249
1250 ptr += templ->entry[i].dst_offset;
1251 unsigned dst_offset = templ->entry[i].dst_offset;
1252 for (unsigned j = 0; j < templ->entry[i].descriptor_count; ++j) {
1253 switch(templ->entry[i].descriptor_type) {
1254 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: {
1255 assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1256 write_ubo_descriptor(set->dynamic_descriptors + dst_offset, src);
1257 break;
1258 }
1259 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1260 write_ubo_descriptor(ptr, src);
1261 break;
1262 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1263 assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1264 write_buffer_descriptor(device, set->dynamic_descriptors + dst_offset, src);
1265 break;
1266 }
1267 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1268 write_buffer_descriptor(device, ptr, src);
1269 break;
1270 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1271 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1272 write_texel_buffer_descriptor(ptr, *(VkBufferView *) src);
1273 break;
1274 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1275 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: {
1276 write_image_descriptor(ptr, templ->entry[i].descriptor_type, src);
1277 break;
1278 }
1279 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1280 write_combined_image_sampler_descriptor(ptr,
1281 templ->entry[i].descriptor_type,
1282 src,
1283 templ->entry[i].has_sampler);
1284 if (samplers)
1285 write_sampler_push(ptr + A6XX_TEX_CONST_DWORDS, &samplers[j]);
1286 break;
1287 case VK_DESCRIPTOR_TYPE_SAMPLER:
1288 if (templ->entry[i].has_sampler)
1289 write_sampler_descriptor(ptr, src);
1290 else if (samplers)
1291 write_sampler_push(ptr, &samplers[j]);
1292 break;
1293 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1294 /* nothing in descriptor set - framebuffer state is used instead */
1295 if (unlikely(device->instance->debug_flags & TU_DEBUG_DYNAMIC))
1296 write_image_descriptor(ptr, templ->entry[i].descriptor_type, src);
1297 break;
1298 default:
1299 unreachable("unimplemented descriptor type");
1300 break;
1301 }
1302 src = (char *) src + templ->entry[i].src_stride;
1303 ptr += templ->entry[i].dst_stride;
1304 dst_offset += templ->entry[i].dst_stride;
1305 }
1306 }
1307 }
1308
1309 VKAPI_ATTR void VKAPI_CALL
tu_UpdateDescriptorSetWithTemplate(VkDevice _device,VkDescriptorSet descriptorSet,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1310 tu_UpdateDescriptorSetWithTemplate(
1311 VkDevice _device,
1312 VkDescriptorSet descriptorSet,
1313 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1314 const void *pData)
1315 {
1316 TU_FROM_HANDLE(tu_device, device, _device);
1317 TU_FROM_HANDLE(tu_descriptor_set, set, descriptorSet);
1318
1319 tu_update_descriptor_set_with_template(device, set, descriptorUpdateTemplate, pData);
1320 }
1321
1322 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateSamplerYcbcrConversion(VkDevice _device,const VkSamplerYcbcrConversionCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkSamplerYcbcrConversion * pYcbcrConversion)1323 tu_CreateSamplerYcbcrConversion(
1324 VkDevice _device,
1325 const VkSamplerYcbcrConversionCreateInfo *pCreateInfo,
1326 const VkAllocationCallbacks *pAllocator,
1327 VkSamplerYcbcrConversion *pYcbcrConversion)
1328 {
1329 TU_FROM_HANDLE(tu_device, device, _device);
1330 struct tu_sampler_ycbcr_conversion *conversion;
1331
1332 conversion = vk_object_alloc(&device->vk, pAllocator, sizeof(*conversion),
1333 VK_OBJECT_TYPE_SAMPLER_YCBCR_CONVERSION);
1334 if (!conversion)
1335 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1336
1337 conversion->format = pCreateInfo->format;
1338 conversion->ycbcr_model = pCreateInfo->ycbcrModel;
1339 conversion->ycbcr_range = pCreateInfo->ycbcrRange;
1340 conversion->components = pCreateInfo->components;
1341 conversion->chroma_offsets[0] = pCreateInfo->xChromaOffset;
1342 conversion->chroma_offsets[1] = pCreateInfo->yChromaOffset;
1343 conversion->chroma_filter = pCreateInfo->chromaFilter;
1344
1345 *pYcbcrConversion = tu_sampler_ycbcr_conversion_to_handle(conversion);
1346 return VK_SUCCESS;
1347 }
1348
1349 VKAPI_ATTR void VKAPI_CALL
tu_DestroySamplerYcbcrConversion(VkDevice _device,VkSamplerYcbcrConversion ycbcrConversion,const VkAllocationCallbacks * pAllocator)1350 tu_DestroySamplerYcbcrConversion(VkDevice _device,
1351 VkSamplerYcbcrConversion ycbcrConversion,
1352 const VkAllocationCallbacks *pAllocator)
1353 {
1354 TU_FROM_HANDLE(tu_device, device, _device);
1355 TU_FROM_HANDLE(tu_sampler_ycbcr_conversion, ycbcr_conversion, ycbcrConversion);
1356
1357 if (!ycbcr_conversion)
1358 return;
1359
1360 vk_object_free(&device->vk, pAllocator, ycbcr_conversion);
1361 }
1362