1 /*
2 * Copyright © 2022 Collabora, LTD
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "vk_pipeline.h"
25
26 #include "vk_alloc.h"
27 #include "vk_common_entrypoints.h"
28 #include "vk_command_buffer.h"
29 #include "vk_descriptor_set_layout.h"
30 #include "vk_device.h"
31 #include "vk_graphics_state.h"
32 #include "vk_log.h"
33 #include "vk_nir.h"
34 #include "vk_physical_device.h"
35 #include "vk_pipeline_layout.h"
36 #include "vk_shader.h"
37 #include "vk_shader_module.h"
38 #include "vk_util.h"
39
40 #include "nir_serialize.h"
41
42 #include "util/mesa-sha1.h"
43
44 bool
vk_pipeline_shader_stage_is_null(const VkPipelineShaderStageCreateInfo * info)45 vk_pipeline_shader_stage_is_null(const VkPipelineShaderStageCreateInfo *info)
46 {
47 if (info->module != VK_NULL_HANDLE)
48 return false;
49
50 vk_foreach_struct_const(ext, info->pNext) {
51 if (ext->sType == VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO ||
52 ext->sType == VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_MODULE_IDENTIFIER_CREATE_INFO_EXT)
53 return false;
54 }
55
56 return true;
57 }
58
59 bool
vk_pipeline_shader_stage_has_identifier(const VkPipelineShaderStageCreateInfo * info)60 vk_pipeline_shader_stage_has_identifier(const VkPipelineShaderStageCreateInfo *info)
61 {
62 const VkPipelineShaderStageModuleIdentifierCreateInfoEXT *id_info =
63 vk_find_struct_const(info->pNext, PIPELINE_SHADER_STAGE_MODULE_IDENTIFIER_CREATE_INFO_EXT);
64
65 return id_info && id_info->identifierSize != 0;
66 }
67
68 static nir_shader *
get_builtin_nir(const VkPipelineShaderStageCreateInfo * info)69 get_builtin_nir(const VkPipelineShaderStageCreateInfo *info)
70 {
71 VK_FROM_HANDLE(vk_shader_module, module, info->module);
72
73 nir_shader *nir = NULL;
74 if (module != NULL) {
75 nir = module->nir;
76 } else {
77 const VkPipelineShaderStageNirCreateInfoMESA *nir_info =
78 vk_find_struct_const(info->pNext, PIPELINE_SHADER_STAGE_NIR_CREATE_INFO_MESA);
79 if (nir_info != NULL)
80 nir = nir_info->nir;
81 }
82
83 if (nir == NULL)
84 return NULL;
85
86 assert(nir->info.stage == vk_to_mesa_shader_stage(info->stage));
87 ASSERTED nir_function_impl *entrypoint = nir_shader_get_entrypoint(nir);
88 assert(strcmp(entrypoint->function->name, info->pName) == 0);
89 assert(info->pSpecializationInfo == NULL);
90
91 return nir;
92 }
93
94 static uint32_t
get_required_subgroup_size(const void * info_pNext)95 get_required_subgroup_size(const void *info_pNext)
96 {
97 const VkPipelineShaderStageRequiredSubgroupSizeCreateInfo *rss_info =
98 vk_find_struct_const(info_pNext,
99 PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO);
100 return rss_info != NULL ? rss_info->requiredSubgroupSize : 0;
101 }
102
103 enum gl_subgroup_size
vk_get_subgroup_size(uint32_t spirv_version,gl_shader_stage stage,const void * info_pNext,bool allow_varying,bool require_full)104 vk_get_subgroup_size(uint32_t spirv_version,
105 gl_shader_stage stage,
106 const void *info_pNext,
107 bool allow_varying,
108 bool require_full)
109 {
110 uint32_t req_subgroup_size = get_required_subgroup_size(info_pNext);
111 if (req_subgroup_size > 0) {
112 assert(util_is_power_of_two_nonzero(req_subgroup_size));
113 assert(req_subgroup_size >= 4 && req_subgroup_size <= 128);
114 return req_subgroup_size;
115 } else if (allow_varying || spirv_version >= 0x10600) {
116 /* Starting with SPIR-V 1.6, varying subgroup size the default */
117 return SUBGROUP_SIZE_VARYING;
118 } else if (require_full) {
119 assert(stage == MESA_SHADER_COMPUTE ||
120 stage == MESA_SHADER_MESH ||
121 stage == MESA_SHADER_TASK);
122 return SUBGROUP_SIZE_FULL_SUBGROUPS;
123 } else {
124 return SUBGROUP_SIZE_API_CONSTANT;
125 }
126 }
127
128 VkResult
vk_pipeline_shader_stage_to_nir(struct vk_device * device,VkPipelineCreateFlags2KHR pipeline_flags,const VkPipelineShaderStageCreateInfo * info,const struct spirv_to_nir_options * spirv_options,const struct nir_shader_compiler_options * nir_options,void * mem_ctx,nir_shader ** nir_out)129 vk_pipeline_shader_stage_to_nir(struct vk_device *device,
130 VkPipelineCreateFlags2KHR pipeline_flags,
131 const VkPipelineShaderStageCreateInfo *info,
132 const struct spirv_to_nir_options *spirv_options,
133 const struct nir_shader_compiler_options *nir_options,
134 void *mem_ctx, nir_shader **nir_out)
135 {
136 VK_FROM_HANDLE(vk_shader_module, module, info->module);
137 const gl_shader_stage stage = vk_to_mesa_shader_stage(info->stage);
138
139 assert(info->sType == VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO);
140
141 nir_shader *builtin_nir = get_builtin_nir(info);
142 if (builtin_nir != NULL) {
143 nir_validate_shader(builtin_nir, "internal shader");
144
145 nir_shader *clone = nir_shader_clone(mem_ctx, builtin_nir);
146 if (clone == NULL)
147 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
148
149 assert(clone->options == NULL || clone->options == nir_options);
150 clone->options = nir_options;
151
152 *nir_out = clone;
153 return VK_SUCCESS;
154 }
155
156 const uint32_t *spirv_data;
157 uint32_t spirv_size;
158 if (module != NULL) {
159 spirv_data = (uint32_t *)module->data;
160 spirv_size = module->size;
161 } else {
162 const VkShaderModuleCreateInfo *minfo =
163 vk_find_struct_const(info->pNext, SHADER_MODULE_CREATE_INFO);
164 if (unlikely(minfo == NULL)) {
165 return vk_errorf(device, VK_ERROR_UNKNOWN,
166 "No shader module provided");
167 }
168 spirv_data = minfo->pCode;
169 spirv_size = minfo->codeSize;
170 }
171
172 enum gl_subgroup_size subgroup_size = vk_get_subgroup_size(
173 vk_spirv_version(spirv_data, spirv_size),
174 stage, info->pNext,
175 info->flags & VK_PIPELINE_SHADER_STAGE_CREATE_ALLOW_VARYING_SUBGROUP_SIZE_BIT,
176 info->flags & VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT);
177
178 nir_shader *nir = vk_spirv_to_nir(device, spirv_data, spirv_size, stage,
179 info->pName, subgroup_size,
180 info->pSpecializationInfo,
181 spirv_options, nir_options,
182 false /* internal */,
183 mem_ctx);
184 if (nir == NULL)
185 return vk_errorf(device, VK_ERROR_UNKNOWN, "spirv_to_nir failed");
186
187 if (pipeline_flags & VK_PIPELINE_CREATE_2_VIEW_INDEX_FROM_DEVICE_INDEX_BIT_KHR)
188 NIR_PASS(_, nir, nir_lower_view_index_to_device_index);
189
190 *nir_out = nir;
191
192 return VK_SUCCESS;
193 }
194
195 void
vk_pipeline_hash_shader_stage(VkPipelineCreateFlags2KHR pipeline_flags,const VkPipelineShaderStageCreateInfo * info,const struct vk_pipeline_robustness_state * rstate,unsigned char * stage_sha1)196 vk_pipeline_hash_shader_stage(VkPipelineCreateFlags2KHR pipeline_flags,
197 const VkPipelineShaderStageCreateInfo *info,
198 const struct vk_pipeline_robustness_state *rstate,
199 unsigned char *stage_sha1)
200 {
201 VK_FROM_HANDLE(vk_shader_module, module, info->module);
202
203 const nir_shader *builtin_nir = get_builtin_nir(info);
204 if (builtin_nir != NULL) {
205 /* Internal NIR module: serialize and hash the NIR shader.
206 * We don't need to hash other info fields since they should match the
207 * NIR data.
208 */
209 struct blob blob;
210
211 blob_init(&blob);
212 nir_serialize(&blob, builtin_nir, false);
213 assert(!blob.out_of_memory);
214 _mesa_sha1_compute(blob.data, blob.size, stage_sha1);
215 blob_finish(&blob);
216 return;
217 }
218
219 const VkShaderModuleCreateInfo *minfo =
220 vk_find_struct_const(info->pNext, SHADER_MODULE_CREATE_INFO);
221 const VkPipelineShaderStageModuleIdentifierCreateInfoEXT *iinfo =
222 vk_find_struct_const(info->pNext, PIPELINE_SHADER_STAGE_MODULE_IDENTIFIER_CREATE_INFO_EXT);
223
224 struct mesa_sha1 ctx;
225
226 _mesa_sha1_init(&ctx);
227
228 /* We only care about one of the pipeline flags */
229 pipeline_flags &= VK_PIPELINE_CREATE_2_VIEW_INDEX_FROM_DEVICE_INDEX_BIT_KHR;
230 _mesa_sha1_update(&ctx, &pipeline_flags, sizeof(pipeline_flags));
231
232 _mesa_sha1_update(&ctx, &info->flags, sizeof(info->flags));
233
234 assert(util_bitcount(info->stage) == 1);
235 _mesa_sha1_update(&ctx, &info->stage, sizeof(info->stage));
236
237 if (module) {
238 _mesa_sha1_update(&ctx, module->hash, sizeof(module->hash));
239 } else if (minfo) {
240 blake3_hash spirv_hash;
241
242 _mesa_blake3_compute(minfo->pCode, minfo->codeSize, spirv_hash);
243 _mesa_sha1_update(&ctx, spirv_hash, sizeof(spirv_hash));
244 } else {
245 /* It is legal to pass in arbitrary identifiers as long as they don't exceed
246 * the limit. Shaders with bogus identifiers are more or less guaranteed to fail. */
247 assert(iinfo);
248 assert(iinfo->identifierSize <= VK_MAX_SHADER_MODULE_IDENTIFIER_SIZE_EXT);
249 _mesa_sha1_update(&ctx, iinfo->pIdentifier, iinfo->identifierSize);
250 }
251
252 if (rstate) {
253 _mesa_sha1_update(&ctx, &rstate->storage_buffers, sizeof(rstate->storage_buffers));
254 _mesa_sha1_update(&ctx, &rstate->uniform_buffers, sizeof(rstate->uniform_buffers));
255 _mesa_sha1_update(&ctx, &rstate->vertex_inputs, sizeof(rstate->vertex_inputs));
256 _mesa_sha1_update(&ctx, &rstate->images, sizeof(rstate->images));
257 }
258
259 _mesa_sha1_update(&ctx, info->pName, strlen(info->pName));
260
261 if (info->pSpecializationInfo) {
262 _mesa_sha1_update(&ctx, info->pSpecializationInfo->pMapEntries,
263 info->pSpecializationInfo->mapEntryCount *
264 sizeof(*info->pSpecializationInfo->pMapEntries));
265 _mesa_sha1_update(&ctx, info->pSpecializationInfo->pData,
266 info->pSpecializationInfo->dataSize);
267 }
268
269 uint32_t req_subgroup_size = get_required_subgroup_size(info);
270 _mesa_sha1_update(&ctx, &req_subgroup_size, sizeof(req_subgroup_size));
271
272 _mesa_sha1_final(&ctx, stage_sha1);
273 }
274
275 static VkPipelineRobustnessBufferBehaviorEXT
vk_device_default_robust_buffer_behavior(const struct vk_device * device)276 vk_device_default_robust_buffer_behavior(const struct vk_device *device)
277 {
278 if (device->enabled_features.robustBufferAccess2) {
279 return VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_ROBUST_BUFFER_ACCESS_2_EXT;
280 } else if (device->enabled_features.robustBufferAccess) {
281 return VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_ROBUST_BUFFER_ACCESS_EXT;
282 } else {
283 return VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_DISABLED_EXT;
284 }
285 }
286
287 static VkPipelineRobustnessImageBehaviorEXT
vk_device_default_robust_image_behavior(const struct vk_device * device)288 vk_device_default_robust_image_behavior(const struct vk_device *device)
289 {
290 if (device->enabled_features.robustImageAccess2) {
291 return VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_ROBUST_IMAGE_ACCESS_2_EXT;
292 } else if (device->enabled_features.robustImageAccess) {
293 return VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_ROBUST_IMAGE_ACCESS_EXT;
294 } else {
295 return VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_DISABLED_EXT;
296 }
297 }
298
299 void
vk_pipeline_robustness_state_fill(const struct vk_device * device,struct vk_pipeline_robustness_state * rs,const void * pipeline_pNext,const void * shader_stage_pNext)300 vk_pipeline_robustness_state_fill(const struct vk_device *device,
301 struct vk_pipeline_robustness_state *rs,
302 const void *pipeline_pNext,
303 const void *shader_stage_pNext)
304 {
305 rs->uniform_buffers = VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_DEVICE_DEFAULT_EXT;
306 rs->storage_buffers = VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_DEVICE_DEFAULT_EXT;
307 rs->vertex_inputs = VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_DEVICE_DEFAULT_EXT;
308 rs->images = VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_DEVICE_DEFAULT_EXT;
309 rs->null_uniform_buffer_descriptor = device->enabled_features.nullDescriptor;
310 rs->null_storage_buffer_descriptor = device->enabled_features.nullDescriptor;
311
312 const VkPipelineRobustnessCreateInfoEXT *shader_info =
313 vk_find_struct_const(shader_stage_pNext,
314 PIPELINE_ROBUSTNESS_CREATE_INFO_EXT);
315 if (shader_info) {
316 rs->storage_buffers = shader_info->storageBuffers;
317 rs->uniform_buffers = shader_info->uniformBuffers;
318 rs->vertex_inputs = shader_info->vertexInputs;
319 rs->images = shader_info->images;
320 } else {
321 const VkPipelineRobustnessCreateInfoEXT *pipeline_info =
322 vk_find_struct_const(pipeline_pNext,
323 PIPELINE_ROBUSTNESS_CREATE_INFO_EXT);
324 if (pipeline_info) {
325 rs->storage_buffers = pipeline_info->storageBuffers;
326 rs->uniform_buffers = pipeline_info->uniformBuffers;
327 rs->vertex_inputs = pipeline_info->vertexInputs;
328 rs->images = pipeline_info->images;
329 }
330 }
331
332 if (rs->storage_buffers ==
333 VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_DEVICE_DEFAULT_EXT)
334 rs->storage_buffers = vk_device_default_robust_buffer_behavior(device);
335
336 if (rs->uniform_buffers ==
337 VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_DEVICE_DEFAULT_EXT)
338 rs->uniform_buffers = vk_device_default_robust_buffer_behavior(device);
339
340 if (rs->vertex_inputs ==
341 VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_DEVICE_DEFAULT_EXT)
342 rs->vertex_inputs = vk_device_default_robust_buffer_behavior(device);
343
344 if (rs->images == VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_DEVICE_DEFAULT_EXT)
345 rs->images = vk_device_default_robust_image_behavior(device);
346 }
347
348 void *
vk_pipeline_zalloc(struct vk_device * device,const struct vk_pipeline_ops * ops,VkPipelineBindPoint bind_point,VkPipelineCreateFlags2KHR flags,const VkAllocationCallbacks * alloc,size_t size)349 vk_pipeline_zalloc(struct vk_device *device,
350 const struct vk_pipeline_ops *ops,
351 VkPipelineBindPoint bind_point,
352 VkPipelineCreateFlags2KHR flags,
353 const VkAllocationCallbacks *alloc,
354 size_t size)
355 {
356 struct vk_pipeline *pipeline;
357
358 pipeline = vk_object_zalloc(device, alloc, size, VK_OBJECT_TYPE_PIPELINE);
359 if (pipeline == NULL)
360 return NULL;
361
362 pipeline->ops = ops;
363 pipeline->bind_point = bind_point;
364 pipeline->flags = flags;
365
366 return pipeline;
367 }
368
369 void
vk_pipeline_free(struct vk_device * device,const VkAllocationCallbacks * alloc,struct vk_pipeline * pipeline)370 vk_pipeline_free(struct vk_device *device,
371 const VkAllocationCallbacks *alloc,
372 struct vk_pipeline *pipeline)
373 {
374 vk_object_free(device, alloc, &pipeline->base);
375 }
376
377 VKAPI_ATTR void VKAPI_CALL
vk_common_DestroyPipeline(VkDevice _device,VkPipeline _pipeline,const VkAllocationCallbacks * pAllocator)378 vk_common_DestroyPipeline(VkDevice _device,
379 VkPipeline _pipeline,
380 const VkAllocationCallbacks *pAllocator)
381 {
382 VK_FROM_HANDLE(vk_device, device, _device);
383 VK_FROM_HANDLE(vk_pipeline, pipeline, _pipeline);
384
385 if (pipeline == NULL)
386 return;
387
388 pipeline->ops->destroy(device, pipeline, pAllocator);
389 }
390
391 VKAPI_ATTR VkResult VKAPI_CALL
vk_common_GetPipelineExecutablePropertiesKHR(VkDevice _device,const VkPipelineInfoKHR * pPipelineInfo,uint32_t * pExecutableCount,VkPipelineExecutablePropertiesKHR * pProperties)392 vk_common_GetPipelineExecutablePropertiesKHR(
393 VkDevice _device,
394 const VkPipelineInfoKHR *pPipelineInfo,
395 uint32_t *pExecutableCount,
396 VkPipelineExecutablePropertiesKHR *pProperties)
397 {
398 VK_FROM_HANDLE(vk_device, device, _device);
399 VK_FROM_HANDLE(vk_pipeline, pipeline, pPipelineInfo->pipeline);
400
401 return pipeline->ops->get_executable_properties(device, pipeline,
402 pExecutableCount,
403 pProperties);
404 }
405
406 VKAPI_ATTR VkResult VKAPI_CALL
vk_common_GetPipelineExecutableStatisticsKHR(VkDevice _device,const VkPipelineExecutableInfoKHR * pExecutableInfo,uint32_t * pStatisticCount,VkPipelineExecutableStatisticKHR * pStatistics)407 vk_common_GetPipelineExecutableStatisticsKHR(
408 VkDevice _device,
409 const VkPipelineExecutableInfoKHR *pExecutableInfo,
410 uint32_t *pStatisticCount,
411 VkPipelineExecutableStatisticKHR *pStatistics)
412 {
413 VK_FROM_HANDLE(vk_device, device, _device);
414 VK_FROM_HANDLE(vk_pipeline, pipeline, pExecutableInfo->pipeline);
415
416 return pipeline->ops->get_executable_statistics(device, pipeline,
417 pExecutableInfo->executableIndex,
418 pStatisticCount, pStatistics);
419 }
420
421 VKAPI_ATTR VkResult VKAPI_CALL
vk_common_GetPipelineExecutableInternalRepresentationsKHR(VkDevice _device,const VkPipelineExecutableInfoKHR * pExecutableInfo,uint32_t * pInternalRepresentationCount,VkPipelineExecutableInternalRepresentationKHR * pInternalRepresentations)422 vk_common_GetPipelineExecutableInternalRepresentationsKHR(
423 VkDevice _device,
424 const VkPipelineExecutableInfoKHR *pExecutableInfo,
425 uint32_t *pInternalRepresentationCount,
426 VkPipelineExecutableInternalRepresentationKHR* pInternalRepresentations)
427 {
428 VK_FROM_HANDLE(vk_device, device, _device);
429 VK_FROM_HANDLE(vk_pipeline, pipeline, pExecutableInfo->pipeline);
430
431 return pipeline->ops->get_internal_representations(device, pipeline,
432 pExecutableInfo->executableIndex,
433 pInternalRepresentationCount,
434 pInternalRepresentations);
435 }
436
437 VKAPI_ATTR void VKAPI_CALL
vk_common_CmdBindPipeline(VkCommandBuffer commandBuffer,VkPipelineBindPoint pipelineBindPoint,VkPipeline _pipeline)438 vk_common_CmdBindPipeline(VkCommandBuffer commandBuffer,
439 VkPipelineBindPoint pipelineBindPoint,
440 VkPipeline _pipeline)
441 {
442 VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
443 VK_FROM_HANDLE(vk_pipeline, pipeline, _pipeline);
444
445 assert(pipeline->bind_point == pipelineBindPoint);
446
447 pipeline->ops->cmd_bind(cmd_buffer, pipeline);
448 }
449
450 static const struct vk_pipeline_cache_object_ops pipeline_shader_cache_ops;
451
452 static struct vk_shader *
vk_shader_from_cache_obj(struct vk_pipeline_cache_object * object)453 vk_shader_from_cache_obj(struct vk_pipeline_cache_object *object)
454 {
455 assert(object->ops == &pipeline_shader_cache_ops);
456 return container_of(object, struct vk_shader, pipeline.cache_obj);
457 }
458
459 static bool
vk_pipeline_shader_serialize(struct vk_pipeline_cache_object * object,struct blob * blob)460 vk_pipeline_shader_serialize(struct vk_pipeline_cache_object *object,
461 struct blob *blob)
462 {
463 struct vk_shader *shader = vk_shader_from_cache_obj(object);
464 struct vk_device *device = shader->base.device;
465
466 return shader->ops->serialize(device, shader, blob);
467 }
468
469 static void
vk_shader_init_cache_obj(struct vk_device * device,struct vk_shader * shader,const void * key_data,size_t key_size)470 vk_shader_init_cache_obj(struct vk_device *device, struct vk_shader *shader,
471 const void *key_data, size_t key_size)
472 {
473 assert(key_size == sizeof(shader->pipeline.cache_key));
474 memcpy(&shader->pipeline.cache_key, key_data,
475 sizeof(shader->pipeline.cache_key));
476
477 vk_pipeline_cache_object_init(device, &shader->pipeline.cache_obj,
478 &pipeline_shader_cache_ops,
479 &shader->pipeline.cache_key,
480 sizeof(shader->pipeline.cache_key));
481 }
482
483 static struct vk_pipeline_cache_object *
vk_pipeline_shader_deserialize(struct vk_pipeline_cache * cache,const void * key_data,size_t key_size,struct blob_reader * blob)484 vk_pipeline_shader_deserialize(struct vk_pipeline_cache *cache,
485 const void *key_data, size_t key_size,
486 struct blob_reader *blob)
487 {
488 struct vk_device *device = cache->base.device;
489 const struct vk_device_shader_ops *ops = device->shader_ops;
490
491 /* TODO: Do we really want to always use the latest version? */
492 const uint32_t version = device->physical->properties.shaderBinaryVersion;
493
494 struct vk_shader *shader;
495 VkResult result = ops->deserialize(device, blob, version,
496 &device->alloc, &shader);
497 if (result != VK_SUCCESS) {
498 assert(result == VK_ERROR_OUT_OF_HOST_MEMORY);
499 return NULL;
500 }
501
502 vk_shader_init_cache_obj(device, shader, key_data, key_size);
503
504 return &shader->pipeline.cache_obj;
505 }
506
507 static void
vk_pipeline_shader_destroy(struct vk_device * device,struct vk_pipeline_cache_object * object)508 vk_pipeline_shader_destroy(struct vk_device *device,
509 struct vk_pipeline_cache_object *object)
510 {
511 struct vk_shader *shader = vk_shader_from_cache_obj(object);
512 assert(shader->base.device == device);
513
514 vk_shader_destroy(device, shader, &device->alloc);
515 }
516
517 static const struct vk_pipeline_cache_object_ops pipeline_shader_cache_ops = {
518 .serialize = vk_pipeline_shader_serialize,
519 .deserialize = vk_pipeline_shader_deserialize,
520 .destroy = vk_pipeline_shader_destroy,
521 };
522
523 static struct vk_shader *
vk_shader_ref(struct vk_shader * shader)524 vk_shader_ref(struct vk_shader *shader)
525 {
526 vk_pipeline_cache_object_ref(&shader->pipeline.cache_obj);
527 return shader;
528 }
529
530 static void
vk_shader_unref(struct vk_device * device,struct vk_shader * shader)531 vk_shader_unref(struct vk_device *device, struct vk_shader *shader)
532 {
533 vk_pipeline_cache_object_unref(device, &shader->pipeline.cache_obj);
534 }
535
536 PRAGMA_DIAGNOSTIC_PUSH
537 PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
538 struct vk_pipeline_tess_info {
539 unsigned tcs_vertices_out : 8;
540 unsigned primitive_mode : 2; /* tess_primitive_mode */
541 unsigned spacing : 2; /* gl_tess_spacing */
542 unsigned ccw : 1;
543 unsigned point_mode : 1;
544 unsigned _pad : 18;
545 };
546 PRAGMA_DIAGNOSTIC_POP
547 static_assert(sizeof(struct vk_pipeline_tess_info) == 4,
548 "This struct has no holes");
549
550 static void
vk_pipeline_gather_nir_tess_info(const nir_shader * nir,struct vk_pipeline_tess_info * info)551 vk_pipeline_gather_nir_tess_info(const nir_shader *nir,
552 struct vk_pipeline_tess_info *info)
553 {
554 info->tcs_vertices_out = nir->info.tess.tcs_vertices_out;
555 info->primitive_mode = nir->info.tess._primitive_mode;
556 info->spacing = nir->info.tess.spacing;
557 info->ccw = nir->info.tess.ccw;
558 info->point_mode = nir->info.tess.point_mode;
559 }
560
561 static void
vk_pipeline_replace_nir_tess_info(nir_shader * nir,const struct vk_pipeline_tess_info * info)562 vk_pipeline_replace_nir_tess_info(nir_shader *nir,
563 const struct vk_pipeline_tess_info *info)
564 {
565 nir->info.tess.tcs_vertices_out = info->tcs_vertices_out;
566 nir->info.tess._primitive_mode = info->primitive_mode;
567 nir->info.tess.spacing = info->spacing;
568 nir->info.tess.ccw = info->ccw;
569 nir->info.tess.point_mode = info->point_mode;
570 }
571
572 static void
vk_pipeline_tess_info_merge(struct vk_pipeline_tess_info * dst,const struct vk_pipeline_tess_info * src)573 vk_pipeline_tess_info_merge(struct vk_pipeline_tess_info *dst,
574 const struct vk_pipeline_tess_info *src)
575 {
576 /* The Vulkan 1.0.38 spec, section 21.1 Tessellator says:
577 *
578 * "PointMode. Controls generation of points rather than triangles
579 * or lines. This functionality defaults to disabled, and is
580 * enabled if either shader stage includes the execution mode.
581 *
582 * and about Triangles, Quads, IsoLines, VertexOrderCw, VertexOrderCcw,
583 * PointMode, SpacingEqual, SpacingFractionalEven, SpacingFractionalOdd,
584 * and OutputVertices, it says:
585 *
586 * "One mode must be set in at least one of the tessellation
587 * shader stages."
588 *
589 * So, the fields can be set in either the TCS or TES, but they must
590 * agree if set in both.
591 */
592 assert(dst->tcs_vertices_out == 0 ||
593 src->tcs_vertices_out == 0 ||
594 dst->tcs_vertices_out == src->tcs_vertices_out);
595 dst->tcs_vertices_out |= src->tcs_vertices_out;
596
597 static_assert(TESS_SPACING_UNSPECIFIED == 0, "");
598 assert(dst->spacing == TESS_SPACING_UNSPECIFIED ||
599 src->spacing == TESS_SPACING_UNSPECIFIED ||
600 dst->spacing == src->spacing);
601 dst->spacing |= src->spacing;
602
603 static_assert(TESS_PRIMITIVE_UNSPECIFIED == 0, "");
604 assert(dst->primitive_mode == TESS_PRIMITIVE_UNSPECIFIED ||
605 src->primitive_mode == TESS_PRIMITIVE_UNSPECIFIED ||
606 dst->primitive_mode == src->primitive_mode);
607 dst->primitive_mode |= src->primitive_mode;
608 dst->ccw |= src->ccw;
609 dst->point_mode |= src->point_mode;
610 }
611
612 struct vk_pipeline_precomp_shader {
613 struct vk_pipeline_cache_object cache_obj;
614
615 /* Key for this cache_obj in the pipeline cache.
616 *
617 * This is always the output of vk_pipeline_hash_shader_stage() so it must
618 * be a SHA1 hash.
619 */
620 uint8_t cache_key[SHA1_DIGEST_LENGTH];
621
622 gl_shader_stage stage;
623
624 struct vk_pipeline_robustness_state rs;
625
626 /* Tessellation info if the shader is a tessellation shader */
627 struct vk_pipeline_tess_info tess;
628
629 /* Hash of the vk_pipeline_precomp_shader
630 *
631 * This is the hash of the final compiled NIR together with tess info and
632 * robustness state. It's used as a key for final binary lookups. By
633 * having this as a separate key, we can de-duplicate cases where you have
634 * different SPIR-V or specialization constants but end up compiling the
635 * same NIR shader in the end anyway.
636 */
637 blake3_hash blake3;
638
639 struct blob nir_blob;
640 };
641
642 static struct vk_pipeline_precomp_shader *
vk_pipeline_precomp_shader_ref(struct vk_pipeline_precomp_shader * shader)643 vk_pipeline_precomp_shader_ref(struct vk_pipeline_precomp_shader *shader)
644 {
645 vk_pipeline_cache_object_ref(&shader->cache_obj);
646 return shader;
647 }
648
649 static void
vk_pipeline_precomp_shader_unref(struct vk_device * device,struct vk_pipeline_precomp_shader * shader)650 vk_pipeline_precomp_shader_unref(struct vk_device *device,
651 struct vk_pipeline_precomp_shader *shader)
652 {
653 vk_pipeline_cache_object_unref(device, &shader->cache_obj);
654 }
655
656 static const struct vk_pipeline_cache_object_ops pipeline_precomp_shader_cache_ops;
657
658 static struct vk_pipeline_precomp_shader *
vk_pipeline_precomp_shader_from_cache_obj(struct vk_pipeline_cache_object * obj)659 vk_pipeline_precomp_shader_from_cache_obj(struct vk_pipeline_cache_object *obj)
660 {
661 assert(obj->ops == & pipeline_precomp_shader_cache_ops);
662 return container_of(obj, struct vk_pipeline_precomp_shader, cache_obj);
663 }
664
665 static struct vk_pipeline_precomp_shader *
vk_pipeline_precomp_shader_create(struct vk_device * device,const void * key_data,size_t key_size,const struct vk_pipeline_robustness_state * rs,nir_shader * nir)666 vk_pipeline_precomp_shader_create(struct vk_device *device,
667 const void *key_data, size_t key_size,
668 const struct vk_pipeline_robustness_state *rs,
669 nir_shader *nir)
670 {
671 struct blob blob;
672 blob_init(&blob);
673
674 nir_serialize(&blob, nir, false);
675
676 if (blob.out_of_memory)
677 goto fail_blob;
678
679 struct vk_pipeline_precomp_shader *shader =
680 vk_zalloc(&device->alloc, sizeof(*shader), 8,
681 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
682 if (shader == NULL)
683 goto fail_blob;
684
685 assert(sizeof(shader->cache_key) == key_size);
686 memcpy(shader->cache_key, key_data, sizeof(shader->cache_key));
687
688 vk_pipeline_cache_object_init(device, &shader->cache_obj,
689 &pipeline_precomp_shader_cache_ops,
690 shader->cache_key,
691 sizeof(shader->cache_key));
692
693 shader->stage = nir->info.stage;
694 shader->rs = *rs;
695
696 vk_pipeline_gather_nir_tess_info(nir, &shader->tess);
697
698 struct mesa_blake3 blake3_ctx;
699 _mesa_blake3_init(&blake3_ctx);
700 _mesa_blake3_update(&blake3_ctx, rs, sizeof(*rs));
701 _mesa_blake3_update(&blake3_ctx, blob.data, blob.size);
702 _mesa_blake3_final(&blake3_ctx, shader->blake3);
703
704 shader->nir_blob = blob;
705
706 return shader;
707
708 fail_blob:
709 blob_finish(&blob);
710
711 return NULL;
712 }
713
714 static bool
vk_pipeline_precomp_shader_serialize(struct vk_pipeline_cache_object * obj,struct blob * blob)715 vk_pipeline_precomp_shader_serialize(struct vk_pipeline_cache_object *obj,
716 struct blob *blob)
717 {
718 struct vk_pipeline_precomp_shader *shader =
719 vk_pipeline_precomp_shader_from_cache_obj(obj);
720
721 blob_write_uint32(blob, shader->stage);
722 blob_write_bytes(blob, &shader->rs, sizeof(shader->rs));
723 blob_write_bytes(blob, &shader->tess, sizeof(shader->tess));
724 blob_write_bytes(blob, shader->blake3, sizeof(shader->blake3));
725 blob_write_uint64(blob, shader->nir_blob.size);
726 blob_write_bytes(blob, shader->nir_blob.data, shader->nir_blob.size);
727
728 return !blob->out_of_memory;
729 }
730
731 static struct vk_pipeline_cache_object *
vk_pipeline_precomp_shader_deserialize(struct vk_pipeline_cache * cache,const void * key_data,size_t key_size,struct blob_reader * blob)732 vk_pipeline_precomp_shader_deserialize(struct vk_pipeline_cache *cache,
733 const void *key_data, size_t key_size,
734 struct blob_reader *blob)
735 {
736 struct vk_device *device = cache->base.device;
737
738 struct vk_pipeline_precomp_shader *shader =
739 vk_zalloc(&device->alloc, sizeof(*shader), 8,
740 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
741 if (shader == NULL)
742 return NULL;
743
744 assert(sizeof(shader->cache_key) == key_size);
745 memcpy(shader->cache_key, key_data, sizeof(shader->cache_key));
746
747 vk_pipeline_cache_object_init(device, &shader->cache_obj,
748 &pipeline_precomp_shader_cache_ops,
749 shader->cache_key,
750 sizeof(shader->cache_key));
751
752 shader->stage = blob_read_uint32(blob);
753 blob_copy_bytes(blob, &shader->rs, sizeof(shader->rs));
754 blob_copy_bytes(blob, &shader->tess, sizeof(shader->tess));
755 blob_copy_bytes(blob, shader->blake3, sizeof(shader->blake3));
756
757 uint64_t nir_size = blob_read_uint64(blob);
758 if (blob->overrun || nir_size > SIZE_MAX)
759 goto fail_shader;
760
761 const void *nir_data = blob_read_bytes(blob, nir_size);
762 if (blob->overrun)
763 goto fail_shader;
764
765 blob_init(&shader->nir_blob);
766 blob_write_bytes(&shader->nir_blob, nir_data, nir_size);
767 if (shader->nir_blob.out_of_memory)
768 goto fail_nir_blob;
769
770 return &shader->cache_obj;
771
772 fail_nir_blob:
773 blob_finish(&shader->nir_blob);
774 fail_shader:
775 vk_pipeline_cache_object_finish(&shader->cache_obj);
776 vk_free(&device->alloc, shader);
777
778 return NULL;
779 }
780
781 static void
vk_pipeline_precomp_shader_destroy(struct vk_device * device,struct vk_pipeline_cache_object * obj)782 vk_pipeline_precomp_shader_destroy(struct vk_device *device,
783 struct vk_pipeline_cache_object *obj)
784 {
785 struct vk_pipeline_precomp_shader *shader =
786 vk_pipeline_precomp_shader_from_cache_obj(obj);
787
788 blob_finish(&shader->nir_blob);
789 vk_pipeline_cache_object_finish(&shader->cache_obj);
790 vk_free(&device->alloc, shader);
791 }
792
793 static nir_shader *
vk_pipeline_precomp_shader_get_nir(const struct vk_pipeline_precomp_shader * shader,const struct nir_shader_compiler_options * nir_options)794 vk_pipeline_precomp_shader_get_nir(const struct vk_pipeline_precomp_shader *shader,
795 const struct nir_shader_compiler_options *nir_options)
796 {
797 struct blob_reader blob;
798 blob_reader_init(&blob, shader->nir_blob.data, shader->nir_blob.size);
799
800 nir_shader *nir = nir_deserialize(NULL, nir_options, &blob);
801 if (blob.overrun) {
802 ralloc_free(nir);
803 return NULL;
804 }
805
806 return nir;
807 }
808
809 static const struct vk_pipeline_cache_object_ops pipeline_precomp_shader_cache_ops = {
810 .serialize = vk_pipeline_precomp_shader_serialize,
811 .deserialize = vk_pipeline_precomp_shader_deserialize,
812 .destroy = vk_pipeline_precomp_shader_destroy,
813 };
814
815 static VkResult
vk_pipeline_precompile_shader(struct vk_device * device,struct vk_pipeline_cache * cache,VkPipelineCreateFlags2KHR pipeline_flags,const void * pipeline_info_pNext,const VkPipelineShaderStageCreateInfo * info,struct vk_pipeline_precomp_shader ** ps_out)816 vk_pipeline_precompile_shader(struct vk_device *device,
817 struct vk_pipeline_cache *cache,
818 VkPipelineCreateFlags2KHR pipeline_flags,
819 const void *pipeline_info_pNext,
820 const VkPipelineShaderStageCreateInfo *info,
821 struct vk_pipeline_precomp_shader **ps_out)
822 {
823 const struct vk_device_shader_ops *ops = device->shader_ops;
824 VkResult result;
825
826 struct vk_pipeline_robustness_state rs;
827 vk_pipeline_robustness_state_fill(device, &rs,
828 pipeline_info_pNext,
829 info->pNext);
830
831 uint8_t stage_sha1[SHA1_DIGEST_LENGTH];
832 vk_pipeline_hash_shader_stage(pipeline_flags, info, &rs, stage_sha1);
833
834 if (cache != NULL) {
835 struct vk_pipeline_cache_object *cache_obj =
836 vk_pipeline_cache_lookup_object(cache, stage_sha1, sizeof(stage_sha1),
837 &pipeline_precomp_shader_cache_ops,
838 NULL /* cache_hit */);
839 if (cache_obj != NULL) {
840 *ps_out = vk_pipeline_precomp_shader_from_cache_obj(cache_obj);
841 return VK_SUCCESS;
842 }
843 }
844
845 if (pipeline_flags &
846 VK_PIPELINE_CREATE_2_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT_KHR)
847 return VK_PIPELINE_COMPILE_REQUIRED;
848
849 const gl_shader_stage stage = vk_to_mesa_shader_stage(info->stage);
850 const struct nir_shader_compiler_options *nir_options =
851 ops->get_nir_options(device->physical, stage, &rs);
852 const struct spirv_to_nir_options spirv_options =
853 ops->get_spirv_options(device->physical, stage, &rs);
854
855 nir_shader *nir;
856 result = vk_pipeline_shader_stage_to_nir(device, pipeline_flags, info,
857 &spirv_options, nir_options,
858 NULL, &nir);
859 if (result != VK_SUCCESS)
860 return result;
861
862 if (ops->preprocess_nir != NULL)
863 ops->preprocess_nir(device->physical, nir);
864
865 struct vk_pipeline_precomp_shader *shader =
866 vk_pipeline_precomp_shader_create(device, stage_sha1,
867 sizeof(stage_sha1),
868 &rs, nir);
869 ralloc_free(nir);
870 if (shader == NULL)
871 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
872
873 if (cache != NULL) {
874 struct vk_pipeline_cache_object *cache_obj = &shader->cache_obj;
875 cache_obj = vk_pipeline_cache_add_object(cache, cache_obj);
876 shader = vk_pipeline_precomp_shader_from_cache_obj(cache_obj);
877 }
878
879 *ps_out = shader;
880
881 return VK_SUCCESS;
882 }
883
884 struct vk_pipeline_stage {
885 gl_shader_stage stage;
886
887 struct vk_pipeline_precomp_shader *precomp;
888 struct vk_shader *shader;
889 };
890
891 static int
cmp_vk_pipeline_stages(const void * _a,const void * _b)892 cmp_vk_pipeline_stages(const void *_a, const void *_b)
893 {
894 const struct vk_pipeline_stage *a = _a, *b = _b;
895 return vk_shader_cmp_graphics_stages(a->stage, b->stage);
896 }
897
898 static bool
vk_pipeline_stage_is_null(const struct vk_pipeline_stage * stage)899 vk_pipeline_stage_is_null(const struct vk_pipeline_stage *stage)
900 {
901 return stage->precomp == NULL && stage->shader == NULL;
902 }
903
904 static void
vk_pipeline_stage_finish(struct vk_device * device,struct vk_pipeline_stage * stage)905 vk_pipeline_stage_finish(struct vk_device *device,
906 struct vk_pipeline_stage *stage)
907 {
908 if (stage->precomp != NULL)
909 vk_pipeline_precomp_shader_unref(device, stage->precomp);
910
911 if (stage->shader)
912 vk_shader_unref(device, stage->shader);
913 }
914
915 static struct vk_pipeline_stage
vk_pipeline_stage_clone(const struct vk_pipeline_stage * in)916 vk_pipeline_stage_clone(const struct vk_pipeline_stage *in)
917 {
918 struct vk_pipeline_stage out = {
919 .stage = in->stage,
920 };
921
922 if (in->precomp)
923 out.precomp = vk_pipeline_precomp_shader_ref(in->precomp);
924
925 if (in->shader)
926 out.shader = vk_shader_ref(in->shader);
927
928 return out;
929 }
930
931 struct vk_graphics_pipeline {
932 struct vk_pipeline base;
933
934 union {
935 struct {
936 struct vk_graphics_pipeline_all_state all_state;
937 struct vk_graphics_pipeline_state state;
938 } lib;
939
940 struct {
941 struct vk_vertex_input_state _dynamic_vi;
942 struct vk_sample_locations_state _dynamic_sl;
943 struct vk_dynamic_graphics_state dynamic;
944 } linked;
945 };
946
947 uint32_t set_layout_count;
948 struct vk_descriptor_set_layout *set_layouts[MESA_VK_MAX_DESCRIPTOR_SETS];
949
950 uint32_t stage_count;
951 struct vk_pipeline_stage stages[MESA_VK_MAX_GRAPHICS_PIPELINE_STAGES];
952 };
953
954 static void
vk_graphics_pipeline_destroy(struct vk_device * device,struct vk_pipeline * pipeline,const VkAllocationCallbacks * pAllocator)955 vk_graphics_pipeline_destroy(struct vk_device *device,
956 struct vk_pipeline *pipeline,
957 const VkAllocationCallbacks *pAllocator)
958 {
959 struct vk_graphics_pipeline *gfx_pipeline =
960 container_of(pipeline, struct vk_graphics_pipeline, base);
961
962 for (uint32_t i = 0; i < gfx_pipeline->stage_count; i++)
963 vk_pipeline_stage_finish(device, &gfx_pipeline->stages[i]);
964
965 for (uint32_t i = 0; i < gfx_pipeline->set_layout_count; i++) {
966 if (gfx_pipeline->set_layouts[i] != NULL)
967 vk_descriptor_set_layout_unref(device, gfx_pipeline->set_layouts[i]);
968 }
969
970 vk_pipeline_free(device, pAllocator, pipeline);
971 }
972
973 static bool
vk_device_supports_stage(struct vk_device * device,gl_shader_stage stage)974 vk_device_supports_stage(struct vk_device *device,
975 gl_shader_stage stage)
976 {
977 const struct vk_features *features = &device->physical->supported_features;
978
979 switch (stage) {
980 case MESA_SHADER_VERTEX:
981 case MESA_SHADER_FRAGMENT:
982 case MESA_SHADER_COMPUTE:
983 return true;
984 case MESA_SHADER_TESS_CTRL:
985 case MESA_SHADER_TESS_EVAL:
986 return features->tessellationShader;
987 case MESA_SHADER_GEOMETRY:
988 return features->geometryShader;
989 case MESA_SHADER_TASK:
990 return features->taskShader;
991 case MESA_SHADER_MESH:
992 return features->meshShader;
993 default:
994 return false;
995 }
996 }
997
998 static const gl_shader_stage all_gfx_stages[] = {
999 MESA_SHADER_VERTEX,
1000 MESA_SHADER_TESS_CTRL,
1001 MESA_SHADER_TESS_EVAL,
1002 MESA_SHADER_GEOMETRY,
1003 MESA_SHADER_TASK,
1004 MESA_SHADER_MESH,
1005 MESA_SHADER_FRAGMENT,
1006 };
1007
1008 static void
vk_graphics_pipeline_cmd_bind(struct vk_command_buffer * cmd_buffer,struct vk_pipeline * pipeline)1009 vk_graphics_pipeline_cmd_bind(struct vk_command_buffer *cmd_buffer,
1010 struct vk_pipeline *pipeline)
1011 {
1012 struct vk_device *device = cmd_buffer->base.device;
1013 const struct vk_device_shader_ops *ops = device->shader_ops;
1014
1015 struct vk_graphics_pipeline *gfx_pipeline = NULL;
1016 struct vk_shader *stage_shader[PIPE_SHADER_MESH_TYPES] = { NULL, };
1017 if (pipeline != NULL) {
1018 assert(pipeline->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS);
1019 assert(!(pipeline->flags & VK_PIPELINE_CREATE_2_LIBRARY_BIT_KHR));
1020 gfx_pipeline = container_of(pipeline, struct vk_graphics_pipeline, base);
1021
1022 for (uint32_t i = 0; i < gfx_pipeline->stage_count; i++) {
1023 struct vk_shader *shader = gfx_pipeline->stages[i].shader;
1024 stage_shader[shader->stage] = shader;
1025 }
1026 }
1027
1028 uint32_t stage_count = 0;
1029 gl_shader_stage stages[ARRAY_SIZE(all_gfx_stages)];
1030 struct vk_shader *shaders[ARRAY_SIZE(all_gfx_stages)];
1031
1032 VkShaderStageFlags vk_stages = 0;
1033 for (uint32_t i = 0; i < ARRAY_SIZE(all_gfx_stages); i++) {
1034 gl_shader_stage stage = all_gfx_stages[i];
1035 if (!vk_device_supports_stage(device, stage)) {
1036 assert(stage_shader[stage] == NULL);
1037 continue;
1038 }
1039
1040 vk_stages |= mesa_to_vk_shader_stage(stage);
1041
1042 stages[stage_count] = stage;
1043 shaders[stage_count] = stage_shader[stage];
1044 stage_count++;
1045 }
1046 ops->cmd_bind_shaders(cmd_buffer, stage_count, stages, shaders);
1047
1048 if (gfx_pipeline != NULL) {
1049 cmd_buffer->pipeline_shader_stages |= vk_stages;
1050 ops->cmd_set_dynamic_graphics_state(cmd_buffer,
1051 &gfx_pipeline->linked.dynamic);
1052 } else {
1053 cmd_buffer->pipeline_shader_stages &= ~vk_stages;
1054 }
1055 }
1056
1057 static VkShaderCreateFlagsEXT
vk_pipeline_to_shader_flags(VkPipelineCreateFlags2KHR pipeline_flags,gl_shader_stage stage)1058 vk_pipeline_to_shader_flags(VkPipelineCreateFlags2KHR pipeline_flags,
1059 gl_shader_stage stage)
1060 {
1061 VkShaderCreateFlagsEXT shader_flags = 0;
1062
1063 if (pipeline_flags & VK_PIPELINE_CREATE_2_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR)
1064 shader_flags |= VK_SHADER_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_MESA;
1065
1066 if (pipeline_flags & VK_PIPELINE_CREATE_2_INDIRECT_BINDABLE_BIT_EXT)
1067 shader_flags |= VK_SHADER_CREATE_INDIRECT_BINDABLE_BIT_EXT;
1068
1069 if (stage == MESA_SHADER_FRAGMENT) {
1070 if (pipeline_flags & VK_PIPELINE_CREATE_2_RENDERING_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR)
1071 shader_flags |= VK_SHADER_CREATE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_EXT;
1072
1073 if (pipeline_flags & VK_PIPELINE_CREATE_2_RENDERING_FRAGMENT_DENSITY_MAP_ATTACHMENT_BIT_EXT)
1074 shader_flags |= VK_SHADER_CREATE_FRAGMENT_DENSITY_MAP_ATTACHMENT_BIT_EXT;
1075 }
1076
1077 if (stage == MESA_SHADER_COMPUTE) {
1078 if (pipeline_flags & VK_PIPELINE_CREATE_2_DISPATCH_BASE_BIT_KHR)
1079 shader_flags |= VK_SHADER_CREATE_DISPATCH_BASE_BIT_EXT;
1080 }
1081
1082 return shader_flags;
1083 }
1084
1085 static VkResult
vk_graphics_pipeline_compile_shaders(struct vk_device * device,struct vk_pipeline_cache * cache,struct vk_graphics_pipeline * pipeline,struct vk_pipeline_layout * pipeline_layout,const struct vk_graphics_pipeline_state * state,uint32_t stage_count,struct vk_pipeline_stage * stages,VkPipelineCreationFeedback * stage_feedbacks)1086 vk_graphics_pipeline_compile_shaders(struct vk_device *device,
1087 struct vk_pipeline_cache *cache,
1088 struct vk_graphics_pipeline *pipeline,
1089 struct vk_pipeline_layout *pipeline_layout,
1090 const struct vk_graphics_pipeline_state *state,
1091 uint32_t stage_count,
1092 struct vk_pipeline_stage *stages,
1093 VkPipelineCreationFeedback *stage_feedbacks)
1094 {
1095 const struct vk_device_shader_ops *ops = device->shader_ops;
1096 VkResult result;
1097
1098 if (stage_count == 0)
1099 return VK_SUCCESS;
1100
1101 /* If we're linking, throw away any previously compiled shaders as they
1102 * likely haven't been properly linked. We keep the precompiled shaders
1103 * and we still look it up in the cache so it may still be fast.
1104 */
1105 if (pipeline->base.flags & VK_PIPELINE_CREATE_2_LINK_TIME_OPTIMIZATION_BIT_EXT) {
1106 for (uint32_t i = 0; i < stage_count; i++) {
1107 if (stages[i].shader != NULL) {
1108 vk_shader_unref(device, stages[i].shader);
1109 stages[i].shader = NULL;
1110 }
1111 }
1112 }
1113
1114 bool have_all_shaders = true;
1115 VkShaderStageFlags all_stages = 0;
1116 struct vk_pipeline_precomp_shader *tcs_precomp = NULL, *tes_precomp = NULL;
1117 for (uint32_t i = 0; i < stage_count; i++) {
1118 all_stages |= mesa_to_vk_shader_stage(stages[i].stage);
1119
1120 if (stages[i].shader == NULL)
1121 have_all_shaders = false;
1122
1123 if (stages[i].stage == MESA_SHADER_TESS_CTRL)
1124 tcs_precomp = stages[i].precomp;
1125
1126 if (stages[i].stage == MESA_SHADER_TESS_EVAL)
1127 tes_precomp = stages[i].precomp;
1128 }
1129
1130 /* If we already have a shader for each stage, there's nothing to do. */
1131 if (have_all_shaders)
1132 return VK_SUCCESS;
1133
1134 struct vk_pipeline_tess_info tess_info = { ._pad = 0 };
1135 if (tcs_precomp != NULL && tes_precomp != NULL) {
1136 tess_info = tcs_precomp->tess;
1137 vk_pipeline_tess_info_merge(&tess_info, &tes_precomp->tess);
1138 }
1139
1140 struct mesa_blake3 blake3_ctx;
1141 _mesa_blake3_init(&blake3_ctx);
1142 for (uint32_t i = 0; i < pipeline->set_layout_count; i++) {
1143 if (pipeline->set_layouts[i] != NULL) {
1144 _mesa_blake3_update(&blake3_ctx, pipeline->set_layouts[i]->blake3,
1145 sizeof(pipeline->set_layouts[i]->blake3));
1146 }
1147 }
1148 if (pipeline_layout != NULL) {
1149 _mesa_blake3_update(&blake3_ctx, &pipeline_layout->push_ranges,
1150 sizeof(pipeline_layout->push_ranges[0]) *
1151 pipeline_layout->push_range_count);
1152 }
1153 blake3_hash layout_blake3;
1154 _mesa_blake3_final(&blake3_ctx, layout_blake3);
1155
1156 /* Partition the shaders */
1157 uint32_t part_count;
1158 uint32_t partition[MESA_VK_MAX_GRAPHICS_PIPELINE_STAGES + 1] = { 0 };
1159 if (pipeline->base.flags & VK_PIPELINE_CREATE_2_LINK_TIME_OPTIMIZATION_BIT_EXT) {
1160 partition[1] = stage_count;
1161 part_count = 1;
1162 } else if (ops->link_geom_stages) {
1163 if (stages[0].stage == MESA_SHADER_FRAGMENT) {
1164 assert(stage_count == 1);
1165 partition[1] = stage_count;
1166 part_count = 1;
1167 } else if (stages[stage_count - 1].stage == MESA_SHADER_FRAGMENT) {
1168 /* In this case we have both */
1169 assert(stage_count > 1);
1170 partition[1] = stage_count - 1;
1171 partition[2] = stage_count;
1172 part_count = 2;
1173 } else {
1174 /* In this case we only have geometry */
1175 partition[1] = stage_count;
1176 part_count = 1;
1177 }
1178 } else {
1179 /* Otherwise, we're don't want to link anything */
1180 part_count = stage_count;
1181 for (uint32_t i = 0; i < stage_count; i++)
1182 partition[i + 1] = i + 1;
1183 }
1184
1185 for (uint32_t p = 0; p < part_count; p++) {
1186 const int64_t part_start = os_time_get_nano();
1187
1188 /* Don't try to re-compile any fast-link shaders */
1189 if (!(pipeline->base.flags &
1190 VK_PIPELINE_CREATE_2_LINK_TIME_OPTIMIZATION_BIT_EXT)) {
1191 assert(partition[p + 1] == partition[p] + 1);
1192 if (stages[partition[p]].shader != NULL)
1193 continue;
1194 }
1195
1196 struct vk_shader_pipeline_cache_key shader_key = { 0 };
1197
1198 _mesa_blake3_init(&blake3_ctx);
1199
1200 VkShaderStageFlags part_stages = 0;
1201 for (uint32_t i = partition[p]; i < partition[p + 1]; i++) {
1202 const struct vk_pipeline_stage *stage = &stages[i];
1203
1204 part_stages |= mesa_to_vk_shader_stage(stage->stage);
1205 _mesa_blake3_update(&blake3_ctx, stage->precomp->blake3,
1206 sizeof(stage->precomp->blake3));
1207
1208 VkShaderCreateFlagsEXT shader_flags =
1209 vk_pipeline_to_shader_flags(pipeline->base.flags, stage->stage);
1210 _mesa_blake3_update(&blake3_ctx, &shader_flags, sizeof(shader_flags));
1211 }
1212
1213 blake3_hash state_blake3;
1214 ops->hash_graphics_state(device->physical, state,
1215 part_stages, state_blake3);
1216
1217 _mesa_blake3_update(&blake3_ctx, state_blake3, sizeof(state_blake3));
1218 _mesa_blake3_update(&blake3_ctx, layout_blake3, sizeof(layout_blake3));
1219
1220 if (part_stages & (VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT |
1221 VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT))
1222 _mesa_blake3_update(&blake3_ctx, &tess_info, sizeof(tess_info));
1223
1224 /* The set of geometry stages used together is used to generate the
1225 * nextStage mask as well as VK_SHADER_CREATE_NO_TASK_SHADER_BIT_EXT.
1226 */
1227 const VkShaderStageFlags geom_stages =
1228 all_stages & ~VK_SHADER_STAGE_FRAGMENT_BIT;
1229 _mesa_blake3_update(&blake3_ctx, &geom_stages, sizeof(geom_stages));
1230
1231 _mesa_blake3_final(&blake3_ctx, shader_key.blake3);
1232
1233 if (cache != NULL) {
1234 /* From the Vulkan 1.3.278 spec:
1235 *
1236 * "VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT
1237 * indicates that a readily usable pipeline or pipeline stage was
1238 * found in the pipelineCache specified by the application in the
1239 * pipeline creation command.
1240 *
1241 * [...]
1242 *
1243 * Note
1244 *
1245 * Implementations are encouraged to provide a meaningful signal
1246 * to applications using this bit. The intention is to communicate
1247 * to the application that the pipeline or pipeline stage was
1248 * created “as fast as it gets” using the pipeline cache provided
1249 * by the application. If an implementation uses an internal
1250 * cache, it is discouraged from setting this bit as the feedback
1251 * would be unactionable."
1252 *
1253 * The cache_hit value returned by vk_pipeline_cache_lookup_object()
1254 * is only set to true when the shader is found in the provided
1255 * pipeline cache. It is left false if we fail to find it in the
1256 * memory cache but find it in the disk cache even though that's
1257 * still a cache hit from the perspective of the compile pipeline.
1258 */
1259 bool all_shaders_found = true;
1260 bool all_cache_hits = true;
1261 for (uint32_t i = partition[p]; i < partition[p + 1]; i++) {
1262 struct vk_pipeline_stage *stage = &stages[i];
1263
1264 shader_key.stage = stage->stage;
1265
1266 if (stage->shader) {
1267 /* If we have a shader from some library pipeline and the key
1268 * matches, just use that.
1269 */
1270 if (memcmp(&stage->shader->pipeline.cache_key,
1271 &shader_key, sizeof(shader_key)) == 0)
1272 continue;
1273
1274 /* Otherwise, throw it away */
1275 vk_shader_unref(device, stage->shader);
1276 stage->shader = NULL;
1277 }
1278
1279 bool cache_hit = false;
1280 struct vk_pipeline_cache_object *cache_obj =
1281 vk_pipeline_cache_lookup_object(cache, &shader_key,
1282 sizeof(shader_key),
1283 &pipeline_shader_cache_ops,
1284 &cache_hit);
1285 if (cache_obj != NULL) {
1286 assert(stage->shader == NULL);
1287 stage->shader = vk_shader_from_cache_obj(cache_obj);
1288 } else {
1289 all_shaders_found = false;
1290 }
1291
1292 if (cache_obj == NULL && !cache_hit)
1293 all_cache_hits = false;
1294 }
1295
1296 if (all_cache_hits && cache != device->mem_cache) {
1297 /* The pipeline cache only really helps if we hit for everything
1298 * in the partition. Otherwise, we have to go re-compile it all
1299 * anyway.
1300 */
1301 for (uint32_t i = partition[p]; i < partition[p + 1]; i++) {
1302 struct vk_pipeline_stage *stage = &stages[i];
1303
1304 stage_feedbacks[stage->stage].flags |=
1305 VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT;
1306 }
1307 }
1308
1309 if (all_shaders_found) {
1310 /* Update duration to take cache lookups into account */
1311 const int64_t part_end = os_time_get_nano();
1312 for (uint32_t i = partition[p]; i < partition[p + 1]; i++) {
1313 struct vk_pipeline_stage *stage = &stages[i];
1314 stage_feedbacks[stage->stage].duration += part_end - part_start;
1315 }
1316 continue;
1317 }
1318 }
1319
1320 if (pipeline->base.flags &
1321 VK_PIPELINE_CREATE_2_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT_KHR)
1322 return VK_PIPELINE_COMPILE_REQUIRED;
1323
1324 struct vk_shader_compile_info infos[MESA_VK_MAX_GRAPHICS_PIPELINE_STAGES];
1325 for (uint32_t i = partition[p]; i < partition[p + 1]; i++) {
1326 struct vk_pipeline_stage *stage = &stages[i];
1327
1328 VkShaderCreateFlagsEXT shader_flags =
1329 vk_pipeline_to_shader_flags(pipeline->base.flags, stage->stage);
1330
1331 if (partition[p + 1] - partition[p] > 1)
1332 shader_flags |= VK_SHADER_CREATE_LINK_STAGE_BIT_EXT;
1333
1334 if ((part_stages & VK_SHADER_STAGE_MESH_BIT_EXT) &&
1335 !(geom_stages & VK_SHADER_STAGE_TASK_BIT_EXT))
1336 shader_flags = VK_SHADER_CREATE_NO_TASK_SHADER_BIT_EXT;
1337
1338 VkShaderStageFlags next_stage;
1339 if (stage->stage == MESA_SHADER_FRAGMENT) {
1340 next_stage = 0;
1341 } else if (i + 1 < stage_count) {
1342 /* We hash geom_stages above so this is safe */
1343 next_stage = mesa_to_vk_shader_stage(stages[i + 1].stage);
1344 } else {
1345 /* We're the last geometry stage */
1346 next_stage = VK_SHADER_STAGE_FRAGMENT_BIT;
1347 }
1348
1349 const struct nir_shader_compiler_options *nir_options =
1350 ops->get_nir_options(device->physical, stage->stage,
1351 &stage->precomp->rs);
1352
1353 nir_shader *nir =
1354 vk_pipeline_precomp_shader_get_nir(stage->precomp, nir_options);
1355 if (nir == NULL) {
1356 for (uint32_t j = partition[p]; j < i; j++)
1357 ralloc_free(infos[i].nir);
1358
1359 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1360 }
1361
1362 if (stage->stage == MESA_SHADER_TESS_CTRL ||
1363 stage->stage == MESA_SHADER_TESS_EVAL)
1364 vk_pipeline_replace_nir_tess_info(nir, &tess_info);
1365
1366 const VkPushConstantRange *push_range = NULL;
1367 if (pipeline_layout != NULL) {
1368 for (uint32_t r = 0; r < pipeline_layout->push_range_count; r++) {
1369 if (pipeline_layout->push_ranges[r].stageFlags &
1370 mesa_to_vk_shader_stage(stage->stage)) {
1371 assert(push_range == NULL);
1372 push_range = &pipeline_layout->push_ranges[r];
1373 }
1374 }
1375 }
1376
1377 infos[i] = (struct vk_shader_compile_info) {
1378 .stage = stage->stage,
1379 .flags = shader_flags,
1380 .next_stage_mask = next_stage,
1381 .nir = nir,
1382 .robustness = &stage->precomp->rs,
1383 .set_layout_count = pipeline->set_layout_count,
1384 .set_layouts = pipeline->set_layouts,
1385 .push_constant_range_count = push_range != NULL,
1386 .push_constant_ranges = push_range != NULL ? push_range : NULL,
1387 };
1388 }
1389
1390 /* vk_shader_ops::compile() consumes the NIR regardless of whether or
1391 * not it succeeds and only generates shaders on success. Once this
1392 * returns, we own the shaders but not the NIR in infos.
1393 */
1394 struct vk_shader *shaders[MESA_VK_MAX_GRAPHICS_PIPELINE_STAGES];
1395 result = ops->compile(device, partition[p + 1] - partition[p],
1396 &infos[partition[p]],
1397 state,
1398 &device->alloc,
1399 &shaders[partition[p]]);
1400 if (result != VK_SUCCESS)
1401 return result;
1402
1403 const int64_t part_end = os_time_get_nano();
1404 for (uint32_t i = partition[p]; i < partition[p + 1]; i++) {
1405 struct vk_pipeline_stage *stage = &stages[i];
1406
1407 shader_key.stage = stage->stage;
1408 vk_shader_init_cache_obj(device, shaders[i], &shader_key,
1409 sizeof(shader_key));
1410
1411 if (stage->shader == NULL) {
1412 struct vk_pipeline_cache_object *cache_obj =
1413 &shaders[i]->pipeline.cache_obj;
1414 if (cache != NULL)
1415 cache_obj = vk_pipeline_cache_add_object(cache, cache_obj);
1416
1417 stage->shader = vk_shader_from_cache_obj(cache_obj);
1418 } else {
1419 /* This can fail to happen if only some of the shaders were found
1420 * in the pipeline cache. In this case, we just throw away the
1421 * shader as vk_pipeline_cache_add_object() would throw it away
1422 * for us anyway.
1423 */
1424 assert(memcmp(&stage->shader->pipeline.cache_key,
1425 &shaders[i]->pipeline.cache_key,
1426 sizeof(shaders[i]->pipeline.cache_key)) == 0);
1427
1428 vk_shader_unref(device, shaders[i]);
1429 }
1430
1431 stage_feedbacks[stage->stage].duration += part_end - part_start;
1432 }
1433 }
1434
1435 return VK_SUCCESS;
1436 }
1437
1438 static VkResult
vk_graphics_pipeline_get_executable_properties(struct vk_device * device,struct vk_pipeline * pipeline,uint32_t * executable_count,VkPipelineExecutablePropertiesKHR * properties)1439 vk_graphics_pipeline_get_executable_properties(
1440 struct vk_device *device,
1441 struct vk_pipeline *pipeline,
1442 uint32_t *executable_count,
1443 VkPipelineExecutablePropertiesKHR *properties)
1444 {
1445 struct vk_graphics_pipeline *gfx_pipeline =
1446 container_of(pipeline, struct vk_graphics_pipeline, base);
1447 VkResult result;
1448
1449 if (properties == NULL) {
1450 *executable_count = 0;
1451 for (uint32_t i = 0; i < gfx_pipeline->stage_count; i++) {
1452 struct vk_shader *shader = gfx_pipeline->stages[i].shader;
1453
1454 uint32_t shader_exec_count = 0;
1455 result = shader->ops->get_executable_properties(device, shader,
1456 &shader_exec_count,
1457 NULL);
1458 assert(result == VK_SUCCESS);
1459 *executable_count += shader_exec_count;
1460 }
1461 } else {
1462 uint32_t arr_len = *executable_count;
1463 *executable_count = 0;
1464 for (uint32_t i = 0; i < gfx_pipeline->stage_count; i++) {
1465 struct vk_shader *shader = gfx_pipeline->stages[i].shader;
1466
1467 uint32_t shader_exec_count = arr_len - *executable_count;
1468 result = shader->ops->get_executable_properties(device, shader,
1469 &shader_exec_count,
1470 &properties[*executable_count]);
1471 if (result != VK_SUCCESS)
1472 return result;
1473
1474 *executable_count += shader_exec_count;
1475 }
1476 }
1477
1478 return VK_SUCCESS;
1479 }
1480
1481 static inline struct vk_shader *
vk_graphics_pipeline_executable_shader(struct vk_device * device,struct vk_graphics_pipeline * gfx_pipeline,uint32_t * executable_index)1482 vk_graphics_pipeline_executable_shader(struct vk_device *device,
1483 struct vk_graphics_pipeline *gfx_pipeline,
1484 uint32_t *executable_index)
1485 {
1486 for (uint32_t i = 0; i < gfx_pipeline->stage_count; i++) {
1487 struct vk_shader *shader = gfx_pipeline->stages[i].shader;
1488
1489 uint32_t shader_exec_count = 0;
1490 shader->ops->get_executable_properties(device, shader,
1491 &shader_exec_count, NULL);
1492
1493 if (*executable_index < shader_exec_count)
1494 return shader;
1495 else
1496 *executable_index -= shader_exec_count;
1497 }
1498
1499 return NULL;
1500 }
1501
1502 static VkResult
vk_graphics_pipeline_get_executable_statistics(struct vk_device * device,struct vk_pipeline * pipeline,uint32_t executable_index,uint32_t * statistic_count,VkPipelineExecutableStatisticKHR * statistics)1503 vk_graphics_pipeline_get_executable_statistics(
1504 struct vk_device *device,
1505 struct vk_pipeline *pipeline,
1506 uint32_t executable_index,
1507 uint32_t *statistic_count,
1508 VkPipelineExecutableStatisticKHR *statistics)
1509 {
1510 struct vk_graphics_pipeline *gfx_pipeline =
1511 container_of(pipeline, struct vk_graphics_pipeline, base);
1512
1513 struct vk_shader *shader =
1514 vk_graphics_pipeline_executable_shader(device, gfx_pipeline,
1515 &executable_index);
1516 if (shader == NULL) {
1517 *statistic_count = 0;
1518 return VK_SUCCESS;
1519 }
1520
1521 return shader->ops->get_executable_statistics(device, shader,
1522 executable_index,
1523 statistic_count,
1524 statistics);
1525 }
1526
1527 static VkResult
vk_graphics_pipeline_get_internal_representations(struct vk_device * device,struct vk_pipeline * pipeline,uint32_t executable_index,uint32_t * internal_representation_count,VkPipelineExecutableInternalRepresentationKHR * internal_representations)1528 vk_graphics_pipeline_get_internal_representations(
1529 struct vk_device *device,
1530 struct vk_pipeline *pipeline,
1531 uint32_t executable_index,
1532 uint32_t *internal_representation_count,
1533 VkPipelineExecutableInternalRepresentationKHR* internal_representations)
1534 {
1535 struct vk_graphics_pipeline *gfx_pipeline =
1536 container_of(pipeline, struct vk_graphics_pipeline, base);
1537
1538 struct vk_shader *shader =
1539 vk_graphics_pipeline_executable_shader(device, gfx_pipeline,
1540 &executable_index);
1541 if (shader == NULL) {
1542 *internal_representation_count = 0;
1543 return VK_SUCCESS;
1544 }
1545
1546 return shader->ops->get_executable_internal_representations(
1547 device, shader, executable_index,
1548 internal_representation_count, internal_representations);
1549 }
1550
1551 static struct vk_shader *
vk_graphics_pipeline_get_shader(struct vk_pipeline * pipeline,gl_shader_stage stage)1552 vk_graphics_pipeline_get_shader(struct vk_pipeline *pipeline,
1553 gl_shader_stage stage)
1554 {
1555 struct vk_graphics_pipeline *gfx_pipeline =
1556 container_of(pipeline, struct vk_graphics_pipeline, base);
1557
1558 for (uint32_t i = 0; i < gfx_pipeline->stage_count; i++) {
1559 if (gfx_pipeline->stages[i].stage == stage)
1560 return gfx_pipeline->stages[i].shader;
1561 }
1562
1563 return NULL;
1564 }
1565
1566 static const struct vk_pipeline_ops vk_graphics_pipeline_ops = {
1567 .destroy = vk_graphics_pipeline_destroy,
1568 .get_executable_statistics = vk_graphics_pipeline_get_executable_statistics,
1569 .get_executable_properties = vk_graphics_pipeline_get_executable_properties,
1570 .get_internal_representations = vk_graphics_pipeline_get_internal_representations,
1571 .cmd_bind = vk_graphics_pipeline_cmd_bind,
1572 .get_shader = vk_graphics_pipeline_get_shader,
1573 };
1574
1575 static VkResult
vk_create_graphics_pipeline(struct vk_device * device,struct vk_pipeline_cache * cache,const VkGraphicsPipelineCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkPipeline * pPipeline)1576 vk_create_graphics_pipeline(struct vk_device *device,
1577 struct vk_pipeline_cache *cache,
1578 const VkGraphicsPipelineCreateInfo *pCreateInfo,
1579 const VkAllocationCallbacks *pAllocator,
1580 VkPipeline *pPipeline)
1581 {
1582 VK_FROM_HANDLE(vk_pipeline_layout, pipeline_layout, pCreateInfo->layout);
1583 const int64_t pipeline_start = os_time_get_nano();
1584 VkResult result;
1585
1586 const VkPipelineCreateFlags2KHR pipeline_flags =
1587 vk_graphics_pipeline_create_flags(pCreateInfo);
1588
1589 const VkPipelineCreationFeedbackCreateInfo *feedback_info =
1590 vk_find_struct_const(pCreateInfo->pNext,
1591 PIPELINE_CREATION_FEEDBACK_CREATE_INFO);
1592
1593 const VkPipelineLibraryCreateInfoKHR *libs_info =
1594 vk_find_struct_const(pCreateInfo->pNext,
1595 PIPELINE_LIBRARY_CREATE_INFO_KHR);
1596
1597 struct vk_graphics_pipeline *pipeline =
1598 vk_pipeline_zalloc(device, &vk_graphics_pipeline_ops,
1599 VK_PIPELINE_BIND_POINT_GRAPHICS,
1600 pipeline_flags, pAllocator, sizeof(*pipeline));
1601 if (pipeline == NULL)
1602 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1603
1604 struct vk_pipeline_stage stages[PIPE_SHADER_MESH_TYPES];
1605 memset(stages, 0, sizeof(stages));
1606
1607 VkPipelineCreationFeedback stage_feedbacks[PIPE_SHADER_MESH_TYPES];
1608 memset(stage_feedbacks, 0, sizeof(stage_feedbacks));
1609
1610 struct vk_graphics_pipeline_state state_tmp, *state;
1611 struct vk_graphics_pipeline_all_state all_state_tmp, *all_state;
1612 if (pipeline->base.flags & VK_PIPELINE_CREATE_2_LIBRARY_BIT_KHR) {
1613 /* For pipeline libraries, the state is stored in the pipeline */
1614 state = &pipeline->lib.state;
1615 all_state = &pipeline->lib.all_state;
1616 } else {
1617 /* For linked pipelines, we throw the state away at the end of pipeline
1618 * creation and only keep the dynamic state.
1619 */
1620 memset(&state_tmp, 0, sizeof(state_tmp));
1621 state = &state_tmp;
1622 all_state = &all_state_tmp;
1623 }
1624
1625 /* If we have libraries, import them first. */
1626 if (libs_info) {
1627 for (uint32_t i = 0; i < libs_info->libraryCount; i++) {
1628 VK_FROM_HANDLE(vk_pipeline, lib_pipeline, libs_info->pLibraries[i]);
1629 assert(lib_pipeline->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS);
1630 assert(lib_pipeline->flags & VK_PIPELINE_CREATE_2_LIBRARY_BIT_KHR);
1631 struct vk_graphics_pipeline *lib_gfx_pipeline =
1632 container_of(lib_pipeline, struct vk_graphics_pipeline, base);
1633
1634 vk_graphics_pipeline_state_merge(state, &lib_gfx_pipeline->lib.state);
1635
1636 pipeline->set_layout_count = MAX2(pipeline->set_layout_count,
1637 lib_gfx_pipeline->set_layout_count);
1638 for (uint32_t i = 0; i < lib_gfx_pipeline->set_layout_count; i++) {
1639 if (lib_gfx_pipeline->set_layouts[i] == NULL)
1640 continue;
1641
1642 if (pipeline->set_layouts[i] == NULL) {
1643 pipeline->set_layouts[i] =
1644 vk_descriptor_set_layout_ref(lib_gfx_pipeline->set_layouts[i]);
1645 }
1646 }
1647
1648 for (uint32_t i = 0; i < lib_gfx_pipeline->stage_count; i++) {
1649 const struct vk_pipeline_stage *lib_stage =
1650 &lib_gfx_pipeline->stages[i];
1651
1652 /* We shouldn't have duplicated stages in the imported pipeline
1653 * but it's cheap enough to protect against it so we may as well.
1654 */
1655 assert(lib_stage->stage < ARRAY_SIZE(stages));
1656 assert(vk_pipeline_stage_is_null(&stages[lib_stage->stage]));
1657 if (!vk_pipeline_stage_is_null(&stages[lib_stage->stage]))
1658 continue;
1659
1660 stages[lib_stage->stage] = vk_pipeline_stage_clone(lib_stage);
1661 }
1662 }
1663 }
1664
1665 result = vk_graphics_pipeline_state_fill(device, state,
1666 pCreateInfo,
1667 NULL /* driver_rp */,
1668 0 /* driver_rp_flags */,
1669 all_state,
1670 NULL, 0, NULL);
1671 if (result != VK_SUCCESS)
1672 goto fail_stages;
1673
1674 if (!(pipeline->base.flags & VK_PIPELINE_CREATE_2_LIBRARY_BIT_KHR)) {
1675 pipeline->linked.dynamic.vi = &pipeline->linked._dynamic_vi;
1676 pipeline->linked.dynamic.ms.sample_locations =
1677 &pipeline->linked._dynamic_sl;
1678 vk_dynamic_graphics_state_fill(&pipeline->linked.dynamic, &state_tmp);
1679 }
1680
1681 if (pipeline_layout != NULL) {
1682 pipeline->set_layout_count = MAX2(pipeline->set_layout_count,
1683 pipeline_layout->set_count);
1684 for (uint32_t i = 0; i < pipeline_layout->set_count; i++) {
1685 if (pipeline_layout->set_layouts[i] == NULL)
1686 continue;
1687
1688 if (pipeline->set_layouts[i] == NULL) {
1689 pipeline->set_layouts[i] =
1690 vk_descriptor_set_layout_ref(pipeline_layout->set_layouts[i]);
1691 }
1692 }
1693 }
1694
1695 for (uint32_t i = 0; i < pCreateInfo->stageCount; i++) {
1696 const VkPipelineShaderStageCreateInfo *stage_info =
1697 &pCreateInfo->pStages[i];
1698
1699 const int64_t stage_start = os_time_get_nano();
1700
1701 assert(util_bitcount(stage_info->stage) == 1);
1702 if (!(state->shader_stages & stage_info->stage))
1703 continue;
1704
1705 gl_shader_stage stage = vk_to_mesa_shader_stage(stage_info->stage);
1706 assert(vk_device_supports_stage(device, stage));
1707
1708 stage_feedbacks[stage].flags |=
1709 VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT;
1710
1711 if (!vk_pipeline_stage_is_null(&stages[stage]))
1712 continue;
1713
1714 struct vk_pipeline_precomp_shader *precomp;
1715 result = vk_pipeline_precompile_shader(device, cache, pipeline_flags,
1716 pCreateInfo->pNext,
1717 stage_info,
1718 &precomp);
1719 if (result != VK_SUCCESS)
1720 goto fail_stages;
1721
1722 stages[stage] = (struct vk_pipeline_stage) {
1723 .stage = stage,
1724 .precomp = precomp,
1725 };
1726
1727 const int64_t stage_end = os_time_get_nano();
1728 stage_feedbacks[stage].duration += stage_end - stage_start;
1729 }
1730
1731 /* Compact the array of stages */
1732 uint32_t stage_count = 0;
1733 for (uint32_t s = 0; s < ARRAY_SIZE(stages); s++) {
1734 assert(s >= stage_count);
1735 if (!vk_pipeline_stage_is_null(&stages[s]))
1736 stages[stage_count++] = stages[s];
1737 }
1738 for (uint32_t s = stage_count; s < ARRAY_SIZE(stages); s++)
1739 memset(&stages[s], 0, sizeof(stages[s]));
1740
1741 /* Sort so we always give the driver shaders in order.
1742 *
1743 * This makes everything easier for everyone. This also helps stabilize
1744 * shader keys so that we get a cache hit even if the client gives us
1745 * the stages in a different order.
1746 */
1747 qsort(stages, stage_count, sizeof(*stages), cmp_vk_pipeline_stages);
1748
1749 result = vk_graphics_pipeline_compile_shaders(device, cache, pipeline,
1750 pipeline_layout, state,
1751 stage_count, stages,
1752 stage_feedbacks);
1753 if (result != VK_SUCCESS)
1754 goto fail_stages;
1755
1756 /* Throw away precompiled shaders unless the client explicitly asks us to
1757 * keep them.
1758 */
1759 if (!(pipeline_flags &
1760 VK_PIPELINE_CREATE_2_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT)) {
1761 for (uint32_t i = 0; i < stage_count; i++) {
1762 if (stages[i].precomp != NULL) {
1763 vk_pipeline_precomp_shader_unref(device, stages[i].precomp);
1764 stages[i].precomp = NULL;
1765 }
1766 }
1767 }
1768
1769 pipeline->stage_count = stage_count;
1770 for (uint32_t i = 0; i < stage_count; i++) {
1771 pipeline->base.stages |= mesa_to_vk_shader_stage(stages[i].stage);
1772 pipeline->stages[i] = stages[i];
1773 }
1774
1775 const int64_t pipeline_end = os_time_get_nano();
1776 if (feedback_info != NULL) {
1777 VkPipelineCreationFeedback pipeline_feedback = {
1778 .flags = VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT,
1779 .duration = pipeline_end - pipeline_start,
1780 };
1781
1782 /* From the Vulkan 1.3.275 spec:
1783 *
1784 * "An implementation should set the
1785 * VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT
1786 * bit if it was able to avoid the large majority of pipeline or
1787 * pipeline stage creation work by using the pipelineCache parameter"
1788 *
1789 * We really shouldn't set this bit unless all the shaders hit the
1790 * cache.
1791 */
1792 uint32_t cache_hit_count = 0;
1793 for (uint32_t i = 0; i < stage_count; i++) {
1794 const gl_shader_stage stage = stages[i].stage;
1795 if (stage_feedbacks[stage].flags &
1796 VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT)
1797 cache_hit_count++;
1798 }
1799 if (cache_hit_count > 0 && cache_hit_count == stage_count) {
1800 pipeline_feedback.flags |=
1801 VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT;
1802 }
1803
1804 *feedback_info->pPipelineCreationFeedback = pipeline_feedback;
1805
1806 /* VUID-VkGraphicsPipelineCreateInfo-pipelineStageCreationFeedbackCount-06594 */
1807 assert(feedback_info->pipelineStageCreationFeedbackCount == 0 ||
1808 feedback_info->pipelineStageCreationFeedbackCount ==
1809 pCreateInfo->stageCount);
1810 for (uint32_t i = 0;
1811 i < feedback_info->pipelineStageCreationFeedbackCount; i++) {
1812 const gl_shader_stage stage =
1813 vk_to_mesa_shader_stage(pCreateInfo->pStages[i].stage);
1814
1815 feedback_info->pPipelineStageCreationFeedbacks[i] =
1816 stage_feedbacks[stage];
1817 }
1818 }
1819
1820 *pPipeline = vk_pipeline_to_handle(&pipeline->base);
1821
1822 return VK_SUCCESS;
1823
1824 fail_stages:
1825 for (uint32_t i = 0; i < ARRAY_SIZE(stages); i++)
1826 vk_pipeline_stage_finish(device, &stages[i]);
1827
1828 vk_graphics_pipeline_destroy(device, &pipeline->base, pAllocator);
1829
1830 return result;
1831 }
1832
1833 VKAPI_ATTR VkResult VKAPI_CALL
vk_common_CreateGraphicsPipelines(VkDevice _device,VkPipelineCache pipelineCache,uint32_t createInfoCount,const VkGraphicsPipelineCreateInfo * pCreateInfos,const VkAllocationCallbacks * pAllocator,VkPipeline * pPipelines)1834 vk_common_CreateGraphicsPipelines(VkDevice _device,
1835 VkPipelineCache pipelineCache,
1836 uint32_t createInfoCount,
1837 const VkGraphicsPipelineCreateInfo *pCreateInfos,
1838 const VkAllocationCallbacks *pAllocator,
1839 VkPipeline *pPipelines)
1840 {
1841 VK_FROM_HANDLE(vk_device, device, _device);
1842 VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
1843 VkResult first_error_or_success = VK_SUCCESS;
1844
1845 /* Use implicit pipeline cache if there's no cache set */
1846 if (!cache && device->mem_cache)
1847 cache = device->mem_cache;
1848
1849 /* From the Vulkan 1.3.274 spec:
1850 *
1851 * "When attempting to create many pipelines in a single command, it is
1852 * possible that creation may fail for a subset of them. In this case,
1853 * the corresponding elements of pPipelines will be set to
1854 * VK_NULL_HANDLE.
1855 */
1856 memset(pPipelines, 0, createInfoCount * sizeof(*pPipelines));
1857
1858 unsigned i = 0;
1859 for (; i < createInfoCount; i++) {
1860 VkResult result = vk_create_graphics_pipeline(device, cache,
1861 &pCreateInfos[i],
1862 pAllocator,
1863 &pPipelines[i]);
1864 if (result == VK_SUCCESS)
1865 continue;
1866
1867 if (first_error_or_success == VK_SUCCESS)
1868 first_error_or_success = result;
1869
1870 /* Bail out on the first error != VK_PIPELINE_COMPILE_REQUIRED as it
1871 * is not obvious what error should be report upon 2 different failures.
1872 */
1873 if (result != VK_PIPELINE_COMPILE_REQUIRED)
1874 return result;
1875
1876 const VkPipelineCreateFlags2KHR flags =
1877 vk_graphics_pipeline_create_flags(&pCreateInfos[i]);
1878 if (flags & VK_PIPELINE_CREATE_2_EARLY_RETURN_ON_FAILURE_BIT_KHR)
1879 return result;
1880 }
1881
1882 return first_error_or_success;
1883 }
1884
1885 struct vk_compute_pipeline {
1886 struct vk_pipeline base;
1887 struct vk_shader *shader;
1888 };
1889
1890 static void
vk_compute_pipeline_destroy(struct vk_device * device,struct vk_pipeline * pipeline,const VkAllocationCallbacks * pAllocator)1891 vk_compute_pipeline_destroy(struct vk_device *device,
1892 struct vk_pipeline *pipeline,
1893 const VkAllocationCallbacks *pAllocator)
1894 {
1895 struct vk_compute_pipeline *comp_pipeline =
1896 container_of(pipeline, struct vk_compute_pipeline, base);
1897
1898 vk_shader_unref(device, comp_pipeline->shader);
1899 vk_pipeline_free(device, pAllocator, pipeline);
1900 }
1901
1902 static void
vk_compute_pipeline_cmd_bind(struct vk_command_buffer * cmd_buffer,struct vk_pipeline * pipeline)1903 vk_compute_pipeline_cmd_bind(struct vk_command_buffer *cmd_buffer,
1904 struct vk_pipeline *pipeline)
1905 {
1906 struct vk_device *device = cmd_buffer->base.device;
1907 const struct vk_device_shader_ops *ops = device->shader_ops;
1908
1909 struct vk_shader *shader = NULL;
1910 if (pipeline != NULL) {
1911 assert(pipeline->bind_point == VK_PIPELINE_BIND_POINT_COMPUTE);
1912 struct vk_compute_pipeline *comp_pipeline =
1913 container_of(pipeline, struct vk_compute_pipeline, base);
1914
1915 shader = comp_pipeline->shader;
1916
1917 cmd_buffer->pipeline_shader_stages |= VK_SHADER_STAGE_COMPUTE_BIT;
1918 } else {
1919 cmd_buffer->pipeline_shader_stages &= ~VK_SHADER_STAGE_COMPUTE_BIT;
1920 }
1921
1922 gl_shader_stage stage = MESA_SHADER_COMPUTE;
1923 ops->cmd_bind_shaders(cmd_buffer, 1, &stage, &shader);
1924 }
1925
1926 static VkResult
vk_pipeline_compile_compute_stage(struct vk_device * device,struct vk_pipeline_cache * cache,struct vk_compute_pipeline * pipeline,struct vk_pipeline_layout * pipeline_layout,struct vk_pipeline_stage * stage,bool * cache_hit)1927 vk_pipeline_compile_compute_stage(struct vk_device *device,
1928 struct vk_pipeline_cache *cache,
1929 struct vk_compute_pipeline *pipeline,
1930 struct vk_pipeline_layout *pipeline_layout,
1931 struct vk_pipeline_stage *stage,
1932 bool *cache_hit)
1933 {
1934 const struct vk_device_shader_ops *ops = device->shader_ops;
1935 VkResult result;
1936
1937 const VkPushConstantRange *push_range = NULL;
1938 if (pipeline_layout != NULL) {
1939 for (uint32_t r = 0; r < pipeline_layout->push_range_count; r++) {
1940 if (pipeline_layout->push_ranges[r].stageFlags &
1941 VK_SHADER_STAGE_COMPUTE_BIT) {
1942 assert(push_range == NULL);
1943 push_range = &pipeline_layout->push_ranges[r];
1944 }
1945 }
1946 }
1947
1948 VkShaderCreateFlagsEXT shader_flags =
1949 vk_pipeline_to_shader_flags(pipeline->base.flags, MESA_SHADER_COMPUTE);
1950
1951 struct mesa_blake3 blake3_ctx;
1952 _mesa_blake3_init(&blake3_ctx);
1953
1954 _mesa_blake3_update(&blake3_ctx, stage->precomp->blake3,
1955 sizeof(stage->precomp->blake3));
1956
1957 _mesa_blake3_update(&blake3_ctx, &shader_flags, sizeof(shader_flags));
1958
1959 for (uint32_t i = 0; i < pipeline_layout->set_count; i++) {
1960 if (pipeline_layout->set_layouts[i] != NULL) {
1961 _mesa_blake3_update(&blake3_ctx,
1962 pipeline_layout->set_layouts[i]->blake3,
1963 sizeof(pipeline_layout->set_layouts[i]->blake3));
1964 }
1965 }
1966 if (push_range != NULL)
1967 _mesa_blake3_update(&blake3_ctx, push_range, sizeof(*push_range));
1968
1969 struct vk_shader_pipeline_cache_key shader_key = {
1970 .stage = MESA_SHADER_COMPUTE,
1971 };
1972 _mesa_blake3_final(&blake3_ctx, shader_key.blake3);
1973
1974 if (cache != NULL) {
1975 struct vk_pipeline_cache_object *cache_obj =
1976 vk_pipeline_cache_lookup_object(cache, &shader_key,
1977 sizeof(shader_key),
1978 &pipeline_shader_cache_ops,
1979 cache_hit);
1980 if (cache_obj != NULL) {
1981 stage->shader = vk_shader_from_cache_obj(cache_obj);
1982 return VK_SUCCESS;
1983 }
1984 }
1985
1986 if (pipeline->base.flags &
1987 VK_PIPELINE_CREATE_2_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT_KHR)
1988 return VK_PIPELINE_COMPILE_REQUIRED;
1989
1990 const struct nir_shader_compiler_options *nir_options =
1991 ops->get_nir_options(device->physical, stage->stage,
1992 &stage->precomp->rs);
1993
1994 nir_shader *nir = vk_pipeline_precomp_shader_get_nir(stage->precomp,
1995 nir_options);
1996 if (nir == NULL)
1997 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1998
1999 /* vk_device_shader_ops::compile() consumes the NIR regardless of whether
2000 * or not it succeeds and only generates shaders on success. Once compile()
2001 * returns, we own the shaders but not the NIR in infos.
2002 */
2003 struct vk_shader_compile_info compile_info = {
2004 .stage = stage->stage,
2005 .flags = shader_flags,
2006 .next_stage_mask = 0,
2007 .nir = nir,
2008 .robustness = &stage->precomp->rs,
2009 .set_layout_count = pipeline_layout->set_count,
2010 .set_layouts = pipeline_layout->set_layouts,
2011 .push_constant_range_count = push_range != NULL,
2012 .push_constant_ranges = push_range != NULL ? push_range : NULL,
2013 };
2014
2015 struct vk_shader *shader;
2016 result = ops->compile(device, 1, &compile_info, NULL,
2017 &device->alloc, &shader);
2018 if (result != VK_SUCCESS)
2019 return result;
2020
2021 vk_shader_init_cache_obj(device, shader, &shader_key, sizeof(shader_key));
2022
2023 struct vk_pipeline_cache_object *cache_obj = &shader->pipeline.cache_obj;
2024 if (cache != NULL)
2025 cache_obj = vk_pipeline_cache_add_object(cache, cache_obj);
2026
2027 stage->shader = vk_shader_from_cache_obj(cache_obj);
2028
2029 return VK_SUCCESS;
2030 }
2031
2032 static VkResult
vk_compute_pipeline_get_executable_properties(struct vk_device * device,struct vk_pipeline * pipeline,uint32_t * executable_count,VkPipelineExecutablePropertiesKHR * properties)2033 vk_compute_pipeline_get_executable_properties(
2034 struct vk_device *device,
2035 struct vk_pipeline *pipeline,
2036 uint32_t *executable_count,
2037 VkPipelineExecutablePropertiesKHR *properties)
2038 {
2039 struct vk_compute_pipeline *comp_pipeline =
2040 container_of(pipeline, struct vk_compute_pipeline, base);
2041 struct vk_shader *shader = comp_pipeline->shader;
2042
2043 return shader->ops->get_executable_properties(device, shader,
2044 executable_count,
2045 properties);
2046 }
2047
2048 static VkResult
vk_compute_pipeline_get_executable_statistics(struct vk_device * device,struct vk_pipeline * pipeline,uint32_t executable_index,uint32_t * statistic_count,VkPipelineExecutableStatisticKHR * statistics)2049 vk_compute_pipeline_get_executable_statistics(
2050 struct vk_device *device,
2051 struct vk_pipeline *pipeline,
2052 uint32_t executable_index,
2053 uint32_t *statistic_count,
2054 VkPipelineExecutableStatisticKHR *statistics)
2055 {
2056 struct vk_compute_pipeline *comp_pipeline =
2057 container_of(pipeline, struct vk_compute_pipeline, base);
2058 struct vk_shader *shader = comp_pipeline->shader;
2059
2060 return shader->ops->get_executable_statistics(device, shader,
2061 executable_index,
2062 statistic_count,
2063 statistics);
2064 }
2065
2066 static VkResult
vk_compute_pipeline_get_internal_representations(struct vk_device * device,struct vk_pipeline * pipeline,uint32_t executable_index,uint32_t * internal_representation_count,VkPipelineExecutableInternalRepresentationKHR * internal_representations)2067 vk_compute_pipeline_get_internal_representations(
2068 struct vk_device *device,
2069 struct vk_pipeline *pipeline,
2070 uint32_t executable_index,
2071 uint32_t *internal_representation_count,
2072 VkPipelineExecutableInternalRepresentationKHR* internal_representations)
2073 {
2074 struct vk_compute_pipeline *comp_pipeline =
2075 container_of(pipeline, struct vk_compute_pipeline, base);
2076 struct vk_shader *shader = comp_pipeline->shader;
2077
2078 return shader->ops->get_executable_internal_representations(
2079 device, shader, executable_index,
2080 internal_representation_count, internal_representations);
2081 }
2082
2083 static struct vk_shader *
vk_compute_pipeline_get_shader(struct vk_pipeline * pipeline,gl_shader_stage stage)2084 vk_compute_pipeline_get_shader(struct vk_pipeline *pipeline,
2085 gl_shader_stage stage)
2086 {
2087 struct vk_compute_pipeline *comp_pipeline =
2088 container_of(pipeline, struct vk_compute_pipeline, base);
2089
2090 assert(stage == MESA_SHADER_COMPUTE);
2091 return comp_pipeline->shader;
2092 }
2093
2094 static const struct vk_pipeline_ops vk_compute_pipeline_ops = {
2095 .destroy = vk_compute_pipeline_destroy,
2096 .get_executable_statistics = vk_compute_pipeline_get_executable_statistics,
2097 .get_executable_properties = vk_compute_pipeline_get_executable_properties,
2098 .get_internal_representations = vk_compute_pipeline_get_internal_representations,
2099 .cmd_bind = vk_compute_pipeline_cmd_bind,
2100 .get_shader = vk_compute_pipeline_get_shader,
2101 };
2102
2103 static VkResult
vk_create_compute_pipeline(struct vk_device * device,struct vk_pipeline_cache * cache,const VkComputePipelineCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkPipeline * pPipeline)2104 vk_create_compute_pipeline(struct vk_device *device,
2105 struct vk_pipeline_cache *cache,
2106 const VkComputePipelineCreateInfo *pCreateInfo,
2107 const VkAllocationCallbacks *pAllocator,
2108 VkPipeline *pPipeline)
2109 {
2110 VK_FROM_HANDLE(vk_pipeline_layout, pipeline_layout, pCreateInfo->layout);
2111 int64_t pipeline_start = os_time_get_nano();
2112 VkResult result;
2113
2114 const VkPipelineCreateFlags2KHR pipeline_flags =
2115 vk_compute_pipeline_create_flags(pCreateInfo);
2116
2117 const VkPipelineCreationFeedbackCreateInfo *feedback_info =
2118 vk_find_struct_const(pCreateInfo->pNext,
2119 PIPELINE_CREATION_FEEDBACK_CREATE_INFO);
2120
2121 struct vk_compute_pipeline *pipeline =
2122 vk_pipeline_zalloc(device, &vk_compute_pipeline_ops,
2123 VK_PIPELINE_BIND_POINT_COMPUTE,
2124 pipeline_flags, pAllocator, sizeof(*pipeline));
2125 if (pipeline == NULL)
2126 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
2127
2128 pipeline->base.stages = VK_SHADER_STAGE_COMPUTE_BIT;
2129
2130 struct vk_pipeline_stage stage = {
2131 .stage = MESA_SHADER_COMPUTE,
2132 };
2133 result = vk_pipeline_precompile_shader(device, cache, pipeline_flags,
2134 pCreateInfo->pNext,
2135 &pCreateInfo->stage,
2136 &stage.precomp);
2137 if (result != VK_SUCCESS)
2138 goto fail_pipeline;
2139
2140 bool cache_hit;
2141 result = vk_pipeline_compile_compute_stage(device, cache, pipeline,
2142 pipeline_layout, &stage,
2143 &cache_hit);
2144 if (result != VK_SUCCESS)
2145 goto fail_stage;
2146
2147 if (stage.precomp != NULL)
2148 vk_pipeline_precomp_shader_unref(device, stage.precomp);
2149 pipeline->shader = stage.shader;
2150
2151 const int64_t pipeline_end = os_time_get_nano();
2152 if (feedback_info != NULL) {
2153 VkPipelineCreationFeedback pipeline_feedback = {
2154 .flags = VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT,
2155 .duration = pipeline_end - pipeline_start,
2156 };
2157 if (cache_hit && cache != device->mem_cache) {
2158 pipeline_feedback.flags |=
2159 VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT;
2160 }
2161
2162 *feedback_info->pPipelineCreationFeedback = pipeline_feedback;
2163 if (feedback_info->pipelineStageCreationFeedbackCount > 0) {
2164 feedback_info->pPipelineStageCreationFeedbacks[0] =
2165 pipeline_feedback;
2166 }
2167 }
2168
2169 *pPipeline = vk_pipeline_to_handle(&pipeline->base);
2170
2171 return VK_SUCCESS;
2172
2173 fail_stage:
2174 vk_pipeline_stage_finish(device, &stage);
2175 fail_pipeline:
2176 vk_pipeline_free(device, pAllocator, &pipeline->base);
2177
2178 return result;
2179 }
2180
2181 VKAPI_ATTR VkResult VKAPI_CALL
vk_common_CreateComputePipelines(VkDevice _device,VkPipelineCache pipelineCache,uint32_t createInfoCount,const VkComputePipelineCreateInfo * pCreateInfos,const VkAllocationCallbacks * pAllocator,VkPipeline * pPipelines)2182 vk_common_CreateComputePipelines(VkDevice _device,
2183 VkPipelineCache pipelineCache,
2184 uint32_t createInfoCount,
2185 const VkComputePipelineCreateInfo *pCreateInfos,
2186 const VkAllocationCallbacks *pAllocator,
2187 VkPipeline *pPipelines)
2188 {
2189 VK_FROM_HANDLE(vk_device, device, _device);
2190 VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
2191 VkResult first_error_or_success = VK_SUCCESS;
2192
2193 /* Use implicit pipeline cache if there's no cache set */
2194 if (!cache && device->mem_cache)
2195 cache = device->mem_cache;
2196
2197 /* From the Vulkan 1.3.274 spec:
2198 *
2199 * "When attempting to create many pipelines in a single command, it is
2200 * possible that creation may fail for a subset of them. In this case,
2201 * the corresponding elements of pPipelines will be set to
2202 * VK_NULL_HANDLE.
2203 */
2204 memset(pPipelines, 0, createInfoCount * sizeof(*pPipelines));
2205
2206 unsigned i = 0;
2207 for (; i < createInfoCount; i++) {
2208 VkResult result = vk_create_compute_pipeline(device, cache,
2209 &pCreateInfos[i],
2210 pAllocator,
2211 &pPipelines[i]);
2212 if (result == VK_SUCCESS)
2213 continue;
2214
2215 if (first_error_or_success == VK_SUCCESS)
2216 first_error_or_success = result;
2217
2218 /* Bail out on the first error != VK_PIPELINE_COMPILE_REQUIRED as it
2219 * is not obvious what error should be report upon 2 different failures.
2220 */
2221 if (result != VK_PIPELINE_COMPILE_REQUIRED)
2222 return result;
2223
2224 const VkPipelineCreateFlags2KHR flags =
2225 vk_compute_pipeline_create_flags(&pCreateInfos[i]);
2226 if (flags & VK_PIPELINE_CREATE_2_EARLY_RETURN_ON_FAILURE_BIT_KHR)
2227 return result;
2228 }
2229
2230 return first_error_or_success;
2231 }
2232
2233 void
vk_cmd_unbind_pipelines_for_stages(struct vk_command_buffer * cmd_buffer,VkShaderStageFlags stages)2234 vk_cmd_unbind_pipelines_for_stages(struct vk_command_buffer *cmd_buffer,
2235 VkShaderStageFlags stages)
2236 {
2237 stages &= cmd_buffer->pipeline_shader_stages;
2238
2239 if (stages & ~VK_SHADER_STAGE_COMPUTE_BIT)
2240 vk_graphics_pipeline_cmd_bind(cmd_buffer, NULL);
2241
2242 if (stages & VK_SHADER_STAGE_COMPUTE_BIT)
2243 vk_compute_pipeline_cmd_bind(cmd_buffer, NULL);
2244 }
2245