1 /*------------------------------------------------------------------------
2 * Vulkan Conformance Tests
3 * ------------------------
4 *
5 * Copyright (c) 2022 The Khronos Group Inc.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file vktGlobalPriorityQueueUtils.cpp
21 * \brief Global Priority Queue Utils
22 *//*--------------------------------------------------------------------*/
23
24 #include "vktGlobalPriorityQueueUtils.hpp"
25 #include "vkImageUtil.hpp"
26 #include "vkQueryUtil.hpp"
27 #include "vkTypeUtil.hpp"
28 #include "vktTestCase.hpp"
29 #include "deStringUtil.hpp"
30 #include <vector>
31
32 using namespace vk;
33
34 namespace vkt
35 {
36 namespace synchronization
37 {
38
findQueueFamilyIndex(const InstanceInterface & vki,VkPhysicalDevice dev,VkQueueFlags includeFlags,VkQueueFlags excludeFlags,bool priorityQueryEnabled,QueueGlobalPriorities priorities,const bool eitherAnyOrAll)39 deUint32 findQueueFamilyIndex (const InstanceInterface& vki,
40 VkPhysicalDevice dev,
41 VkQueueFlags includeFlags,
42 VkQueueFlags excludeFlags,
43 bool priorityQueryEnabled,
44 QueueGlobalPriorities priorities,
45 const bool eitherAnyOrAll)
46 {
47 deUint32 queueFamilyPropertyCount = 0;
48 vki.getPhysicalDeviceQueueFamilyProperties2(dev, &queueFamilyPropertyCount, nullptr);
49
50 std::vector<VkQueueFamilyGlobalPriorityPropertiesKHR> familyPriorityProperties
51 (priorityQueryEnabled ? queueFamilyPropertyCount : 0);
52 std::vector<VkQueueFamilyProperties2> familyProperties2(queueFamilyPropertyCount);
53
54 for (deUint32 i = 0; i < queueFamilyPropertyCount; ++i)
55 {
56 VkQueueFamilyGlobalPriorityPropertiesKHR* item = nullptr;
57 if (priorityQueryEnabled)
58 {
59 item = &familyPriorityProperties[i];
60 *item = {};
61 item->sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_GLOBAL_PRIORITY_PROPERTIES_KHR;
62 }
63
64 VkQueueFamilyProperties2& item2 = familyProperties2[i];
65 item2 = {};
66 item2.sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2;
67 item2.pNext = item;
68 }
69
70 vki.getPhysicalDeviceQueueFamilyProperties2(dev, &queueFamilyPropertyCount, familyProperties2.data());
71
72 for (deUint32 familyIndex = 0; familyIndex < queueFamilyPropertyCount; ++familyIndex)
73 {
74 const VkQueueFamilyProperties& props = familyProperties2[familyIndex].queueFamilyProperties;
75 const VkQueueFlags queueFlags = props.queueFlags;
76
77 if ( ((queueFlags & excludeFlags) == 0) && ((queueFlags & includeFlags) == includeFlags) )
78 {
79 bool matches = true;
80 if (priorityQueryEnabled)
81 {
82 const QueueGlobalPriorities prios(familyPriorityProperties[familyIndex]);
83 if (eitherAnyOrAll)
84 matches = priorities.any(prios);
85 else matches = priorities.all(prios);
86 }
87 if (matches)
88 {
89 return familyIndex;
90 }
91 }
92 }
93
94 return INVALID_UINT32;
95 }
96
QueueGlobalPriorities()97 QueueGlobalPriorities::QueueGlobalPriorities ()
98 : m_priorities {}
99 {
100 }
QueueGlobalPriorities(const QueueGlobalPriorities & other)101 QueueGlobalPriorities::QueueGlobalPriorities (const QueueGlobalPriorities& other)
102 : m_priorities (other.m_priorities)
103 {
104 }
QueueGlobalPriorities(const VkQueueFamilyGlobalPriorityPropertiesKHR & source)105 QueueGlobalPriorities::QueueGlobalPriorities (const VkQueueFamilyGlobalPriorityPropertiesKHR& source)
106 : m_priorities ()
107 {
108 for (deUint32 i = 0; i < source.priorityCount; ++i)
109 m_priorities.insert(source.priorities[i]);
110 }
QueueGlobalPriorities(std::initializer_list<Priority> priorities)111 QueueGlobalPriorities::QueueGlobalPriorities (std::initializer_list<Priority> priorities)
112 {
113 for (auto i = priorities.begin(); i != priorities.end(); ++i)
114 m_priorities.insert(*i);
115 }
full()116 QueueGlobalPriorities QueueGlobalPriorities::full ()
117 {
118 return QueueGlobalPriorities(
119 {
120 VK_QUEUE_GLOBAL_PRIORITY_LOW_KHR,
121 VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_KHR,
122 VK_QUEUE_GLOBAL_PRIORITY_HIGH_KHR,
123 VK_QUEUE_GLOBAL_PRIORITY_REALTIME_KHR
124 });
125 }
insert(const Priority & prio)126 bool QueueGlobalPriorities::insert (const Priority& prio)
127 {
128 auto item = m_priorities.find(prio);
129 if (m_priorities.end() == item) {
130 m_priorities.insert(prio);
131 return true;
132 }
133 return false;
134 }
remove(const Priority & prio)135 bool QueueGlobalPriorities::remove (const Priority& prio)
136 {
137 auto item = m_priorities.find(prio);
138 if (m_priorities.end() != item) {
139 m_priorities.erase(item);
140 return true;
141 }
142 return false;
143 }
any(const Priority & prio) const144 bool QueueGlobalPriorities::any (const Priority& prio) const
145 {
146 return m_priorities.find(prio) != m_priorities.end();
147 }
make(void * pNext) const148 auto QueueGlobalPriorities::make (void* pNext) const -> VkQueueFamilyGlobalPriorityPropertiesKHR
149 {
150 auto start = m_priorities.begin();
151 VkQueueFamilyGlobalPriorityPropertiesKHR res{};
152 res.sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_GLOBAL_PRIORITY_PROPERTIES_KHR;
153 res.pNext = pNext;
154 res.priorityCount = static_cast<deUint32>(m_priorities.size());
155 for (auto i = start; i != m_priorities.end(); ++i)
156 res.priorities[std::distance(start, i)] = *i;
157 return res;
158 }
any(const QueueGlobalPriorities & other) const159 bool QueueGlobalPriorities::any (const QueueGlobalPriorities& other) const
160 {
161 for (auto i = other.m_priorities.begin(); i != other.m_priorities.end(); ++i)
162 if (any(*i)) return true;
163 return false;
164 }
all(const QueueGlobalPriorities & other) const165 bool QueueGlobalPriorities::all (const QueueGlobalPriorities& other) const
166 {
167 return m_priorities == other.m_priorities;
168 }
169
SpecialDevice(Context & ctx,VkQueueFlagBits transitionFrom,VkQueueFlagBits transitionTo,VkQueueGlobalPriorityKHR priorityFrom,VkQueueGlobalPriorityKHR priorityTo,bool enableProtected,bool enableSparseBinding)170 SpecialDevice::SpecialDevice (Context& ctx,
171 VkQueueFlagBits transitionFrom,
172 VkQueueFlagBits transitionTo,
173 VkQueueGlobalPriorityKHR priorityFrom,
174 VkQueueGlobalPriorityKHR priorityTo,
175 bool enableProtected,
176 bool enableSparseBinding)
177 : queueFamilyIndexFrom (m_queueFamilyIndexFrom)
178 , queueFamilyIndexTo (m_queueFamilyIndexTo)
179 , device (m_device)
180 , queueFrom (m_queueFrom)
181 , queueTo (m_queueTo)
182 , m_vkd (ctx.getDeviceInterface())
183 , m_transitionFrom (transitionFrom)
184 , m_transitionTo (transitionTo)
185 , m_queueFamilyIndexFrom (INVALID_UINT32)
186 , m_queueFamilyIndexTo (INVALID_UINT32)
187 , m_device (0)
188 , m_queueFrom (0)
189 , m_queueTo (0)
190 , m_allocator ()
191 , m_creationResult (VK_RESULT_MAX_ENUM)
192 {
193 const InstanceInterface& vki = ctx.getInstanceInterface();
194 const DeviceInterface& vkd = ctx.getDeviceInterface();
195 const VkPhysicalDevice dev = ctx.getPhysicalDevice();
196 const VkPhysicalDeviceMemoryProperties memoryProperties = getPhysicalDeviceMemoryProperties(vki, dev);
197
198 VkQueueFlags flagFrom = transitionFrom;
199 VkQueueFlags flagTo = transitionTo;
200 if (enableProtected)
201 {
202 flagFrom |= VK_QUEUE_PROTECTED_BIT;
203 flagTo |= VK_QUEUE_PROTECTED_BIT;
204 }
205 if (enableSparseBinding)
206 {
207 flagFrom |= VK_QUEUE_SPARSE_BINDING_BIT;
208 flagTo |= VK_QUEUE_SPARSE_BINDING_BIT;
209 }
210
211 m_queueFamilyIndexFrom = findQueueFamilyIndex(vki, dev, flagFrom, getColissionFlags(transitionFrom), true, { priorityFrom });
212 m_queueFamilyIndexTo = findQueueFamilyIndex(vki, dev, flagTo, getColissionFlags(transitionTo), true, { priorityTo });
213
214 DE_ASSERT(m_queueFamilyIndexFrom != INVALID_UINT32);
215 DE_ASSERT(m_queueFamilyIndexTo != INVALID_UINT32);
216
217 const float queuePriority = 1.0f;
218 VkDeviceQueueCreateInfo queueCreateInfos[2];
219 VkDeviceQueueGlobalPriorityCreateInfoKHR priorityCreateInfos[2];
220 {
221 priorityCreateInfos[0].sType = priorityCreateInfos[1].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_KHR;
222 priorityCreateInfos[0].pNext = priorityCreateInfos[1].pNext = nullptr;
223
224 queueCreateInfos[0].sType = queueCreateInfos[1].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
225 queueCreateInfos[0].flags = queueCreateInfos[1].flags = enableProtected ? VK_DEVICE_QUEUE_CREATE_PROTECTED_BIT : 0;
226 queueCreateInfos[0].queueCount = queueCreateInfos[1].queueCount = 1;
227
228 priorityCreateInfos[0].globalPriority = priorityFrom;
229 queueCreateInfos[0].pNext = &priorityCreateInfos[0];
230 queueCreateInfos[0].queueFamilyIndex = m_queueFamilyIndexFrom;
231
232 priorityCreateInfos[1].globalPriority = priorityTo;
233 queueCreateInfos[1].pNext = &priorityCreateInfos[1];
234 queueCreateInfos[1].queueFamilyIndex = m_queueFamilyIndexTo;
235
236 queueCreateInfos[0].pQueuePriorities = &queuePriority;
237 queueCreateInfos[1].pQueuePriorities = &queuePriority;
238 }
239
240 const VkPhysicalDeviceFeatures& deviceFeatures = ctx.getDeviceFeatures();
241
242 std::vector<const char*> extensions;
243 std::vector<std::string> filteredExtensions;
244 {
245 const deUint32 majorApi = VK_API_VERSION_MAJOR(ctx.getUsedApiVersion());
246 const deUint32 minorApi = VK_API_VERSION_MINOR(ctx.getUsedApiVersion());
247 std::vector<VkExtensionProperties> availableExtensions = enumerateDeviceExtensionProperties(vki, dev, nullptr);
248 const bool khrBufferAddress = availableExtensions.end() !=
249 std::find_if(availableExtensions.begin(), availableExtensions.end(),
250 [](const VkExtensionProperties& p) { return deStringEqual(p.extensionName, "VK_KHR_buffer_device_address"); });
251 for (const auto& ext : availableExtensions)
252 {
253 if (khrBufferAddress && deStringEqual(ext.extensionName, "VK_EXT_buffer_device_address"))
254 continue;
255 if (VK_API_VERSION_MAJOR(ext.specVersion) <= majorApi && VK_API_VERSION_MINOR(ext.specVersion) <= minorApi)
256 filteredExtensions.emplace_back(ext.extensionName);
257 }
258 extensions.resize(filteredExtensions.size());
259 std::transform(filteredExtensions.begin(), filteredExtensions.end(), extensions.begin(), [](const std::string& s) { return s.c_str(); });
260 }
261
262 VkDeviceCreateInfo deviceCreateInfo {};
263 deviceCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
264 deviceCreateInfo.queueCreateInfoCount = 2;
265 deviceCreateInfo.pQueueCreateInfos = queueCreateInfos;
266
267 deviceCreateInfo.pEnabledFeatures = &deviceFeatures;
268 deviceCreateInfo.enabledExtensionCount = static_cast<deUint32>(extensions.size());
269 deviceCreateInfo.ppEnabledExtensionNames = de::dataOrNull(extensions);
270 deviceCreateInfo.ppEnabledLayerNames = nullptr;
271 deviceCreateInfo.enabledLayerCount = 0;
272
273 m_creationResult = vki.createDevice(dev, &deviceCreateInfo, nullptr, &m_device);
274
275 if (VK_SUCCESS == m_creationResult && VkDevice(0) != m_device)
276 {
277 m_allocator = de::MovePtr<vk::Allocator>(new SimpleAllocator(vkd, m_device, memoryProperties));
278
279 if (enableProtected)
280 {
281 VkDeviceQueueInfo2 queueInfo{};
282 queueInfo.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_INFO_2;
283 queueInfo.flags = VK_DEVICE_QUEUE_CREATE_PROTECTED_BIT;
284 queueInfo.queueIndex = 0;
285
286 queueInfo.queueFamilyIndex = m_queueFamilyIndexFrom;
287 m_vkd.getDeviceQueue2(m_device, &queueInfo, &m_queueFrom);
288
289 queueInfo.queueFamilyIndex = m_queueFamilyIndexTo;
290 m_vkd.getDeviceQueue2(m_device, &queueInfo, &m_queueTo);
291 }
292 else
293 {
294 m_vkd.getDeviceQueue(m_device, m_queueFamilyIndexFrom, 0, &m_queueFrom);
295 m_vkd.getDeviceQueue(m_device, m_queueFamilyIndexTo, 0, &m_queueTo);
296 }
297 }
298 }
getColissionFlags(VkQueueFlagBits bits)299 VkQueueFlags SpecialDevice::getColissionFlags (VkQueueFlagBits bits)
300 {
301 switch (bits)
302 {
303 case VK_QUEUE_TRANSFER_BIT:
304 return (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT);
305 case VK_QUEUE_COMPUTE_BIT:
306 return VK_QUEUE_GRAPHICS_BIT;
307 case VK_QUEUE_GRAPHICS_BIT:
308 return 0;
309 default: break;
310 }
311 DE_ASSERT(false);
312 return 0;
313 }
getAllocator() const314 Allocator& SpecialDevice::getAllocator() const
315 {
316 return *m_allocator;
317 }
SpecialDevice(SpecialDevice && src)318 SpecialDevice::SpecialDevice (SpecialDevice&& src)
319 : queueFamilyIndexFrom (m_queueFamilyIndexFrom)
320 , queueFamilyIndexTo (m_queueFamilyIndexTo)
321 , device (m_device)
322 , queueFrom (m_queueFrom)
323 , queueTo (m_queueTo)
324 , m_vkd (src.m_vkd)
325 , m_transitionFrom (src.m_transitionFrom)
326 , m_transitionTo (src.m_transitionTo)
327 , m_queueFamilyIndexFrom (src.m_queueFamilyIndexFrom)
328 , m_queueFamilyIndexTo (src.m_queueFamilyIndexTo)
329 , m_device (src.m_device)
330 , m_queueFrom (src.m_queueFrom)
331 , m_queueTo (src.m_queueTo)
332 , m_allocator (src.m_allocator.release())
333 , m_creationResult (src.m_creationResult)
334 {
335 src.m_queueFamilyIndexFrom = INVALID_UINT32;
336 src.m_queueFamilyIndexTo = INVALID_UINT32;
337 src.m_device = VkDevice(0);
338 src.m_queueFrom = VkQueue(0);
339 src.m_queueTo = VkQueue(0);
340 }
~SpecialDevice()341 SpecialDevice::~SpecialDevice ()
342 {
343 if (VkDevice(0) != m_device)
344 {
345 m_vkd.destroyDevice (m_device, nullptr);
346 m_device = VkDevice(0);
347 }
348 }
isValid(VkResult & creationResult) const349 bool SpecialDevice::isValid (VkResult& creationResult) const
350 {
351 creationResult = m_creationResult;
352 return (VkDevice(0) != m_device);
353 }
354
BufferWithMemory(const vk::InstanceInterface & vki,const DeviceInterface & vkd,const vk::VkPhysicalDevice phys,const VkDevice device,Allocator & allocator,const VkBufferCreateInfo & bufferCreateInfo,const MemoryRequirement memoryRequirement,const VkQueue sparseQueue)355 BufferWithMemory::BufferWithMemory (const vk::InstanceInterface& vki,
356 const DeviceInterface& vkd,
357 const vk::VkPhysicalDevice phys,
358 const VkDevice device,
359 Allocator& allocator,
360 const VkBufferCreateInfo& bufferCreateInfo,
361 const MemoryRequirement memoryRequirement,
362 const VkQueue sparseQueue)
363 : m_amISparse ((bufferCreateInfo.flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) != 0)
364 , m_buffer (createBuffer(vkd, device, &bufferCreateInfo))
365 , m_requirements (getBufferMemoryRequirements(vkd, device, *m_buffer))
366 {
367 if (m_amISparse)
368 {
369 DE_ASSERT(sparseQueue != VkQueue(0));
370 const VkPhysicalDeviceMemoryProperties memoryProperties = getPhysicalDeviceMemoryProperties(vki, phys);
371 const deUint32 memoryTypeIndex = selectMatchingMemoryType(memoryProperties, m_requirements.memoryTypeBits, memoryRequirement);
372 const VkDeviceSize alignment = 0;
373 const VkDeviceSize lastChunkSize = m_requirements.size % m_requirements.alignment;
374 const uint32_t chunkCount = static_cast<uint32_t>(m_requirements.size / m_requirements.alignment + (lastChunkSize ? 1 : 0));
375 Move<VkFence> fence = createFence(vkd, device);
376
377 std::vector<VkSparseMemoryBind> bindings(chunkCount);
378
379 for (uint32_t i = 0; i < chunkCount; ++i)
380 {
381 VkMemoryAllocateInfo allocInfo{};
382 allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
383 allocInfo.pNext = nullptr;
384 allocInfo.allocationSize = m_requirements.alignment;
385 allocInfo.memoryTypeIndex = memoryTypeIndex;
386
387 de::MovePtr<Allocation> allocation = allocator.allocate(allocInfo, alignment /*unreferenced parameter*/);
388
389 VkSparseMemoryBind& binding = bindings[i];
390 binding.resourceOffset = m_requirements.alignment * i;
391 binding.size = m_requirements.alignment;
392 binding.memory = allocation->getMemory();
393 binding.memoryOffset = allocation->getOffset();
394 binding.flags = 0;
395
396 m_allocations.emplace_back(allocation.release());
397 }
398
399 VkSparseBufferMemoryBindInfo bindInfo{};
400 bindInfo.buffer = *m_buffer;
401 bindInfo.bindCount = chunkCount;
402 bindInfo.pBinds = de::dataOrNull(bindings);
403
404 VkBindSparseInfo sparseInfo{};
405 sparseInfo.sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO;
406 sparseInfo.pNext = nullptr;
407 sparseInfo.waitSemaphoreCount = 0;
408 sparseInfo.pWaitSemaphores = nullptr;
409 sparseInfo.bufferBindCount = 1;
410 sparseInfo.pBufferBinds = &bindInfo;
411 sparseInfo.imageOpaqueBindCount = 0;
412 sparseInfo.pImageOpaqueBinds = nullptr;
413 sparseInfo.imageBindCount = 0;
414 sparseInfo.pImageBinds = nullptr;
415 sparseInfo.signalSemaphoreCount = 0;
416 sparseInfo.pSignalSemaphores = nullptr;
417
418 VK_CHECK(vkd.queueBindSparse(sparseQueue, 1, &sparseInfo, *fence));
419 VK_CHECK(vkd.waitForFences(device, 1u, &fence.get(), DE_TRUE, ~0ull));
420 }
421 else
422 {
423 de::MovePtr<Allocation> allocation = allocator.allocate(m_requirements, memoryRequirement);
424 VK_CHECK(vkd.bindBufferMemory(device, *m_buffer, allocation->getMemory(), allocation->getOffset()));
425 m_allocations.emplace_back(allocation.release());
426 }
427 }
428
assertIAmSparse() const429 void BufferWithMemory::assertIAmSparse () const
430 {
431 if (m_amISparse) TCU_THROW(NotSupportedError, "Host access pointer not implemented for sparse buffers");
432 }
433
getHostPtr(void) const434 void* BufferWithMemory::getHostPtr (void) const
435 {
436 assertIAmSparse();
437 return m_allocations[0]->getHostPtr();
438 }
439
invalidateAlloc(const DeviceInterface & vk,const VkDevice device) const440 void BufferWithMemory::invalidateAlloc (const DeviceInterface& vk, const VkDevice device) const
441 {
442 assertIAmSparse();
443 ::vk::invalidateAlloc(vk, device, *m_allocations[0]);
444 }
445
flushAlloc(const DeviceInterface & vk,const VkDevice device) const446 void BufferWithMemory::flushAlloc (const DeviceInterface& vk, const VkDevice device) const
447 {
448 assertIAmSparse();
449 ::vk::flushAlloc(vk, device, *m_allocations[0]);
450 }
451
ImageWithMemory(const InstanceInterface & vki,const DeviceInterface & vkd,const VkPhysicalDevice phys,const VkDevice device,Allocator & allocator,const VkImageCreateInfo & imageCreateInfo,const VkQueue sparseQueue,const MemoryRequirement memoryRequirement)452 ImageWithMemory::ImageWithMemory (const InstanceInterface& vki,
453 const DeviceInterface& vkd,
454 const VkPhysicalDevice phys,
455 const VkDevice device,
456 Allocator& allocator,
457 const VkImageCreateInfo& imageCreateInfo,
458 const VkQueue sparseQueue,
459 const MemoryRequirement memoryRequirement)
460 : m_image(((imageCreateInfo.flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) != 0)
461 ? (new image::SparseImage(vkd, device, phys, vki, imageCreateInfo, sparseQueue, allocator, mapVkFormat(imageCreateInfo.format)))
462 : (new image::Image(vkd, device, allocator, imageCreateInfo, memoryRequirement)))
463 {
464 }
465
466 } // synchronization
467 } // vkt
468