1 //
2 // Copyright 2022 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // Suballocation.h:
7 // Defines class interface for BufferBlock and Suballocation and other related classes.
8 //
9
10 #ifndef LIBANGLE_RENDERER_VULKAN_SUBALLOCATION_H_
11 #define LIBANGLE_RENDERER_VULKAN_SUBALLOCATION_H_
12
13 #include "common/SimpleMutex.h"
14 #include "common/debug.h"
15 #include "libANGLE/angletypes.h"
16 #include "libANGLE/renderer/serial_utils.h"
17 #include "libANGLE/renderer/vulkan/vk_cache_utils.h"
18 #include "libANGLE/renderer/vulkan/vk_resource.h"
19 #include "libANGLE/renderer/vulkan/vk_utils.h"
20 #include "libANGLE/renderer/vulkan/vk_wrapper.h"
21
22 namespace rx
23 {
24 enum class MemoryAllocationType;
25
26 namespace vk
27 {
28 class Context;
29
30 // BufferBlock
31 class BufferBlock final : angle::NonCopyable
32 {
33 public:
34 BufferBlock();
35 BufferBlock(BufferBlock &&other);
36 ~BufferBlock();
37
38 void destroy(Renderer *renderer);
39 VkResult init(Context *context,
40 Buffer &buffer,
41 uint32_t memoryTypeIndex,
42 vma::VirtualBlockCreateFlags flags,
43 DeviceMemory &deviceMemory,
44 VkMemoryPropertyFlags memoryPropertyFlags,
45 VkDeviceSize size);
46 void initWithoutVirtualBlock(Context *context,
47 Buffer &buffer,
48 MemoryAllocationType memoryAllocationType,
49 uint32_t memoryTypeIndex,
50 DeviceMemory &deviceMemory,
51 VkMemoryPropertyFlags memoryPropertyFlags,
52 VkDeviceSize size,
53 VkDeviceSize allocatedBufferSize);
54
55 BufferBlock &operator=(BufferBlock &&other);
56
getBuffer()57 const Buffer &getBuffer() const { return mBuffer; }
getDeviceMemory()58 const DeviceMemory &getDeviceMemory() const { return mDeviceMemory; }
getDeviceMemory()59 DeviceMemory &getDeviceMemory() { return mDeviceMemory; }
getBufferSerial()60 BufferSerial getBufferSerial() const { return mSerial; }
61
62 VkMemoryPropertyFlags getMemoryPropertyFlags() const;
63 VkDeviceSize getMemorySize() const;
64
65 VkResult allocate(VkDeviceSize size,
66 VkDeviceSize alignment,
67 VmaVirtualAllocation *allocationOut,
68 VkDeviceSize *offsetOut);
69 void free(VmaVirtualAllocation allocation, VkDeviceSize offset);
70 VkBool32 isEmpty();
71
hasVirtualBlock()72 bool hasVirtualBlock() const { return mVirtualBlock.valid(); }
73 bool isHostVisible() const;
74 bool isCoherent() const;
75 bool isCached() const;
76 bool isMapped() const;
77 VkResult map(const VkDevice device);
78 void unmap(const VkDevice device);
79 uint8_t *getMappedMemory() const;
80
81 // This should be called whenever this found to be empty. The total number of count of empty is
82 // returned.
83 int32_t getAndIncrementEmptyCounter();
84 void calculateStats(vma::StatInfo *pStatInfo) const;
85
onNewDescriptorSet(const SharedDescriptorSetCacheKey & sharedCacheKey)86 void onNewDescriptorSet(const SharedDescriptorSetCacheKey &sharedCacheKey)
87 {
88 mDescriptorSetCacheManager.addKey(sharedCacheKey);
89 }
releaseAllCachedDescriptorSetCacheKeys(Renderer * renderer)90 void releaseAllCachedDescriptorSetCacheKeys(Renderer *renderer)
91 {
92 if (!mDescriptorSetCacheManager.empty())
93 {
94 mDescriptorSetCacheManager.releaseKeys(renderer);
95 }
96 }
97
98 private:
99 mutable angle::SimpleMutex mVirtualBlockMutex;
100 VirtualBlock mVirtualBlock;
101
102 Buffer mBuffer;
103 DeviceMemory mDeviceMemory;
104 VkMemoryPropertyFlags mMemoryPropertyFlags;
105
106 // Memory size that user of this object thinks we have.
107 VkDeviceSize mSize;
108 // Memory size that was actually allocated for this object.
109 VkDeviceSize mAllocatedBufferSize;
110 // Memory allocation type used for this object.
111 MemoryAllocationType mMemoryAllocationType;
112 // Memory type index used for the allocation. It can be used to determine the heap index.
113 uint32_t mMemoryTypeIndex;
114
115 uint8_t *mMappedMemory;
116 BufferSerial mSerial;
117 // Heuristic information for pruneEmptyBuffer. This tracks how many times (consecutively) this
118 // buffer block is found to be empty when pruneEmptyBuffer is called. This gets reset whenever
119 // it becomes non-empty.
120 int32_t mCountRemainsEmpty;
121 // Manages the descriptorSet cache that created with this BufferBlock.
122 DescriptorSetCacheManager mDescriptorSetCacheManager;
123 };
124 using BufferBlockPointer = std::unique_ptr<BufferBlock>;
125 using BufferBlockPointerVector = std::vector<BufferBlockPointer>;
126
127 class BufferBlockGarbageList final : angle::NonCopyable
128 {
129 public:
BufferBlockGarbageList()130 BufferBlockGarbageList() : mBufferBlockQueue(kInitialQueueCapacity) {}
~BufferBlockGarbageList()131 ~BufferBlockGarbageList() { ASSERT(mBufferBlockQueue.empty()); }
132
add(BufferBlock * bufferBlock)133 void add(BufferBlock *bufferBlock)
134 {
135 std::unique_lock<angle::SimpleMutex> lock(mMutex);
136 if (mBufferBlockQueue.full())
137 {
138 size_t newCapacity = mBufferBlockQueue.capacity() << 1;
139 mBufferBlockQueue.updateCapacity(newCapacity);
140 }
141 mBufferBlockQueue.push(bufferBlock);
142 }
143
pruneEmptyBufferBlocks(Renderer * renderer)144 void pruneEmptyBufferBlocks(Renderer *renderer)
145 {
146 if (!mBufferBlockQueue.empty())
147 {
148 std::unique_lock<angle::SimpleMutex> lock(mMutex);
149 size_t count = mBufferBlockQueue.size();
150 for (size_t i = 0; i < count; i++)
151 {
152 BufferBlock *block = mBufferBlockQueue.front();
153 mBufferBlockQueue.pop();
154 if (block->isEmpty())
155 {
156 block->destroy(renderer);
157 }
158 else
159 {
160 mBufferBlockQueue.push(block);
161 }
162 }
163 }
164 }
165
empty()166 bool empty() const { return mBufferBlockQueue.empty(); }
167
168 private:
169 static constexpr size_t kInitialQueueCapacity = 4;
170 angle::SimpleMutex mMutex;
171 angle::FixedQueue<BufferBlock *> mBufferBlockQueue;
172 };
173
174 // BufferSuballocation
175 class BufferSuballocation final : angle::NonCopyable
176 {
177 public:
178 BufferSuballocation();
179
180 BufferSuballocation(BufferSuballocation &&other);
181 BufferSuballocation &operator=(BufferSuballocation &&other);
182
183 void destroy(Renderer *renderer);
184
185 void init(BufferBlock *block,
186 VmaVirtualAllocation allocation,
187 VkDeviceSize offset,
188 VkDeviceSize size);
189 void initWithEntireBuffer(Context *context,
190 Buffer &buffer,
191 MemoryAllocationType memoryAllocationType,
192 uint32_t memoryTypeIndex,
193 DeviceMemory &deviceMemory,
194 VkMemoryPropertyFlags memoryPropertyFlags,
195 VkDeviceSize size,
196 VkDeviceSize allocatedBufferSize);
197
198 const Buffer &getBuffer() const;
199 VkDeviceSize getSize() const;
200 const DeviceMemory &getDeviceMemory() const;
201 VkMemoryMapFlags getMemoryPropertyFlags() const;
202 bool isHostVisible() const;
203 bool isCoherent() const;
204 bool isCached() const;
205 bool isMapped() const;
206 uint8_t *getMappedMemory() const;
207 void flush(const VkDevice &device);
208 void invalidate(const VkDevice &device);
209 VkDeviceSize getOffset() const;
210 bool valid() const;
211 VkResult map(Context *context);
212 BufferSerial getBlockSerial() const;
213 uint8_t *getBlockMemory() const;
214 VkDeviceSize getBlockMemorySize() const;
isSuballocated()215 bool isSuballocated() const { return mBufferBlock->hasVirtualBlock(); }
getBufferBlock()216 BufferBlock *getBufferBlock() const { return mBufferBlock; }
217
218 private:
219 // Only used by DynamicBuffer where DynamicBuffer does the actual suballocation and pass the
220 // offset/size to this object. Since DynamicBuffer does not have a VMA virtual allocator, they
221 // will be ignored at destroy time. The offset/size is set here mainly for easy retrieval when
222 // the BufferHelper object is passed around.
223 friend class BufferHelper;
224 void setOffsetAndSize(VkDeviceSize offset, VkDeviceSize size);
225
226 BufferBlock *mBufferBlock;
227 VmaVirtualAllocation mAllocation;
228 VkDeviceSize mOffset;
229 VkDeviceSize mSize;
230 };
231
232 class BufferSuballocationGarbage
233 {
234 public:
235 BufferSuballocationGarbage() = default;
BufferSuballocationGarbage(BufferSuballocationGarbage && other)236 BufferSuballocationGarbage(BufferSuballocationGarbage &&other)
237 : mLifetime(other.mLifetime),
238 mSuballocation(std::move(other.mSuballocation)),
239 mBuffer(std::move(other.mBuffer))
240 {}
241 BufferSuballocationGarbage &operator=(BufferSuballocationGarbage &&other)
242 {
243 mLifetime = other.mLifetime;
244 mSuballocation = std::move(other.mSuballocation);
245 mBuffer = std::move(other.mBuffer);
246 return *this;
247 }
BufferSuballocationGarbage(const ResourceUse & use,BufferSuballocation && suballocation,Buffer && buffer)248 BufferSuballocationGarbage(const ResourceUse &use,
249 BufferSuballocation &&suballocation,
250 Buffer &&buffer)
251 : mLifetime(use), mSuballocation(std::move(suballocation)), mBuffer(std::move(buffer))
252 {}
253 ~BufferSuballocationGarbage() = default;
254
255 bool destroyIfComplete(Renderer *renderer);
256 bool hasResourceUseSubmitted(Renderer *renderer) const;
getSize()257 VkDeviceSize getSize() const { return mSuballocation.getSize(); }
isSuballocated()258 bool isSuballocated() const { return mSuballocation.isSuballocated(); }
259
260 private:
261 ResourceUse mLifetime;
262 BufferSuballocation mSuballocation;
263 Buffer mBuffer;
264 };
265
266 // BufferBlock implementation.
getMemoryPropertyFlags()267 ANGLE_INLINE VkMemoryPropertyFlags BufferBlock::getMemoryPropertyFlags() const
268 {
269 return mMemoryPropertyFlags;
270 }
271
getMemorySize()272 ANGLE_INLINE VkDeviceSize BufferBlock::getMemorySize() const
273 {
274 return mSize;
275 }
276
isEmpty()277 ANGLE_INLINE VkBool32 BufferBlock::isEmpty()
278 {
279 std::unique_lock<angle::SimpleMutex> lock(mVirtualBlockMutex);
280 return vma::IsVirtualBlockEmpty(mVirtualBlock.getHandle());
281 }
282
isHostVisible()283 ANGLE_INLINE bool BufferBlock::isHostVisible() const
284 {
285 return (mMemoryPropertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0;
286 }
287
isCoherent()288 ANGLE_INLINE bool BufferBlock::isCoherent() const
289 {
290 return (mMemoryPropertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
291 }
292
isCached()293 ANGLE_INLINE bool BufferBlock::isCached() const
294 {
295 return (mMemoryPropertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0;
296 }
297
isMapped()298 ANGLE_INLINE bool BufferBlock::isMapped() const
299 {
300 return mMappedMemory != nullptr;
301 }
302
getMappedMemory()303 ANGLE_INLINE uint8_t *BufferBlock::getMappedMemory() const
304 {
305 ASSERT(mMappedMemory != nullptr);
306 return mMappedMemory;
307 }
308
309 // BufferSuballocation implementation.
BufferSuballocation()310 ANGLE_INLINE BufferSuballocation::BufferSuballocation()
311 : mBufferBlock(nullptr), mAllocation(VK_NULL_HANDLE), mOffset(0), mSize(0)
312 {}
313
BufferSuballocation(BufferSuballocation && other)314 ANGLE_INLINE BufferSuballocation::BufferSuballocation(BufferSuballocation &&other)
315 : BufferSuballocation()
316 {
317 *this = std::move(other);
318 }
319
320 ANGLE_INLINE BufferSuballocation &BufferSuballocation::operator=(BufferSuballocation &&other)
321 {
322 std::swap(mBufferBlock, other.mBufferBlock);
323 std::swap(mSize, other.mSize);
324 std::swap(mAllocation, other.mAllocation);
325 std::swap(mOffset, other.mOffset);
326 return *this;
327 }
328
valid()329 ANGLE_INLINE bool BufferSuballocation::valid() const
330 {
331 return mBufferBlock != nullptr;
332 }
333
destroy(Renderer * renderer)334 ANGLE_INLINE void BufferSuballocation::destroy(Renderer *renderer)
335 {
336 if (valid())
337 {
338 ASSERT(mBufferBlock);
339 if (mBufferBlock->hasVirtualBlock())
340 {
341 mBufferBlock->free(mAllocation, mOffset);
342 mBufferBlock = nullptr;
343 }
344 else
345 {
346 // When virtual block is invalid, this is the standalone buffer that are created by
347 // BufferSuballocation::initWithEntireBuffer call. In this case, vmaBufferSuballocation
348 // owns block, we must properly delete the block object.
349 mBufferBlock->destroy(renderer);
350 SafeDelete(mBufferBlock);
351 }
352 mAllocation = VK_NULL_HANDLE;
353 mOffset = 0;
354 mSize = 0;
355 }
356 }
357
init(BufferBlock * block,VmaVirtualAllocation allocation,VkDeviceSize offset,VkDeviceSize size)358 ANGLE_INLINE void BufferSuballocation::init(BufferBlock *block,
359 VmaVirtualAllocation allocation,
360 VkDeviceSize offset,
361 VkDeviceSize size)
362 {
363 ASSERT(!valid());
364 ASSERT(block != nullptr);
365 ASSERT(allocation != VK_NULL_HANDLE);
366 ASSERT(offset != VK_WHOLE_SIZE);
367 mBufferBlock = block;
368 mAllocation = allocation;
369 mOffset = offset;
370 mSize = size;
371 }
372
initWithEntireBuffer(Context * context,Buffer & buffer,MemoryAllocationType memoryAllocationType,uint32_t memoryTypeIndex,DeviceMemory & deviceMemory,VkMemoryPropertyFlags memoryPropertyFlags,VkDeviceSize size,VkDeviceSize allocatedBufferSize)373 ANGLE_INLINE void BufferSuballocation::initWithEntireBuffer(
374 Context *context,
375 Buffer &buffer,
376 MemoryAllocationType memoryAllocationType,
377 uint32_t memoryTypeIndex,
378 DeviceMemory &deviceMemory,
379 VkMemoryPropertyFlags memoryPropertyFlags,
380 VkDeviceSize size,
381 VkDeviceSize allocatedBufferSize)
382 {
383 ASSERT(!valid());
384
385 std::unique_ptr<BufferBlock> block = std::make_unique<BufferBlock>();
386 block->initWithoutVirtualBlock(context, buffer, memoryAllocationType, memoryTypeIndex,
387 deviceMemory, memoryPropertyFlags, size, allocatedBufferSize);
388
389 mBufferBlock = block.release();
390 mAllocation = VK_NULL_HANDLE;
391 mOffset = 0;
392 mSize = mBufferBlock->getMemorySize();
393 }
394
getBuffer()395 ANGLE_INLINE const Buffer &BufferSuballocation::getBuffer() const
396 {
397 return mBufferBlock->getBuffer();
398 }
399
getSize()400 ANGLE_INLINE VkDeviceSize BufferSuballocation::getSize() const
401 {
402 return mSize;
403 }
404
getDeviceMemory()405 ANGLE_INLINE const DeviceMemory &BufferSuballocation::getDeviceMemory() const
406 {
407 return mBufferBlock->getDeviceMemory();
408 }
409
getMemoryPropertyFlags()410 ANGLE_INLINE VkMemoryMapFlags BufferSuballocation::getMemoryPropertyFlags() const
411 {
412 return mBufferBlock->getMemoryPropertyFlags();
413 }
414
isHostVisible()415 ANGLE_INLINE bool BufferSuballocation::isHostVisible() const
416 {
417 return mBufferBlock->isHostVisible();
418 }
isCoherent()419 ANGLE_INLINE bool BufferSuballocation::isCoherent() const
420 {
421 return mBufferBlock->isCoherent();
422 }
isCached()423 ANGLE_INLINE bool BufferSuballocation::isCached() const
424 {
425 return mBufferBlock->isCached();
426 }
isMapped()427 ANGLE_INLINE bool BufferSuballocation::isMapped() const
428 {
429 return mBufferBlock->isMapped();
430 }
getMappedMemory()431 ANGLE_INLINE uint8_t *BufferSuballocation::getMappedMemory() const
432 {
433 return mBufferBlock->getMappedMemory() + getOffset();
434 }
435
flush(const VkDevice & device)436 ANGLE_INLINE void BufferSuballocation::flush(const VkDevice &device)
437 {
438 if (!isCoherent())
439 {
440 VkMappedMemoryRange mappedRange = {};
441 mappedRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
442 mappedRange.memory = mBufferBlock->getDeviceMemory().getHandle();
443 mappedRange.offset = getOffset();
444 mappedRange.size = mSize;
445 mBufferBlock->getDeviceMemory().flush(device, mappedRange);
446 }
447 }
448
invalidate(const VkDevice & device)449 ANGLE_INLINE void BufferSuballocation::invalidate(const VkDevice &device)
450 {
451 if (!isCoherent())
452 {
453 VkMappedMemoryRange mappedRange = {};
454 mappedRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
455 mappedRange.memory = mBufferBlock->getDeviceMemory().getHandle();
456 mappedRange.offset = getOffset();
457 mappedRange.size = mSize;
458 mBufferBlock->getDeviceMemory().invalidate(device, mappedRange);
459 }
460 }
461
getOffset()462 ANGLE_INLINE VkDeviceSize BufferSuballocation::getOffset() const
463 {
464 return mOffset;
465 }
466
setOffsetAndSize(VkDeviceSize offset,VkDeviceSize size)467 ANGLE_INLINE void BufferSuballocation::setOffsetAndSize(VkDeviceSize offset, VkDeviceSize size)
468 {
469 mOffset = offset;
470 mSize = size;
471 }
472
getBlockMemory()473 ANGLE_INLINE uint8_t *BufferSuballocation::getBlockMemory() const
474 {
475 return mBufferBlock->getMappedMemory();
476 }
getBlockMemorySize()477 ANGLE_INLINE VkDeviceSize BufferSuballocation::getBlockMemorySize() const
478 {
479 return mBufferBlock->getMemorySize();
480 }
getBlockSerial()481 ANGLE_INLINE BufferSerial BufferSuballocation::getBlockSerial() const
482 {
483 ASSERT(valid());
484 return mBufferBlock->getBufferSerial();
485 }
486 } // namespace vk
487 } // namespace rx
488
489 #endif // LIBANGLE_RENDERER_VULKAN_SUBALLOCATION_H_
490