1 //
2 // Copyright 2022 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // Suballocation.h:
7 // Defines class interface for BufferBlock and Suballocation and other related classes.
8 //
9
10 #ifndef LIBANGLE_RENDERER_VULKAN_SUBALLOCATION_H_
11 #define LIBANGLE_RENDERER_VULKAN_SUBALLOCATION_H_
12
13 #include "common/debug.h"
14 #include "libANGLE/angletypes.h"
15 #include "libANGLE/renderer/serial_utils.h"
16 #include "libANGLE/renderer/vulkan/ResourceVk.h"
17 #include "libANGLE/renderer/vulkan/vk_cache_utils.h"
18 #include "libANGLE/renderer/vulkan/vk_utils.h"
19 #include "libANGLE/renderer/vulkan/vk_wrapper.h"
20
21 namespace rx
22 {
23 class RendererVk;
24 enum class MemoryAllocationType;
25
26 namespace vk
27 {
28 class Context;
29
30 // BufferBlock
31 class BufferBlock final : angle::NonCopyable
32 {
33 public:
34 BufferBlock();
35 BufferBlock(BufferBlock &&other);
36 ~BufferBlock();
37
38 void destroy(RendererVk *renderer);
39 angle::Result init(Context *context,
40 Buffer &buffer,
41 uint32_t memoryTypeIndex,
42 vma::VirtualBlockCreateFlags flags,
43 DeviceMemory &deviceMemory,
44 VkMemoryPropertyFlags memoryPropertyFlags,
45 VkDeviceSize size);
46 void initWithoutVirtualBlock(Context *context,
47 Buffer &buffer,
48 MemoryAllocationType memoryAllocationType,
49 uint32_t memoryTypeIndex,
50 DeviceMemory &deviceMemory,
51 VkMemoryPropertyFlags memoryPropertyFlags,
52 VkDeviceSize size,
53 VkDeviceSize allocatedBufferSize);
54
55 BufferBlock &operator=(BufferBlock &&other);
56
getBuffer()57 const Buffer &getBuffer() const { return mBuffer; }
getDeviceMemory()58 const DeviceMemory &getDeviceMemory() const { return mDeviceMemory; }
getDeviceMemory()59 DeviceMemory &getDeviceMemory() { return mDeviceMemory; }
getBufferSerial()60 BufferSerial getBufferSerial() const { return mSerial; }
61
62 VkMemoryPropertyFlags getMemoryPropertyFlags() const;
63 VkDeviceSize getMemorySize() const;
64
65 VkResult allocate(VkDeviceSize size,
66 VkDeviceSize alignment,
67 VmaVirtualAllocation *allocationOut,
68 VkDeviceSize *offsetOut);
69 void free(VmaVirtualAllocation allocation, VkDeviceSize offset);
70 VkBool32 isEmpty();
71
hasVirtualBlock()72 bool hasVirtualBlock() const { return mVirtualBlock.valid(); }
73 bool isHostVisible() const;
74 bool isCoherent() const;
75 bool isMapped() const;
76 VkResult map(const VkDevice device);
77 void unmap(const VkDevice device);
78 uint8_t *getMappedMemory() const;
79
80 // This should be called whenever this found to be empty. The total number of count of empty is
81 // returned.
82 int32_t getAndIncrementEmptyCounter();
83 void calculateStats(vma::StatInfo *pStatInfo) const;
84
onNewDescriptorSet(const SharedDescriptorSetCacheKey & sharedCacheKey)85 void onNewDescriptorSet(const SharedDescriptorSetCacheKey &sharedCacheKey)
86 {
87 mDescriptorSetCacheManager.addKey(sharedCacheKey);
88 }
89
90 private:
91 mutable std::mutex mVirtualBlockMutex;
92 VirtualBlock mVirtualBlock;
93
94 Buffer mBuffer;
95 DeviceMemory mDeviceMemory;
96 VkMemoryPropertyFlags mMemoryPropertyFlags;
97
98 // Memory size that user of this object thinks we have.
99 VkDeviceSize mSize;
100 // Memory size that was actually allocated for this object.
101 VkDeviceSize mAllocatedBufferSize;
102 // Memory allocation type used for this object.
103 MemoryAllocationType mMemoryAllocationType;
104 // Memory type index used for the allocation. It can be used to determine the heap index.
105 uint32_t mMemoryTypeIndex;
106
107 uint8_t *mMappedMemory;
108 BufferSerial mSerial;
109 // Heuristic information for pruneEmptyBuffer. This tracks how many times (consecutively) this
110 // buffer block is found to be empty when pruneEmptyBuffer is called. This gets reset whenever
111 // it becomes non-empty.
112 int32_t mCountRemainsEmpty;
113 // Manages the descriptorSet cache that created with this BufferBlock.
114 DescriptorSetCacheManager mDescriptorSetCacheManager;
115 };
116 using BufferBlockPointerVector = std::vector<std::unique_ptr<BufferBlock>>;
117
118 // BufferSuballocation
119 class BufferSuballocation final : angle::NonCopyable
120 {
121 public:
122 BufferSuballocation();
123
124 BufferSuballocation(BufferSuballocation &&other);
125 BufferSuballocation &operator=(BufferSuballocation &&other);
126
127 void destroy(RendererVk *renderer);
128
129 void init(BufferBlock *block,
130 VmaVirtualAllocation allocation,
131 VkDeviceSize offset,
132 VkDeviceSize size);
133 void initWithEntireBuffer(Context *context,
134 Buffer &buffer,
135 MemoryAllocationType memoryAllocationType,
136 uint32_t memoryTypeIndex,
137 DeviceMemory &deviceMemory,
138 VkMemoryPropertyFlags memoryPropertyFlags,
139 VkDeviceSize size,
140 VkDeviceSize allocatedBufferSize);
141
142 const Buffer &getBuffer() const;
143 VkDeviceSize getSize() const;
144 const DeviceMemory &getDeviceMemory() const;
145 VkMemoryMapFlags getMemoryPropertyFlags() const;
146 bool isHostVisible() const;
147 bool isCoherent() const;
148 bool isMapped() const;
149 uint8_t *getMappedMemory() const;
150 void flush(const VkDevice &device);
151 void invalidate(const VkDevice &device);
152 VkDeviceSize getOffset() const;
153 bool valid() const;
154 VkResult map(Context *context);
155 BufferSerial getBlockSerial() const;
156 uint8_t *getBlockMemory() const;
157 VkDeviceSize getBlockMemorySize() const;
isSuballocated()158 bool isSuballocated() const { return mBufferBlock->hasVirtualBlock(); }
getBufferBlock()159 BufferBlock *getBufferBlock() const { return mBufferBlock; }
160
161 private:
162 // Only used by DynamicBuffer where DynamicBuffer does the actual suballocation and pass the
163 // offset/size to this object. Since DynamicBuffer does not have a VMA virtual allocator, they
164 // will be ignored at destroy time. The offset/size is set here mainly for easy retrieval when
165 // the BufferHelper object is passed around.
166 friend class BufferHelper;
167 void setOffsetAndSize(VkDeviceSize offset, VkDeviceSize size);
168
169 BufferBlock *mBufferBlock;
170 VmaVirtualAllocation mAllocation;
171 VkDeviceSize mOffset;
172 VkDeviceSize mSize;
173 };
174
175 class SharedBufferSuballocationGarbage
176 {
177 public:
178 SharedBufferSuballocationGarbage() = default;
SharedBufferSuballocationGarbage(SharedBufferSuballocationGarbage && other)179 SharedBufferSuballocationGarbage(SharedBufferSuballocationGarbage &&other)
180 : mLifetime(other.mLifetime),
181 mSuballocation(std::move(other.mSuballocation)),
182 mBuffer(std::move(other.mBuffer))
183 {}
SharedBufferSuballocationGarbage(const ResourceUse & use,BufferSuballocation && suballocation,Buffer && buffer)184 SharedBufferSuballocationGarbage(const ResourceUse &use,
185 BufferSuballocation &&suballocation,
186 Buffer &&buffer)
187 : mLifetime(use), mSuballocation(std::move(suballocation)), mBuffer(std::move(buffer))
188 {}
189 ~SharedBufferSuballocationGarbage() = default;
190
191 bool destroyIfComplete(RendererVk *renderer);
192 bool hasResourceUseSubmitted(RendererVk *renderer) const;
getSize()193 VkDeviceSize getSize() const { return mSuballocation.getSize(); }
isSuballocated()194 bool isSuballocated() const { return mSuballocation.isSuballocated(); }
195
196 private:
197 ResourceUse mLifetime;
198 BufferSuballocation mSuballocation;
199 Buffer mBuffer;
200 };
201 using SharedBufferSuballocationGarbageList = std::queue<SharedBufferSuballocationGarbage>;
202
203 // BufferBlock implementation.
getMemoryPropertyFlags()204 ANGLE_INLINE VkMemoryPropertyFlags BufferBlock::getMemoryPropertyFlags() const
205 {
206 return mMemoryPropertyFlags;
207 }
208
getMemorySize()209 ANGLE_INLINE VkDeviceSize BufferBlock::getMemorySize() const
210 {
211 return mSize;
212 }
213
isEmpty()214 ANGLE_INLINE VkBool32 BufferBlock::isEmpty()
215 {
216 std::unique_lock<std::mutex> lock(mVirtualBlockMutex);
217 return vma::IsVirtualBlockEmpty(mVirtualBlock.getHandle());
218 }
219
isHostVisible()220 ANGLE_INLINE bool BufferBlock::isHostVisible() const
221 {
222 return (mMemoryPropertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0;
223 }
224
isCoherent()225 ANGLE_INLINE bool BufferBlock::isCoherent() const
226 {
227 return (mMemoryPropertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
228 }
229
isMapped()230 ANGLE_INLINE bool BufferBlock::isMapped() const
231 {
232 return mMappedMemory != nullptr;
233 }
234
getMappedMemory()235 ANGLE_INLINE uint8_t *BufferBlock::getMappedMemory() const
236 {
237 ASSERT(mMappedMemory != nullptr);
238 return mMappedMemory;
239 }
240
241 // BufferSuballocation implementation.
BufferSuballocation()242 ANGLE_INLINE BufferSuballocation::BufferSuballocation()
243 : mBufferBlock(nullptr), mAllocation(VK_NULL_HANDLE), mOffset(0), mSize(0)
244 {}
245
BufferSuballocation(BufferSuballocation && other)246 ANGLE_INLINE BufferSuballocation::BufferSuballocation(BufferSuballocation &&other)
247 : BufferSuballocation()
248 {
249 *this = std::move(other);
250 }
251
252 ANGLE_INLINE BufferSuballocation &BufferSuballocation::operator=(BufferSuballocation &&other)
253 {
254 std::swap(mBufferBlock, other.mBufferBlock);
255 std::swap(mSize, other.mSize);
256 std::swap(mAllocation, other.mAllocation);
257 std::swap(mOffset, other.mOffset);
258 return *this;
259 }
260
valid()261 ANGLE_INLINE bool BufferSuballocation::valid() const
262 {
263 return mBufferBlock != nullptr;
264 }
265
destroy(RendererVk * renderer)266 ANGLE_INLINE void BufferSuballocation::destroy(RendererVk *renderer)
267 {
268 if (valid())
269 {
270 ASSERT(mBufferBlock);
271 if (mBufferBlock->hasVirtualBlock())
272 {
273 mBufferBlock->free(mAllocation, mOffset);
274 mBufferBlock = nullptr;
275 }
276 else
277 {
278 // When virtual block is invalid, this is the standalone buffer that are created by
279 // BufferSuballocation::initWithEntireBuffer call. In this case, vmaBufferSuballocation
280 // owns block, we must properly delete the block object.
281 mBufferBlock->destroy(renderer);
282 SafeDelete(mBufferBlock);
283 }
284 mAllocation = VK_NULL_HANDLE;
285 mOffset = 0;
286 mSize = 0;
287 }
288 }
289
init(BufferBlock * block,VmaVirtualAllocation allocation,VkDeviceSize offset,VkDeviceSize size)290 ANGLE_INLINE void BufferSuballocation::init(BufferBlock *block,
291 VmaVirtualAllocation allocation,
292 VkDeviceSize offset,
293 VkDeviceSize size)
294 {
295 ASSERT(!valid());
296 ASSERT(block != nullptr);
297 #if ANGLE_VMA_VERSION >= 3000000
298 ASSERT(allocation != VK_NULL_HANDLE);
299 #endif // ANGLE_VMA_VERSION >= 3000000
300 ASSERT(offset != VK_WHOLE_SIZE);
301 mBufferBlock = block;
302 mAllocation = allocation;
303 mOffset = offset;
304 mSize = size;
305 }
306
initWithEntireBuffer(Context * context,Buffer & buffer,MemoryAllocationType memoryAllocationType,uint32_t memoryTypeIndex,DeviceMemory & deviceMemory,VkMemoryPropertyFlags memoryPropertyFlags,VkDeviceSize size,VkDeviceSize allocatedBufferSize)307 ANGLE_INLINE void BufferSuballocation::initWithEntireBuffer(
308 Context *context,
309 Buffer &buffer,
310 MemoryAllocationType memoryAllocationType,
311 uint32_t memoryTypeIndex,
312 DeviceMemory &deviceMemory,
313 VkMemoryPropertyFlags memoryPropertyFlags,
314 VkDeviceSize size,
315 VkDeviceSize allocatedBufferSize)
316 {
317 ASSERT(!valid());
318
319 std::unique_ptr<BufferBlock> block = std::make_unique<BufferBlock>();
320 block->initWithoutVirtualBlock(context, buffer, memoryAllocationType, memoryTypeIndex,
321 deviceMemory, memoryPropertyFlags, size, allocatedBufferSize);
322
323 mBufferBlock = block.release();
324 mAllocation = VK_NULL_HANDLE;
325 mOffset = 0;
326 mSize = mBufferBlock->getMemorySize();
327 }
328
getBuffer()329 ANGLE_INLINE const Buffer &BufferSuballocation::getBuffer() const
330 {
331 return mBufferBlock->getBuffer();
332 }
333
getSize()334 ANGLE_INLINE VkDeviceSize BufferSuballocation::getSize() const
335 {
336 return mSize;
337 }
338
getDeviceMemory()339 ANGLE_INLINE const DeviceMemory &BufferSuballocation::getDeviceMemory() const
340 {
341 return mBufferBlock->getDeviceMemory();
342 }
343
getMemoryPropertyFlags()344 ANGLE_INLINE VkMemoryMapFlags BufferSuballocation::getMemoryPropertyFlags() const
345 {
346 return mBufferBlock->getMemoryPropertyFlags();
347 }
348
isHostVisible()349 ANGLE_INLINE bool BufferSuballocation::isHostVisible() const
350 {
351 return mBufferBlock->isHostVisible();
352 }
isCoherent()353 ANGLE_INLINE bool BufferSuballocation::isCoherent() const
354 {
355 return mBufferBlock->isCoherent();
356 }
isMapped()357 ANGLE_INLINE bool BufferSuballocation::isMapped() const
358 {
359 return mBufferBlock->isMapped();
360 }
getMappedMemory()361 ANGLE_INLINE uint8_t *BufferSuballocation::getMappedMemory() const
362 {
363 return mBufferBlock->getMappedMemory() + getOffset();
364 }
365
flush(const VkDevice & device)366 ANGLE_INLINE void BufferSuballocation::flush(const VkDevice &device)
367 {
368 if (!isCoherent())
369 {
370 VkMappedMemoryRange mappedRange = {};
371 mappedRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
372 mappedRange.memory = mBufferBlock->getDeviceMemory().getHandle();
373 mappedRange.offset = getOffset();
374 mappedRange.size = mSize;
375 mBufferBlock->getDeviceMemory().flush(device, mappedRange);
376 }
377 }
378
invalidate(const VkDevice & device)379 ANGLE_INLINE void BufferSuballocation::invalidate(const VkDevice &device)
380 {
381 if (!isCoherent())
382 {
383 VkMappedMemoryRange mappedRange = {};
384 mappedRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
385 mappedRange.memory = mBufferBlock->getDeviceMemory().getHandle();
386 mappedRange.offset = getOffset();
387 mappedRange.size = mSize;
388 mBufferBlock->getDeviceMemory().invalidate(device, mappedRange);
389 }
390 }
391
getOffset()392 ANGLE_INLINE VkDeviceSize BufferSuballocation::getOffset() const
393 {
394 return mOffset;
395 }
396
setOffsetAndSize(VkDeviceSize offset,VkDeviceSize size)397 ANGLE_INLINE void BufferSuballocation::setOffsetAndSize(VkDeviceSize offset, VkDeviceSize size)
398 {
399 mOffset = offset;
400 mSize = size;
401 }
402
getBlockMemory()403 ANGLE_INLINE uint8_t *BufferSuballocation::getBlockMemory() const
404 {
405 return mBufferBlock->getMappedMemory();
406 }
getBlockMemorySize()407 ANGLE_INLINE VkDeviceSize BufferSuballocation::getBlockMemorySize() const
408 {
409 return mBufferBlock->getMemorySize();
410 }
getBlockSerial()411 ANGLE_INLINE BufferSerial BufferSuballocation::getBlockSerial() const
412 {
413 ASSERT(valid());
414 return mBufferBlock->getBufferSerial();
415 }
416 } // namespace vk
417 } // namespace rx
418
419 #endif // LIBANGLE_RENDERER_VULKAN_SUBALLOCATION_H_
420