1 // Copyright 2019 The Dawn Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #include "dawn_native/vulkan/ResourceMemoryAllocatorVk.h" 16 17 #include "common/Math.h" 18 #include "dawn_native/BuddyMemoryAllocator.h" 19 #include "dawn_native/ResourceHeapAllocator.h" 20 #include "dawn_native/vulkan/DeviceVk.h" 21 #include "dawn_native/vulkan/FencedDeleter.h" 22 #include "dawn_native/vulkan/ResourceHeapVk.h" 23 #include "dawn_native/vulkan/VulkanError.h" 24 25 namespace dawn_native { namespace vulkan { 26 27 namespace { 28 29 // TODO(crbug.com/dawn/849): This is a hardcoded heurstic to choose when to 30 // suballocate but it should ideally depend on the size of the memory heaps and other 31 // factors. 32 constexpr uint64_t kMaxSizeForSubAllocation = 4ull * 1024ull * 1024ull; // 4MiB 33 34 // Have each bucket of the buddy system allocate at least some resource of the maximum 35 // size 36 constexpr uint64_t kBuddyHeapsSize = 2 * kMaxSizeForSubAllocation; 37 38 } // anonymous namespace 39 40 // SingleTypeAllocator is a combination of a BuddyMemoryAllocator and its client and can 41 // service suballocation requests, but for a single Vulkan memory type. 42 43 class ResourceMemoryAllocator::SingleTypeAllocator : public ResourceHeapAllocator { 44 public: SingleTypeAllocator(Device * device,size_t memoryTypeIndex,VkDeviceSize memoryHeapSize)45 SingleTypeAllocator(Device* device, size_t memoryTypeIndex, VkDeviceSize memoryHeapSize) 46 : mDevice(device), 47 mMemoryTypeIndex(memoryTypeIndex), 48 mMemoryHeapSize(memoryHeapSize), 49 mPooledMemoryAllocator(this), 50 mBuddySystem( 51 // Round down to a power of 2 that's <= mMemoryHeapSize. This will always 52 // be a multiple of kBuddyHeapsSize because kBuddyHeapsSize is a power of 2. 53 uint64_t(1) << Log2(mMemoryHeapSize), 54 // Take the min in the very unlikely case the memory heap is tiny. 55 std::min(uint64_t(1) << Log2(mMemoryHeapSize), kBuddyHeapsSize), 56 &mPooledMemoryAllocator) { 57 ASSERT(IsPowerOfTwo(kBuddyHeapsSize)); 58 } 59 ~SingleTypeAllocator() override = default; 60 DestroyPool()61 void DestroyPool() { 62 mPooledMemoryAllocator.DestroyPool(); 63 } 64 AllocateMemory(uint64_t size,uint64_t alignment)65 ResultOrError<ResourceMemoryAllocation> AllocateMemory(uint64_t size, uint64_t alignment) { 66 return mBuddySystem.Allocate(size, alignment); 67 } 68 DeallocateMemory(const ResourceMemoryAllocation & allocation)69 void DeallocateMemory(const ResourceMemoryAllocation& allocation) { 70 mBuddySystem.Deallocate(allocation); 71 } 72 73 // Implementation of the MemoryAllocator interface to be a client of BuddyMemoryAllocator 74 AllocateResourceHeap(uint64_t size)75 ResultOrError<std::unique_ptr<ResourceHeapBase>> AllocateResourceHeap( 76 uint64_t size) override { 77 if (size > mMemoryHeapSize) { 78 return DAWN_OUT_OF_MEMORY_ERROR("Allocation size too large"); 79 } 80 81 VkMemoryAllocateInfo allocateInfo; 82 allocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; 83 allocateInfo.pNext = nullptr; 84 allocateInfo.allocationSize = size; 85 allocateInfo.memoryTypeIndex = mMemoryTypeIndex; 86 87 VkDeviceMemory allocatedMemory = VK_NULL_HANDLE; 88 89 // First check OOM that we want to surface to the application. 90 DAWN_TRY(CheckVkOOMThenSuccess( 91 mDevice->fn.AllocateMemory(mDevice->GetVkDevice(), &allocateInfo, nullptr, 92 &*allocatedMemory), 93 "vkAllocateMemory")); 94 95 ASSERT(allocatedMemory != VK_NULL_HANDLE); 96 return {std::make_unique<ResourceHeap>(allocatedMemory, mMemoryTypeIndex)}; 97 } 98 DeallocateResourceHeap(std::unique_ptr<ResourceHeapBase> allocation)99 void DeallocateResourceHeap(std::unique_ptr<ResourceHeapBase> allocation) override { 100 mDevice->GetFencedDeleter()->DeleteWhenUnused(ToBackend(allocation.get())->GetMemory()); 101 } 102 103 private: 104 Device* mDevice; 105 size_t mMemoryTypeIndex; 106 VkDeviceSize mMemoryHeapSize; 107 PooledResourceMemoryAllocator mPooledMemoryAllocator; 108 BuddyMemoryAllocator mBuddySystem; 109 }; 110 111 // Implementation of ResourceMemoryAllocator 112 ResourceMemoryAllocator(Device * device)113 ResourceMemoryAllocator::ResourceMemoryAllocator(Device* device) : mDevice(device) { 114 const VulkanDeviceInfo& info = mDevice->GetDeviceInfo(); 115 mAllocatorsPerType.reserve(info.memoryTypes.size()); 116 117 for (size_t i = 0; i < info.memoryTypes.size(); i++) { 118 mAllocatorsPerType.emplace_back(std::make_unique<SingleTypeAllocator>( 119 mDevice, i, info.memoryHeaps[info.memoryTypes[i].heapIndex].size)); 120 } 121 } 122 123 ResourceMemoryAllocator::~ResourceMemoryAllocator() = default; 124 Allocate(const VkMemoryRequirements & requirements,MemoryKind kind)125 ResultOrError<ResourceMemoryAllocation> ResourceMemoryAllocator::Allocate( 126 const VkMemoryRequirements& requirements, 127 MemoryKind kind) { 128 // The Vulkan spec guarantees at least on memory type is valid. 129 int memoryType = FindBestTypeIndex(requirements, kind); 130 ASSERT(memoryType >= 0); 131 132 VkDeviceSize size = requirements.size; 133 134 // Sub-allocate non-mappable resources because at the moment the mapped pointer 135 // is part of the resource and not the heap, which doesn't match the Vulkan model. 136 // TODO(crbug.com/dawn/849): allow sub-allocating mappable resources, maybe. 137 if (requirements.size < kMaxSizeForSubAllocation && kind != MemoryKind::LinearMappable) { 138 // When sub-allocating, Vulkan requires that we respect bufferImageGranularity. Some 139 // hardware puts information on the memory's page table entry and allocating a linear 140 // resource in the same page as a non-linear (aka opaque) resource can cause issues. 141 // Probably because some texture compression flags are stored on the page table entry, 142 // and allocating a linear resource removes these flags. 143 // 144 // Anyway, just to be safe we ask that all sub-allocated resources are allocated with at 145 // least this alignment. TODO(crbug.com/dawn/849): this is suboptimal because multiple 146 // linear (resp. opaque) resources can coexist in the same page. In particular Nvidia 147 // GPUs often use a granularity of 64k which will lead to a lot of wasted spec. Revisit 148 // with a more efficient algorithm later. 149 uint64_t alignment = 150 std::max(requirements.alignment, 151 mDevice->GetDeviceInfo().properties.limits.bufferImageGranularity); 152 153 ResourceMemoryAllocation subAllocation; 154 DAWN_TRY_ASSIGN(subAllocation, mAllocatorsPerType[memoryType]->AllocateMemory( 155 requirements.size, alignment)); 156 if (subAllocation.GetInfo().mMethod != AllocationMethod::kInvalid) { 157 return std::move(subAllocation); 158 } 159 } 160 161 // If sub-allocation failed, allocate memory just for it. 162 std::unique_ptr<ResourceHeapBase> resourceHeap; 163 DAWN_TRY_ASSIGN(resourceHeap, mAllocatorsPerType[memoryType]->AllocateResourceHeap(size)); 164 165 void* mappedPointer = nullptr; 166 if (kind == MemoryKind::LinearMappable) { 167 DAWN_TRY_WITH_CLEANUP( 168 CheckVkSuccess(mDevice->fn.MapMemory(mDevice->GetVkDevice(), 169 ToBackend(resourceHeap.get())->GetMemory(), 0, 170 size, 0, &mappedPointer), 171 "vkMapMemory"), 172 { 173 mAllocatorsPerType[memoryType]->DeallocateResourceHeap(std::move(resourceHeap)); 174 }); 175 } 176 177 AllocationInfo info; 178 info.mMethod = AllocationMethod::kDirect; 179 return ResourceMemoryAllocation(info, /*offset*/ 0, resourceHeap.release(), 180 static_cast<uint8_t*>(mappedPointer)); 181 } 182 Deallocate(ResourceMemoryAllocation * allocation)183 void ResourceMemoryAllocator::Deallocate(ResourceMemoryAllocation* allocation) { 184 switch (allocation->GetInfo().mMethod) { 185 // Some memory allocation can never be initialized, for example when wrapping 186 // swapchain VkImages with a Texture. 187 case AllocationMethod::kInvalid: 188 break; 189 190 // For direct allocation we can put the memory for deletion immediately and the fence 191 // deleter will make sure the resources are freed before the memory. 192 case AllocationMethod::kDirect: { 193 ResourceHeap* heap = ToBackend(allocation->GetResourceHeap()); 194 allocation->Invalidate(); 195 mDevice->GetFencedDeleter()->DeleteWhenUnused(heap->GetMemory()); 196 delete heap; 197 break; 198 } 199 200 // Suballocations aren't freed immediately, otherwise another resource allocation could 201 // happen just after that aliases the old one and would require a barrier. 202 // TODO(crbug.com/dawn/851): Maybe we can produce the correct barriers to reduce the 203 // latency to reclaim memory. 204 case AllocationMethod::kSubAllocated: 205 mSubAllocationsToDelete.Enqueue(*allocation, mDevice->GetPendingCommandSerial()); 206 break; 207 208 default: 209 UNREACHABLE(); 210 break; 211 } 212 213 // Invalidate the underlying resource heap in case the client accidentally 214 // calls DeallocateMemory again using the same allocation. 215 allocation->Invalidate(); 216 } 217 Tick(ExecutionSerial completedSerial)218 void ResourceMemoryAllocator::Tick(ExecutionSerial completedSerial) { 219 for (const ResourceMemoryAllocation& allocation : 220 mSubAllocationsToDelete.IterateUpTo(completedSerial)) { 221 ASSERT(allocation.GetInfo().mMethod == AllocationMethod::kSubAllocated); 222 size_t memoryType = ToBackend(allocation.GetResourceHeap())->GetMemoryType(); 223 224 mAllocatorsPerType[memoryType]->DeallocateMemory(allocation); 225 } 226 227 mSubAllocationsToDelete.ClearUpTo(completedSerial); 228 } 229 FindBestTypeIndex(VkMemoryRequirements requirements,MemoryKind kind)230 int ResourceMemoryAllocator::FindBestTypeIndex(VkMemoryRequirements requirements, 231 MemoryKind kind) { 232 const VulkanDeviceInfo& info = mDevice->GetDeviceInfo(); 233 bool mappable = kind == MemoryKind::LinearMappable; 234 235 // Find a suitable memory type for this allocation 236 int bestType = -1; 237 for (size_t i = 0; i < info.memoryTypes.size(); ++i) { 238 // Resource must support this memory type 239 if ((requirements.memoryTypeBits & (1 << i)) == 0) { 240 continue; 241 } 242 243 // Mappable resource must be host visible 244 if (mappable && 245 (info.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) { 246 continue; 247 } 248 249 // Mappable must also be host coherent. 250 if (mappable && 251 (info.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0) { 252 continue; 253 } 254 255 // Found the first candidate memory type 256 if (bestType == -1) { 257 bestType = static_cast<int>(i); 258 continue; 259 } 260 261 // For non-mappable resources, favor device local memory. 262 bool currentDeviceLocal = 263 info.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; 264 bool bestDeviceLocal = 265 info.memoryTypes[bestType].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; 266 if (!mappable && (currentDeviceLocal != bestDeviceLocal)) { 267 if (currentDeviceLocal) { 268 bestType = static_cast<int>(i); 269 } 270 continue; 271 } 272 273 // All things equal favor the memory in the biggest heap 274 VkDeviceSize bestTypeHeapSize = 275 info.memoryHeaps[info.memoryTypes[bestType].heapIndex].size; 276 VkDeviceSize candidateHeapSize = info.memoryHeaps[info.memoryTypes[i].heapIndex].size; 277 if (candidateHeapSize > bestTypeHeapSize) { 278 bestType = static_cast<int>(i); 279 continue; 280 } 281 } 282 283 return bestType; 284 } 285 DestroyPool()286 void ResourceMemoryAllocator::DestroyPool() { 287 for (auto& alloc : mAllocatorsPerType) { 288 alloc->DestroyPool(); 289 } 290 } 291 292 }} // namespace dawn_native::vulkan 293