1 // Copyright 2019 The Dawn Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #include "dawn_native/vulkan/ResourceMemoryAllocatorVk.h" 16 17 #include "common/Math.h" 18 #include "dawn_native/BuddyMemoryAllocator.h" 19 #include "dawn_native/ResourceHeapAllocator.h" 20 #include "dawn_native/vulkan/DeviceVk.h" 21 #include "dawn_native/vulkan/FencedDeleter.h" 22 #include "dawn_native/vulkan/ResourceHeapVk.h" 23 #include "dawn_native/vulkan/VulkanError.h" 24 25 namespace dawn_native { namespace vulkan { 26 27 namespace { 28 29 // TODO(crbug.com/dawn/849): This is a hardcoded heurstic to choose when to 30 // suballocate but it should ideally depend on the size of the memory heaps and other 31 // factors. 32 constexpr uint64_t kMaxSizeForSubAllocation = 4ull * 1024ull * 1024ull; // 4MiB 33 34 // Have each bucket of the buddy system allocate at least some resource of the maximum 35 // size 36 constexpr uint64_t kBuddyHeapsSize = 2 * kMaxSizeForSubAllocation; 37 38 } // anonymous namespace 39 40 // SingleTypeAllocator is a combination of a BuddyMemoryAllocator and its client and can 41 // service suballocation requests, but for a single Vulkan memory type. 42 43 class ResourceMemoryAllocator::SingleTypeAllocator : public ResourceHeapAllocator { 44 public: SingleTypeAllocator(Device * device,size_t memoryTypeIndex,VkDeviceSize memoryHeapSize)45 SingleTypeAllocator(Device* device, size_t memoryTypeIndex, VkDeviceSize memoryHeapSize) 46 : mDevice(device), 47 mMemoryTypeIndex(memoryTypeIndex), 48 mMemoryHeapSize(memoryHeapSize), 49 mPooledMemoryAllocator(this), 50 mBuddySystem( 51 // Round down to a power of 2 that's <= mMemoryHeapSize. This will always 52 // be a multiple of kBuddyHeapsSize because kBuddyHeapsSize is a power of 2. 53 uint64_t(1) << Log2(mMemoryHeapSize), 54 // Take the min in the very unlikely case the memory heap is tiny. 55 std::min(uint64_t(1) << Log2(mMemoryHeapSize), kBuddyHeapsSize), 56 &mPooledMemoryAllocator) { 57 ASSERT(IsPowerOfTwo(kBuddyHeapsSize)); 58 } 59 ~SingleTypeAllocator() override = default; 60 DestroyPool()61 void DestroyPool() { 62 mPooledMemoryAllocator.DestroyPool(); 63 } 64 AllocateMemory(uint64_t size,uint64_t alignment)65 ResultOrError<ResourceMemoryAllocation> AllocateMemory(uint64_t size, uint64_t alignment) { 66 return mBuddySystem.Allocate(size, alignment); 67 } 68 DeallocateMemory(const ResourceMemoryAllocation & allocation)69 void DeallocateMemory(const ResourceMemoryAllocation& allocation) { 70 mBuddySystem.Deallocate(allocation); 71 } 72 73 // Implementation of the MemoryAllocator interface to be a client of BuddyMemoryAllocator 74 AllocateResourceHeap(uint64_t size)75 ResultOrError<std::unique_ptr<ResourceHeapBase>> AllocateResourceHeap( 76 uint64_t size) override { 77 if (size > mMemoryHeapSize) { 78 return DAWN_OUT_OF_MEMORY_ERROR("Allocation size too large"); 79 } 80 81 VkMemoryAllocateInfo allocateInfo; 82 allocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; 83 allocateInfo.pNext = nullptr; 84 allocateInfo.allocationSize = size; 85 allocateInfo.memoryTypeIndex = mMemoryTypeIndex; 86 87 VkDeviceMemory allocatedMemory = VK_NULL_HANDLE; 88 89 // First check OOM that we want to surface to the application. 90 DAWN_TRY(CheckVkOOMThenSuccess( 91 mDevice->fn.AllocateMemory(mDevice->GetVkDevice(), &allocateInfo, nullptr, 92 &*allocatedMemory), 93 "vkAllocateMemory")); 94 95 ASSERT(allocatedMemory != VK_NULL_HANDLE); 96 return {std::make_unique<ResourceHeap>(allocatedMemory, mMemoryTypeIndex)}; 97 } 98 DeallocateResourceHeap(std::unique_ptr<ResourceHeapBase> allocation)99 void DeallocateResourceHeap(std::unique_ptr<ResourceHeapBase> allocation) override { 100 mDevice->GetFencedDeleter()->DeleteWhenUnused(ToBackend(allocation.get())->GetMemory()); 101 } 102 103 private: 104 Device* mDevice; 105 size_t mMemoryTypeIndex; 106 VkDeviceSize mMemoryHeapSize; 107 PooledResourceMemoryAllocator mPooledMemoryAllocator; 108 BuddyMemoryAllocator mBuddySystem; 109 }; 110 111 // Implementation of ResourceMemoryAllocator 112 ResourceMemoryAllocator(Device * device)113 ResourceMemoryAllocator::ResourceMemoryAllocator(Device* device) : mDevice(device) { 114 const VulkanDeviceInfo& info = mDevice->GetDeviceInfo(); 115 mAllocatorsPerType.reserve(info.memoryTypes.size()); 116 117 for (size_t i = 0; i < info.memoryTypes.size(); i++) { 118 mAllocatorsPerType.emplace_back(std::make_unique<SingleTypeAllocator>( 119 mDevice, i, info.memoryHeaps[info.memoryTypes[i].heapIndex].size)); 120 } 121 } 122 123 ResourceMemoryAllocator::~ResourceMemoryAllocator() = default; 124 Allocate(const VkMemoryRequirements & requirements,MemoryKind kind)125 ResultOrError<ResourceMemoryAllocation> ResourceMemoryAllocator::Allocate( 126 const VkMemoryRequirements& requirements, 127 MemoryKind kind) { 128 // The Vulkan spec guarantees at least on memory type is valid. 129 int memoryType = FindBestTypeIndex(requirements, kind); 130 ASSERT(memoryType >= 0); 131 132 VkDeviceSize size = requirements.size; 133 134 // Sub-allocate non-mappable resources because at the moment the mapped pointer 135 // is part of the resource and not the heap, which doesn't match the Vulkan model. 136 // TODO(crbug.com/dawn/849): allow sub-allocating mappable resources, maybe. 137 if (requirements.size < kMaxSizeForSubAllocation && kind != MemoryKind::LinearMappable && 138 !mDevice->IsToggleEnabled(Toggle::DisableResourceSuballocation)) { 139 // When sub-allocating, Vulkan requires that we respect bufferImageGranularity. Some 140 // hardware puts information on the memory's page table entry and allocating a linear 141 // resource in the same page as a non-linear (aka opaque) resource can cause issues. 142 // Probably because some texture compression flags are stored on the page table entry, 143 // and allocating a linear resource removes these flags. 144 // 145 // Anyway, just to be safe we ask that all sub-allocated resources are allocated with at 146 // least this alignment. TODO(crbug.com/dawn/849): this is suboptimal because multiple 147 // linear (resp. opaque) resources can coexist in the same page. In particular Nvidia 148 // GPUs often use a granularity of 64k which will lead to a lot of wasted spec. Revisit 149 // with a more efficient algorithm later. 150 uint64_t alignment = 151 std::max(requirements.alignment, 152 mDevice->GetDeviceInfo().properties.limits.bufferImageGranularity); 153 154 ResourceMemoryAllocation subAllocation; 155 DAWN_TRY_ASSIGN(subAllocation, mAllocatorsPerType[memoryType]->AllocateMemory( 156 requirements.size, alignment)); 157 if (subAllocation.GetInfo().mMethod != AllocationMethod::kInvalid) { 158 return std::move(subAllocation); 159 } 160 } 161 162 // If sub-allocation failed, allocate memory just for it. 163 std::unique_ptr<ResourceHeapBase> resourceHeap; 164 DAWN_TRY_ASSIGN(resourceHeap, mAllocatorsPerType[memoryType]->AllocateResourceHeap(size)); 165 166 void* mappedPointer = nullptr; 167 if (kind == MemoryKind::LinearMappable) { 168 DAWN_TRY_WITH_CLEANUP( 169 CheckVkSuccess(mDevice->fn.MapMemory(mDevice->GetVkDevice(), 170 ToBackend(resourceHeap.get())->GetMemory(), 0, 171 size, 0, &mappedPointer), 172 "vkMapMemory"), 173 { 174 mAllocatorsPerType[memoryType]->DeallocateResourceHeap(std::move(resourceHeap)); 175 }); 176 } 177 178 AllocationInfo info; 179 info.mMethod = AllocationMethod::kDirect; 180 return ResourceMemoryAllocation(info, /*offset*/ 0, resourceHeap.release(), 181 static_cast<uint8_t*>(mappedPointer)); 182 } 183 Deallocate(ResourceMemoryAllocation * allocation)184 void ResourceMemoryAllocator::Deallocate(ResourceMemoryAllocation* allocation) { 185 switch (allocation->GetInfo().mMethod) { 186 // Some memory allocation can never be initialized, for example when wrapping 187 // swapchain VkImages with a Texture. 188 case AllocationMethod::kInvalid: 189 break; 190 191 // For direct allocation we can put the memory for deletion immediately and the fence 192 // deleter will make sure the resources are freed before the memory. 193 case AllocationMethod::kDirect: { 194 ResourceHeap* heap = ToBackend(allocation->GetResourceHeap()); 195 allocation->Invalidate(); 196 mDevice->GetFencedDeleter()->DeleteWhenUnused(heap->GetMemory()); 197 delete heap; 198 break; 199 } 200 201 // Suballocations aren't freed immediately, otherwise another resource allocation could 202 // happen just after that aliases the old one and would require a barrier. 203 // TODO(crbug.com/dawn/851): Maybe we can produce the correct barriers to reduce the 204 // latency to reclaim memory. 205 case AllocationMethod::kSubAllocated: 206 mSubAllocationsToDelete.Enqueue(*allocation, mDevice->GetPendingCommandSerial()); 207 break; 208 209 default: 210 UNREACHABLE(); 211 break; 212 } 213 214 // Invalidate the underlying resource heap in case the client accidentally 215 // calls DeallocateMemory again using the same allocation. 216 allocation->Invalidate(); 217 } 218 Tick(ExecutionSerial completedSerial)219 void ResourceMemoryAllocator::Tick(ExecutionSerial completedSerial) { 220 for (const ResourceMemoryAllocation& allocation : 221 mSubAllocationsToDelete.IterateUpTo(completedSerial)) { 222 ASSERT(allocation.GetInfo().mMethod == AllocationMethod::kSubAllocated); 223 size_t memoryType = ToBackend(allocation.GetResourceHeap())->GetMemoryType(); 224 225 mAllocatorsPerType[memoryType]->DeallocateMemory(allocation); 226 } 227 228 mSubAllocationsToDelete.ClearUpTo(completedSerial); 229 } 230 FindBestTypeIndex(VkMemoryRequirements requirements,MemoryKind kind)231 int ResourceMemoryAllocator::FindBestTypeIndex(VkMemoryRequirements requirements, 232 MemoryKind kind) { 233 const VulkanDeviceInfo& info = mDevice->GetDeviceInfo(); 234 bool mappable = kind == MemoryKind::LinearMappable; 235 236 // Find a suitable memory type for this allocation 237 int bestType = -1; 238 for (size_t i = 0; i < info.memoryTypes.size(); ++i) { 239 // Resource must support this memory type 240 if ((requirements.memoryTypeBits & (1 << i)) == 0) { 241 continue; 242 } 243 244 // Mappable resource must be host visible 245 if (mappable && 246 (info.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) { 247 continue; 248 } 249 250 // Mappable must also be host coherent. 251 if (mappable && 252 (info.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0) { 253 continue; 254 } 255 256 // Found the first candidate memory type 257 if (bestType == -1) { 258 bestType = static_cast<int>(i); 259 continue; 260 } 261 262 // For non-mappable resources, favor device local memory. 263 bool currentDeviceLocal = 264 info.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; 265 bool bestDeviceLocal = 266 info.memoryTypes[bestType].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; 267 if (!mappable && (currentDeviceLocal != bestDeviceLocal)) { 268 if (currentDeviceLocal) { 269 bestType = static_cast<int>(i); 270 } 271 continue; 272 } 273 274 // All things equal favor the memory in the biggest heap 275 VkDeviceSize bestTypeHeapSize = 276 info.memoryHeaps[info.memoryTypes[bestType].heapIndex].size; 277 VkDeviceSize candidateHeapSize = info.memoryHeaps[info.memoryTypes[i].heapIndex].size; 278 if (candidateHeapSize > bestTypeHeapSize) { 279 bestType = static_cast<int>(i); 280 continue; 281 } 282 } 283 284 return bestType; 285 } 286 DestroyPool()287 void ResourceMemoryAllocator::DestroyPool() { 288 for (auto& alloc : mAllocatorsPerType) { 289 alloc->DestroyPool(); 290 } 291 } 292 293 }} // namespace dawn_native::vulkan 294