1 /*
2 * Copyright (c) 2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "gpu_resource_manager.h"
17
18 #include <algorithm>
19 #include <cinttypes>
20 #if !defined(NDEBUG) || (defined(PLUGIN_LOG_DEBUG) && (PLUGIN_LOG_DEBUG == 1))
21 #include <sstream>
22 #include <thread>
23 #endif
24
25 #include <base/math/mathf.h>
26 #include <render/namespace.h>
27
28 #include "device/device.h"
29 #include "device/gpu_buffer.h"
30 #include "device/gpu_image.h"
31 #include "device/gpu_resource_cache.h"
32 #include "device/gpu_resource_desc_flag_validation.h"
33 #include "device/gpu_resource_manager_base.h"
34 #include "device/gpu_sampler.h"
35 #include "resource_handle_impl.h"
36 #include "util/log.h"
37
38 #if (RENDER_VULKAN_VALIDATION_ENABLED == 1)
39 #include "device/gpu_resource_util.h"
40 #endif
41
42 using namespace BASE_NS;
43 using namespace CORE_NS;
44
45 RENDER_BEGIN_NAMESPACE()
46 namespace {
47 constexpr uint32_t INVALID_PENDING_INDEX { ~0u };
48 constexpr uint32_t MAX_IMAGE_EXTENT { 32768u }; // should be fetched from the device
49
50 constexpr MemoryPropertyFlags NEEDED_DEVICE_MEMORY_PROPERTY_FLAGS_FOR_STAGING_MEM_OPT {
51 CORE_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT
52 };
53 constexpr MemoryPropertyFlags ADD_STAGING_MEM_OPT_FLAGS { CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
54 CORE_MEMORY_PROPERTY_HOST_COHERENT_BIT };
55
56 constexpr uint32_t BUFFER_ALIGNMENT { 256U };
57
Align(uint32_t value,uint32_t align)58 constexpr uint32_t Align(uint32_t value, uint32_t align)
59 {
60 if (value == 0) {
61 return 0;
62 }
63 return ((value + align) / align) * align;
64 }
65
66 // make sure that generation is valid
InvalidateWithGeneration(const EngineResourceHandle handle)67 EngineResourceHandle InvalidateWithGeneration(const EngineResourceHandle handle)
68 {
69 return { handle.id | ~RenderHandleUtil::RES_HANDLE_GENERATION_MASK };
70 }
71
UnpackNewHandle(const EngineResourceHandle & handle,const RenderHandleType type,const uint32_t arrayIndex)72 EngineResourceHandle UnpackNewHandle(
73 const EngineResourceHandle& handle, const RenderHandleType type, const uint32_t arrayIndex)
74 {
75 // increments generation counter
76 const uint32_t gpuGenIndex = RenderHandleUtil::GetGenerationIndexPart(handle) + 1;
77 return RenderHandleUtil::CreateEngineResourceHandle(type, arrayIndex, gpuGenIndex);
78 }
79
80 // we need to know if image is a depth format when binding to descriptor set as read only
GetAdditionalImageFlagsFromFormat(const Format format)81 constexpr RenderHandleInfoFlags GetAdditionalImageFlagsFromFormat(const Format format)
82 {
83 RenderHandleInfoFlags flags {};
84
85 const bool isDepthFormat =
86 ((format == Format::BASE_FORMAT_D16_UNORM) || (format == Format::BASE_FORMAT_X8_D24_UNORM_PACK32) ||
87 (format == Format::BASE_FORMAT_D32_SFLOAT) || (format == Format::BASE_FORMAT_D16_UNORM_S8_UINT) ||
88 (format == Format::BASE_FORMAT_D24_UNORM_S8_UINT) || (format == Format::BASE_FORMAT_D32_SFLOAT_S8_UINT));
89 if (isDepthFormat) {
90 flags |= CORE_RESOURCE_HANDLE_DEPTH_IMAGE;
91 }
92
93 return flags;
94 }
95
96 #if (RENDER_VALIDATION_ENABLED == 1)
ValidateGpuBufferDesc(const GpuBufferDesc & desc)97 void ValidateGpuBufferDesc(const GpuBufferDesc& desc)
98 {
99 if (desc.usageFlags == 0) {
100 PLUGIN_LOG_E("RENDER_VALIDATION: BufferUsageFlags must not be 0");
101 }
102 if ((desc.usageFlags & (~GpuResourceDescFlagValidation::ALL_GPU_BUFFER_USAGE_FLAGS)) != 0) {
103 PLUGIN_LOG_E("RENDER_VALIDATION: Invalid BufferUsageFlags (%u)", desc.usageFlags);
104 }
105 if (desc.memoryPropertyFlags == 0) {
106 PLUGIN_LOG_E("RENDER_VALIDATION: MemoryPropertyFlags must not be 0");
107 }
108 if ((desc.memoryPropertyFlags & (~GpuResourceDescFlagValidation::ALL_MEMORY_PROPERTY_FLAGS)) != 0) {
109 PLUGIN_LOG_E("RENDER_VALIDATION: Invalid MemoryPropertyFlags (%u)", desc.memoryPropertyFlags);
110 }
111 if ((desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_MAP_OUTSIDE_RENDERER) &&
112 ((desc.memoryPropertyFlags & MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)) {
113 PLUGIN_LOG_E(
114 "RENDER_VALIDATION: Invalid MemoryPropertyFlags for CORE_ENGINE_BUFFER_CREATION_MAP_OUTSIDE_RENDERER (%u)",
115 desc.memoryPropertyFlags);
116 }
117 if (desc.byteSize == 0) {
118 PLUGIN_LOG_E("RENDER_VALIDATION: Buffer byteSize must larger than zero");
119 }
120 }
121
ValidateGpuImageDesc(const GpuImageDesc & desc,const string_view name)122 void ValidateGpuImageDesc(const GpuImageDesc& desc, const string_view name)
123 {
124 bool valid = true;
125 if (desc.format == Format::BASE_FORMAT_UNDEFINED) {
126 PLUGIN_LOG_E("RENDER_VALIDATION: Undefined image format");
127 valid = false;
128 }
129 if (desc.imageType > ImageType::CORE_IMAGE_TYPE_3D) {
130 PLUGIN_LOG_E("RENDER_VALIDATION: Unsupported image type");
131 valid = false;
132 }
133 if ((desc.imageViewType == ImageViewType::CORE_IMAGE_VIEW_TYPE_2D) && (desc.layerCount > 1u)) {
134 PLUGIN_LOG_E(
135 "RENDER_VALIDATION: Default image view is done for sampling / shader resource access and needs to be "
136 "CORE_IMAGE_VIEW_TYPE_2D_ARRAY with multiple layers");
137 valid = false;
138 }
139 if (desc.imageTiling > ImageTiling::CORE_IMAGE_TILING_LINEAR) {
140 PLUGIN_LOG_E("RENDER_VALIDATION: Unsupported image tiling mode (%u)", static_cast<uint32_t>(desc.imageTiling));
141 valid = false;
142 }
143 if (desc.usageFlags == 0) {
144 PLUGIN_LOG_E("RENDER_VALIDATION: ImageUsageFlags must not be 0");
145 valid = false;
146 }
147 if ((desc.usageFlags & (~GpuResourceDescFlagValidation::ALL_GPU_IMAGE_USAGE_FLAGS)) != 0) {
148 PLUGIN_LOG_E("RENDER_VALIDATION: Invalid ImageUsageFlags (%u)", desc.usageFlags);
149 valid = false;
150 }
151 if (desc.memoryPropertyFlags == 0) {
152 PLUGIN_LOG_E("RENDER_VALIDATION: MemoryPropertyFlags must not be 0");
153 valid = false;
154 }
155 if ((desc.memoryPropertyFlags & (~GpuResourceDescFlagValidation::ALL_MEMORY_PROPERTY_FLAGS)) != 0) {
156 PLUGIN_LOG_E("RENDER_VALIDATION: Invalid MemoryPropertyFlags (%u)", desc.memoryPropertyFlags);
157 valid = false;
158 }
159 if (desc.width == 0 || desc.height == 0 || desc.depth == 0) {
160 PLUGIN_LOG_E("RENDER_VALIDATION: Image extents must not be zero (x:%u, y:%u, z:%u)", desc.width, desc.height,
161 desc.depth);
162 valid = false;
163 }
164 if (desc.width > MAX_IMAGE_EXTENT || desc.height > MAX_IMAGE_EXTENT || desc.depth > MAX_IMAGE_EXTENT) {
165 PLUGIN_LOG_E("RENDER_VALIDATION: Image extents must not be bigger than (%u) (x:%u, y:%u, z:%u)",
166 MAX_IMAGE_EXTENT, desc.width, desc.height, desc.depth);
167 valid = false;
168 }
169 if (desc.mipCount == 0 || desc.layerCount == 0) {
170 PLUGIN_LOG_E("RENDER_VALIDATION: Image mip and layer count must be >=1 (mipCount:%u, layerCount:%u)",
171 desc.mipCount, desc.layerCount);
172 valid = false;
173 }
174 if ((desc.createFlags & (~GpuResourceDescFlagValidation::ALL_IMAGE_CREATE_FLAGS)) != 0) {
175 PLUGIN_LOG_E("RENDER_VALIDATION: Invalid ImageCreateFlags (%u)", desc.createFlags);
176 valid = false;
177 }
178 if ((desc.engineCreationFlags & CORE_ENGINE_IMAGE_CREATION_GENERATE_MIPS) &&
179 ((desc.usageFlags & CORE_IMAGE_USAGE_TRANSFER_SRC_BIT) == 0)) {
180 PLUGIN_LOG_E(
181 "RENDER_VALIDATION: Must use usage flags CORE_IMAGE_USAGE_TRANSFER_SRC_BIT when generating mip maps");
182 valid = false;
183 }
184 if (desc.usageFlags & CORE_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT) {
185 const ImageUsageFlags usageFlags =
186 desc.usageFlags &
187 ~(CORE_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT | CORE_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
188 CORE_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | CORE_IMAGE_USAGE_COLOR_ATTACHMENT_BIT);
189 if (usageFlags != 0) {
190 PLUGIN_LOG_E(
191 "RENDER_VALIDATION: If image usage flags contain CORE_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT, only "
192 "DEPTH_STENCIL_ATTACHMENT_BIT, INPUT_ATTACHMENT_BIT, and COLOR_ATTACHMENT_BIT can be set.");
193 valid = false;
194 }
195 }
196 if ((desc.layerCount > 1u) && (desc.imageViewType <= CORE_IMAGE_VIEW_TYPE_3D)) {
197 PLUGIN_LOG_E("RENDER_VALIDATION: If image layer count (%u) is larger than 1, then image view type must be "
198 "CORE_IMAGE_VIEW_TYPE_XX_ARRAY",
199 desc.layerCount);
200 valid = false;
201 }
202
203 if ((!valid) && (!name.empty())) {
204 PLUGIN_LOG_E("RENDER_VALIDATION: validation issue(s) with image (name: %s)", name.data());
205 }
206 }
207
ValidateGpuAccelerationStructureDesc(const GpuAccelerationStructureDesc & desc)208 void ValidateGpuAccelerationStructureDesc(const GpuAccelerationStructureDesc& desc)
209 {
210 ValidateGpuBufferDesc(desc.bufferDesc);
211 if ((desc.bufferDesc.usageFlags & CORE_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT) == 0) {
212 PLUGIN_LOG_E("RENDER_VALIDATION: usageFlags must include CORE_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT "
213 "for acceleration structures");
214 }
215 }
216
ValidateGpuImageCopy(const GpuImageDesc & desc,const BufferImageCopy & copy,const string_view name)217 void ValidateGpuImageCopy(const GpuImageDesc& desc, const BufferImageCopy& copy, const string_view name)
218 {
219 const uint32_t mip = copy.imageSubresource.mipLevel;
220 const Size3D imageSize { desc.width >> mip, desc.height >> mip, desc.depth };
221 if ((copy.imageOffset.width >= imageSize.width) || (copy.imageOffset.width >= imageSize.height) ||
222 (copy.imageOffset.depth >= imageSize.depth)) {
223 PLUGIN_LOG_E(
224 "RENDER_VALIDATION: BufferImageCopy offset exceeds GpuImageDesc. Mipsize: %u, %u, %u, offset: %u %u %u. "
225 "(name: %s)",
226 imageSize.width, imageSize.height, imageSize.depth, copy.imageOffset.width, copy.imageOffset.height,
227 copy.imageOffset.depth, name.data());
228 }
229 }
230 #endif
231
GetValidGpuBufferDesc(const GpuBufferDesc & desc)232 inline constexpr GpuBufferDesc GetValidGpuBufferDesc(const GpuBufferDesc& desc)
233 {
234 return GpuBufferDesc {
235 desc.usageFlags & GpuResourceDescFlagValidation::ALL_GPU_BUFFER_USAGE_FLAGS,
236 desc.memoryPropertyFlags & GpuResourceDescFlagValidation::ALL_MEMORY_PROPERTY_FLAGS,
237 desc.engineCreationFlags,
238 desc.byteSize,
239 };
240 }
241
GetMapBufferDesc(const uint32_t byteSize)242 inline constexpr GpuBufferDesc GetMapBufferDesc(const uint32_t byteSize)
243 {
244 return {
245 BufferUsageFlagBits::CORE_BUFFER_USAGE_UNIFORM_BUFFER_BIT,
246 MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_COHERENT_BIT |
247 MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT,
248 0U,
249 byteSize,
250 };
251 }
252
CheckAndEnableMemoryOptimizations(const uint32_t gpuResourceMgrFlags,GpuBufferDesc & desc)253 inline constexpr void CheckAndEnableMemoryOptimizations(const uint32_t gpuResourceMgrFlags, GpuBufferDesc& desc)
254 {
255 if (gpuResourceMgrFlags & GpuResourceManager::GPU_RESOURCE_MANAGER_OPTIMIZE_STAGING_MEMORY) {
256 if ((desc.memoryPropertyFlags == CORE_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) &&
257 (desc.usageFlags & CORE_BUFFER_USAGE_TRANSFER_DST_BIT) &&
258 (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_ENABLE_MEMORY_OPTIMIZATIONS)) {
259 desc.memoryPropertyFlags |= ADD_STAGING_MEM_OPT_FLAGS;
260 }
261 }
262 }
263
GetScalingImageNeed(const GpuImageDesc & desc,const array_view<const IImageContainer::SubImageDesc> & copies)264 inline constexpr bool GetScalingImageNeed(
265 const GpuImageDesc& desc, const array_view<const IImageContainer::SubImageDesc>& copies)
266 {
267 bool scale = false;
268 if (desc.engineCreationFlags & EngineImageCreationFlagBits::CORE_ENGINE_IMAGE_CREATION_SCALE) {
269 // we only support single image (single buffer image copy) scaling
270 if (copies.size() == 1) {
271 scale = (copies[0].width != desc.width) || (copies[0].height != desc.height);
272 }
273 }
274 return scale;
275 }
276
GetScalingImageNeed(const GpuImageDesc & desc,const array_view<const BufferImageCopy> & copies)277 inline constexpr bool GetScalingImageNeed(const GpuImageDesc& desc, const array_view<const BufferImageCopy>& copies)
278 {
279 bool scale = false;
280 if (desc.engineCreationFlags & EngineImageCreationFlagBits::CORE_ENGINE_IMAGE_CREATION_SCALE) {
281 // we only support single image (single buffer image copy) scaling
282 if (copies.size() == 1) {
283 scale = (copies[0].imageExtent.width != desc.width) || (copies[0].imageExtent.height != desc.height);
284 }
285 }
286 return scale;
287 }
288
289 // staging needs to be locked when called with the input resources
UpdateStagingScaling(const Format format,const array_view<const IImageContainer::SubImageDesc> & copies,ScalingImageDataStruct & siData)290 void UpdateStagingScaling(
291 const Format format, const array_view<const IImageContainer::SubImageDesc>& copies, ScalingImageDataStruct& siData)
292 {
293 PLUGIN_ASSERT(copies.size() == 1);
294 if (auto iter = siData.formatToScalingImages.find(format); iter != siData.formatToScalingImages.end()) {
295 const size_t index = iter->second;
296 PLUGIN_ASSERT(index < siData.scalingImages.size());
297 auto& scaleImage = siData.scalingImages[index];
298 scaleImage.maxWidth = Math::max(scaleImage.maxWidth, copies[0].width);
299 scaleImage.maxHeight = Math::max(scaleImage.maxHeight, copies[0].height);
300 } else {
301 const size_t index = siData.scalingImages.size();
302 siData.scalingImages.push_back({ {}, format, copies[0].width, copies[0].height });
303 siData.formatToScalingImages[format] = index;
304 }
305 }
306
UpdateStagingScaling(const Format format,const array_view<const BufferImageCopy> & copies,ScalingImageDataStruct & siData)307 void UpdateStagingScaling(
308 const Format format, const array_view<const BufferImageCopy>& copies, ScalingImageDataStruct& siData)
309 {
310 PLUGIN_ASSERT(copies.size() == 1);
311 const auto& extent = copies[0].imageExtent;
312 if (auto iter = siData.formatToScalingImages.find(format); iter != siData.formatToScalingImages.end()) {
313 const size_t index = iter->second;
314 PLUGIN_ASSERT(index < siData.scalingImages.size());
315 auto& scaleImage = siData.scalingImages[index];
316 scaleImage.maxWidth = Math::max(scaleImage.maxWidth, extent.width);
317 scaleImage.maxHeight = Math::max(scaleImage.maxHeight, extent.height);
318 } else {
319 const size_t index = siData.scalingImages.size();
320 siData.scalingImages.push_back({ {}, format, extent.width, extent.height });
321 siData.formatToScalingImages[format] = index;
322 }
323 }
324
GetStagingScalingImageDesc(const Format format,const uint32_t width,const uint32_t height)325 GpuImageDesc GetStagingScalingImageDesc(const Format format, const uint32_t width, const uint32_t height)
326 {
327 return GpuImageDesc {
328 ImageType::CORE_IMAGE_TYPE_2D,
329 ImageViewType::CORE_IMAGE_VIEW_TYPE_2D,
330 format,
331 ImageTiling::CORE_IMAGE_TILING_OPTIMAL,
332 // NOTE sampled is not needed, but image view should not be created
333 ImageUsageFlagBits::CORE_IMAGE_USAGE_TRANSFER_SRC_BIT | ImageUsageFlagBits::CORE_IMAGE_USAGE_TRANSFER_DST_BIT |
334 ImageUsageFlagBits::CORE_IMAGE_USAGE_SAMPLED_BIT,
335 MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
336 0,
337 0, // No dynamic barrriers
338 width,
339 height,
340 1u,
341 1u,
342 1u,
343 1u,
344 {},
345 };
346 }
347
ConvertCoreBufferImageCopy(const IImageContainer::SubImageDesc & bufferImageCopy,const uint32_t additionalBufferOffset)348 BufferImageCopy ConvertCoreBufferImageCopy(
349 const IImageContainer::SubImageDesc& bufferImageCopy, const uint32_t additionalBufferOffset)
350 {
351 return BufferImageCopy {
352 /** Buffer offset */
353 bufferImageCopy.bufferOffset + additionalBufferOffset,
354 /** Buffer row length */
355 bufferImageCopy.bufferRowLength,
356 /** Buffer image height */
357 bufferImageCopy.bufferImageHeight,
358 /** Image subresource */
359 { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, bufferImageCopy.mipLevel, 0, bufferImageCopy.layerCount },
360 /** Image offset */
361 { 0, 0, 0 },
362 /** Image extent */
363 { bufferImageCopy.width, bufferImageCopy.height, bufferImageCopy.depth },
364 };
365 }
366
367 #if (RENDER_DEBUG_GPU_RESOURCE_IDS == 1)
LogGpuResource(const RenderHandle & gpuHandle,const EngineResourceHandle engineHandle)368 void LogGpuResource(const RenderHandle& gpuHandle, const EngineResourceHandle engineHandle)
369 {
370 constexpr string_view names[] = { "buffer", "image", "sampler" };
371 const RenderHandleType handleType = RenderHandleUtil::GetHandleType(gpuHandle);
372 uint32_t idx = 0;
373 if (handleType == RenderHandleType::GPU_IMAGE) {
374 idx = 1u;
375 } else if (handleType == RenderHandleType::GPU_SAMPLER) {
376 idx = 2u;
377 }
378 PLUGIN_LOG_E("gpu %s > %" PRIx64 "=%" PRIx64 " generation: %u=%u", names[idx].data(), gpuHandle.id, engineHandle.id,
379 RenderHandleUtil::GetGenerationIndexPart(gpuHandle), RenderHandleUtil::GetGenerationIndexPart(engineHandle));
380 }
381 #endif
382 } // namespace
383
GpuResourceManager(Device & device,const CreateInfo & createInfo)384 GpuResourceManager::GpuResourceManager(Device& device, const CreateInfo& createInfo)
385 : device_(device), gpuResourceMgrFlags_(createInfo.flags),
386 gpuBufferMgr_(make_unique<GpuResourceManagerTyped<GpuBuffer, GpuBufferDesc>>(device)),
387 gpuImageMgr_(make_unique<GpuResourceManagerTyped<GpuImage, GpuImageDesc>>(device)),
388 gpuSamplerMgr_(make_unique<GpuResourceManagerTyped<GpuSampler, GpuSamplerDesc>>(device))
389 {
390 gpuResourceCache_ = make_unique<GpuResourceCache>(*this);
391
392 bufferStore_.mgr = gpuBufferMgr_.get();
393 imageStore_.mgr = gpuImageMgr_.get();
394 samplerStore_.mgr = gpuSamplerMgr_.get();
395
396 const MemoryPropertyFlags deviceSharedMemoryPropertyFlags = device_.GetSharedMemoryPropertyFlags();
397 // remove create info flag if not really available
398 if (((gpuResourceMgrFlags_ & GPU_RESOURCE_MANAGER_OPTIMIZE_STAGING_MEMORY) == 0) ||
399 (deviceSharedMemoryPropertyFlags & NEEDED_DEVICE_MEMORY_PROPERTY_FLAGS_FOR_STAGING_MEM_OPT) !=
400 NEEDED_DEVICE_MEMORY_PROPERTY_FLAGS_FOR_STAGING_MEM_OPT) {
401 gpuResourceMgrFlags_ = gpuResourceMgrFlags_ & ~GPU_RESOURCE_MANAGER_OPTIMIZE_STAGING_MEMORY;
402 }
403 }
404
~GpuResourceManager()405 GpuResourceManager::~GpuResourceManager()
406 {
407 // reset cache before logging
408 // cache logs it's own un-released resources
409 gpuResourceCache_.reset();
410
411 #if (RENDER_VALIDATION_ENABLED == 1)
412 auto checkAndPrintValidation = [](const PerManagerStore& store, const string_view name) {
413 uint32_t aliveCounter = 0;
414 const auto clientLock = std::lock_guard(store.clientMutex);
415 for (const auto& ref : store.clientHandles) {
416 if (ref && (ref.GetRefCount() > 1)) {
417 aliveCounter++;
418 }
419 }
420 if (aliveCounter > 0) {
421 PLUGIN_LOG_W(
422 "RENDER_VALIDATION: Not all %s handle references released (count: %u)", name.data(), aliveCounter);
423 }
424 };
425 checkAndPrintValidation(bufferStore_, "GPU buffer");
426 checkAndPrintValidation(imageStore_, "GPU image");
427 checkAndPrintValidation(samplerStore_, "GPU sampler");
428 #endif
429 }
430
Get(const RenderHandle & handle) const431 RenderHandleReference GpuResourceManager::Get(const RenderHandle& handle) const
432 {
433 if (RenderHandleUtil::IsValid(handle)) {
434 const RenderHandleType handleType = RenderHandleUtil::GetHandleType(handle);
435 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
436 if (handleType == RenderHandleType::GPU_BUFFER) {
437 auto& store = bufferStore_;
438 auto const clientLock = std::shared_lock(store.clientMutex);
439 if (arrayIndex < static_cast<uint32_t>(store.clientHandles.size())) {
440 return store.clientHandles[arrayIndex];
441 }
442 } else if (handleType == RenderHandleType::GPU_IMAGE) {
443 auto& store = imageStore_;
444 auto const clientLock = std::shared_lock(store.clientMutex);
445 if (arrayIndex < static_cast<uint32_t>(store.clientHandles.size())) {
446 return store.clientHandles[arrayIndex];
447 }
448 } else if (handleType == RenderHandleType::GPU_SAMPLER) {
449 auto& store = samplerStore_;
450 auto const clientLock = std::shared_lock(store.clientMutex);
451 if (arrayIndex < static_cast<uint32_t>(store.clientHandles.size())) {
452 return store.clientHandles[arrayIndex];
453 }
454 }
455 }
456 return RenderHandleReference {};
457 }
458
GetRawHandle(const RenderHandle & handle) const459 RenderHandle GpuResourceManager::GetRawHandle(const RenderHandle& handle) const
460 {
461 if (RenderHandleUtil::IsValid(handle)) {
462 const RenderHandleType handleType = RenderHandleUtil::GetHandleType(handle);
463 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
464 if (handleType == RenderHandleType::GPU_BUFFER) {
465 auto& store = bufferStore_;
466 auto const clientLock = std::shared_lock(store.clientMutex);
467 if (arrayIndex < static_cast<uint32_t>(store.clientHandles.size())) {
468 return store.clientHandles[arrayIndex].GetHandle();
469 }
470 } else if (handleType == RenderHandleType::GPU_IMAGE) {
471 auto& store = imageStore_;
472 auto const clientLock = std::shared_lock(store.clientMutex);
473 if (arrayIndex < static_cast<uint32_t>(store.clientHandles.size())) {
474 return store.clientHandles[arrayIndex].GetHandle();
475 }
476 } else if (handleType == RenderHandleType::GPU_SAMPLER) {
477 auto& store = samplerStore_;
478 auto const clientLock = std::shared_lock(store.clientMutex);
479 if (arrayIndex < static_cast<uint32_t>(store.clientHandles.size())) {
480 return store.clientHandles[arrayIndex].GetHandle();
481 }
482 }
483 }
484 return RenderHandle {};
485 }
486
GetStagingBufferDesc(const uint32_t byteSize)487 GpuBufferDesc GpuResourceManager::GetStagingBufferDesc(const uint32_t byteSize)
488 {
489 return {
490 BufferUsageFlagBits::CORE_BUFFER_USAGE_TRANSFER_SRC_BIT,
491 MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_COHERENT_BIT |
492 MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT,
493 EngineBufferCreationFlagBits::CORE_ENGINE_BUFFER_CREATION_SINGLE_SHOT_STAGING,
494 byteSize,
495 };
496 }
497
498 // call to evaluate if there's already pending resources which we will replace
499 // store.clientMutex needs to be locked
GetPendingOptionalResourceIndex(const PerManagerStore & store,const RenderHandle & handle,const string_view name)500 uint32_t GpuResourceManager::GetPendingOptionalResourceIndex(
501 const PerManagerStore& store, const RenderHandle& handle, const string_view name)
502 {
503 uint32_t optionalResourceIndex = ~0u;
504 uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
505 const bool hasReplaceHandle = (arrayIndex < static_cast<uint32_t>(store.clientHandles.size()));
506 if ((!hasReplaceHandle) && (!name.empty())) {
507 if (auto const iter = store.nameToClientIndex.find(name); iter != store.nameToClientIndex.cend()) {
508 arrayIndex = RenderHandleUtil::GetIndexPart(iter->second);
509 }
510 }
511 if (arrayIndex < static_cast<uint32_t>(store.clientHandles.size())) {
512 if (const uint32_t pendingArrIndex = store.additionalData[arrayIndex].indexToPendingData;
513 pendingArrIndex != INVALID_PENDING_INDEX) {
514 PLUGIN_ASSERT(pendingArrIndex < store.pendingData.allocations.size());
515 if (pendingArrIndex < static_cast<uint32_t>(store.pendingData.allocations.size())) {
516 const auto& allocOp = store.pendingData.allocations[pendingArrIndex];
517 optionalResourceIndex = allocOp.optionalResourceIndex;
518 }
519 }
520 }
521 return optionalResourceIndex;
522 }
523
524 // needs to be locked when called
CreateStagingBuffer(const GpuBufferDesc & desc)525 RenderHandleReference GpuResourceManager::CreateStagingBuffer(const GpuBufferDesc& desc)
526 {
527 PerManagerStore& store = bufferStore_;
528 return StoreAllocation(store, { ResourceDescriptor { desc }, {}, {}, RenderHandleType::GPU_BUFFER, ~0u, 0u })
529 .handle;
530 }
531
532 // needs to be locked when called
CreateBuffer(const string_view name,const RenderHandle & replacedHandle,const GpuBufferDesc & desc)533 GpuResourceManager::StoreAllocationData GpuResourceManager::CreateBuffer(
534 const string_view name, const RenderHandle& replacedHandle, const GpuBufferDesc& desc)
535 {
536 #if (RENDER_VALIDATION_ENABLED == 1)
537 ValidateGpuBufferDesc(desc);
538 #endif
539 MemoryPropertyFlags additionalMemPropFlags = 0U;
540 if (device_.GetBackendType() == DeviceBackendType::VULKAN) {
541 additionalMemPropFlags = (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_MAP_OUTSIDE_RENDERER)
542 ? (MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
543 : 0U;
544 } else {
545 additionalMemPropFlags = (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_MAP_OUTSIDE_RENDERER)
546 ? (MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_COHERENT_BIT |
547 MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
548 : 0U;
549 }
550 const GpuBufferDesc validatedDesc {
551 desc.usageFlags | defaultBufferUsageFlags_,
552 desc.memoryPropertyFlags | additionalMemPropFlags,
553 desc.engineCreationFlags,
554 Math::max(desc.byteSize, 1u),
555 desc.format,
556 };
557 PerManagerStore& store = bufferStore_;
558 if (validatedDesc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
559 // replace immediate created if still pending (i.e. not usable on the GPU)
560 // memory pointers on client become invalid
561 const auto emplaceResourceIndex = static_cast<uint32_t>(store.pendingData.buffers.size());
562 const uint32_t optionalResourceIndex =
563 Math::min(emplaceResourceIndex, GetPendingOptionalResourceIndex(store, replacedHandle, name));
564
565 if (unique_ptr<GpuBuffer> gpuBuffer = [this](const GpuBufferDesc validatedDesc) {
566 // protect GPU memory allocations
567 auto lock = std::lock_guard(allocationMutex_);
568 return device_.CreateGpuBuffer(validatedDesc);
569 }(validatedDesc)) {
570 // safety checks
571 if ((optionalResourceIndex < emplaceResourceIndex) &&
572 (optionalResourceIndex < store.pendingData.buffers.size())) {
573 store.pendingData.buffers[optionalResourceIndex] = move(gpuBuffer);
574 } else {
575 store.pendingData.buffers.push_back(move(gpuBuffer));
576 }
577 }
578
579 StoreAllocationData sad = StoreAllocation(store, { ResourceDescriptor { validatedDesc }, name, replacedHandle,
580 RenderHandleType::GPU_BUFFER, optionalResourceIndex, 0u });
581 // additional data is increased in StoreAllocation
582 // there are as many additional data elements as clientHandle elements
583 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(sad.handle.GetHandle());
584 PLUGIN_ASSERT(arrayIndex < store.additionalData.size());
585 if (GpuBuffer* buffer = store.pendingData.buffers[optionalResourceIndex].get(); buffer) {
586 store.additionalData[arrayIndex].resourcePtr = reinterpret_cast<uintptr_t>(reinterpret_cast<void*>(buffer));
587 }
588 return sad;
589 } else {
590 return StoreAllocation(store,
591 { ResourceDescriptor { validatedDesc }, name, replacedHandle, RenderHandleType::GPU_BUFFER, ~0u, 0u });
592 }
593 }
594
GetOrCreate(const string_view name,const GpuBufferDesc & desc)595 RenderHandleReference GpuResourceManager::GetOrCreate(const string_view name, const GpuBufferDesc& desc)
596 {
597 RenderHandleReference handle;
598
599 GpuBufferDesc validDesc = GetValidGpuBufferDesc(desc);
600 CheckAndEnableMemoryOptimizations(gpuResourceMgrFlags_, validDesc);
601 {
602 const auto lock = std::lock_guard(bufferStore_.clientMutex);
603
604 // check if present (not locked inside)
605 handle = GetHandleNoLock(bufferStore_, name);
606 }
607 // create
608 if (!handle) {
609 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
610 device_.Activate();
611 }
612
613 const auto lock = std::lock_guard(bufferStore_.clientMutex);
614 handle = CreateBuffer(name, {}, validDesc).handle;
615
616 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
617 device_.Deactivate();
618 }
619 }
620 return handle;
621 }
622
Create(const string_view name,const GpuBufferDesc & desc)623 RenderHandleReference GpuResourceManager::Create(const string_view name, const GpuBufferDesc& desc)
624 {
625 RenderHandleReference handle;
626
627 GpuBufferDesc validDesc = GetValidGpuBufferDesc(desc);
628 CheckAndEnableMemoryOptimizations(gpuResourceMgrFlags_, validDesc);
629
630 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
631 device_.Activate();
632 }
633 PerManagerStore& store = bufferStore_;
634 {
635 const auto lock = std::lock_guard(store.clientMutex);
636
637 handle = CreateBuffer(name, {}, validDesc).handle;
638 }
639 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
640 device_.Deactivate();
641 }
642 return handle;
643 }
644
Create(const RenderHandleReference & replacedHandle,const GpuBufferDesc & desc)645 RenderHandleReference GpuResourceManager::Create(const RenderHandleReference& replacedHandle, const GpuBufferDesc& desc)
646 {
647 RenderHandleReference handle;
648
649 const RenderHandle rawHandle = replacedHandle.GetHandle();
650 #if (RENDER_VALIDATION_ENABLED == 1)
651 const bool valid = RenderHandleUtil::IsValid(rawHandle);
652 const RenderHandleType type = RenderHandleUtil::GetHandleType(rawHandle);
653 if (valid && (type != RenderHandleType::GPU_BUFFER)) {
654 PLUGIN_LOG_E("RENDER_VALIDATION: trying to replace a non GPU buffer handle (type: %u) with GpuBufferDesc",
655 (uint32_t)type);
656 }
657 #endif
658 GpuBufferDesc validDesc = GetValidGpuBufferDesc(desc);
659 CheckAndEnableMemoryOptimizations(gpuResourceMgrFlags_, validDesc);
660
661 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
662 device_.Activate();
663 }
664 {
665 PerManagerStore& store = bufferStore_;
666 const auto lock = std::lock_guard(store.clientMutex);
667
668 handle = CreateBuffer({}, rawHandle, validDesc).handle;
669 }
670 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
671 device_.Deactivate();
672 }
673 return handle;
674 }
675
Create(const string_view name,const GpuBufferDesc & desc,const array_view<const uint8_t> data)676 RenderHandleReference GpuResourceManager::Create(
677 const string_view name, const GpuBufferDesc& desc, const array_view<const uint8_t> data)
678 {
679 RenderHandleReference handle;
680
681 #if (RENDER_VALIDATION_ENABLED == 1)
682 ValidateGpuBufferDesc(desc);
683 #endif
684
685 GpuBufferDesc validDesc = GetValidGpuBufferDesc(desc);
686 CheckAndEnableMemoryOptimizations(gpuResourceMgrFlags_, validDesc);
687 const bool useStagingBuffer = (validDesc.memoryPropertyFlags & CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT) !=
688 CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
689
690 auto& store = bufferStore_;
691 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
692 device_.Activate();
693 }
694 {
695 StoreAllocationData sad;
696 const auto lock = std::lock_guard(store.clientMutex);
697
698 sad = CreateBuffer(name, {}, validDesc);
699 const uint32_t minByteSize = std::min(validDesc.byteSize, (uint32_t)data.size_bytes());
700
701 auto const stagingLock = std::lock_guard(stagingMutex_);
702
703 stagingOperations_.bufferCopies.push_back(BufferCopy { 0, 0, minByteSize });
704 const uint32_t beginIndex = (uint32_t)stagingOperations_.bufferCopies.size() - 1;
705 vector<uint8_t> copiedData(data.cbegin().ptr(), data.cbegin().ptr() + minByteSize);
706
707 // add staging vector index handle to resource handle in pending allocations
708 PLUGIN_ASSERT(sad.allocationIndex < store.pendingData.allocations.size());
709 auto& allocRef = store.pendingData.allocations[sad.allocationIndex];
710 allocRef.optionalStagingVectorIndex = static_cast<uint32_t>(stagingOperations_.bufferToBuffer.size());
711 allocRef.optionalStagingCopyType = useStagingBuffer ? StagingCopyStruct::CopyType::BUFFER_TO_BUFFER
712 : StagingCopyStruct::CopyType::CPU_TO_BUFFER;
713
714 if (useStagingBuffer) {
715 const uint32_t stagingBufferByteOffset = stagingOperations_.stagingByteSize;
716 // offset to data copy
717 stagingOperations_.bufferCopies[beginIndex].srcOffset += stagingBufferByteOffset;
718 const uint32_t stagingBufferByteSize =
719 useStagingBuffer ? static_cast<uint32_t>(copiedData.size_in_bytes()) : 0u;
720 stagingOperations_.stagingByteSize =
721 Align(stagingOperations_.stagingByteSize + stagingBufferByteSize, BUFFER_ALIGNMENT);
722 stagingOperations_.bufferToBuffer.push_back(StagingCopyStruct {
723 StagingCopyStruct::DataType::DATA_TYPE_VECTOR, {}, sad.handle, beginIndex, 1, move(copiedData), nullptr,
724 Format::BASE_FORMAT_UNDEFINED, stagingBufferByteOffset, stagingBufferByteSize, false });
725 } else {
726 stagingOperations_.cpuToBuffer.push_back(
727 StagingCopyStruct { StagingCopyStruct::DataType::DATA_TYPE_VECTOR, {}, sad.handle, beginIndex, 1,
728 move(copiedData), nullptr, Format::BASE_FORMAT_UNDEFINED, 0U, 0U, false });
729 }
730 handle = move(sad.handle);
731 }
732 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
733 device_.Deactivate();
734 }
735 return handle;
736 }
737
Create(const GpuBufferDesc & desc)738 RenderHandleReference GpuResourceManager::Create(const GpuBufferDesc& desc)
739 {
740 RenderHandleReference handle;
741 GpuBufferDesc validDesc = GetValidGpuBufferDesc(desc);
742 CheckAndEnableMemoryOptimizations(gpuResourceMgrFlags_, validDesc);
743 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
744 device_.Activate();
745 }
746 {
747 auto& store = bufferStore_;
748 const auto lock = std::lock_guard(store.clientMutex);
749
750 handle = CreateBuffer({}, {}, validDesc).handle;
751 }
752 if (desc.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE) {
753 device_.Deactivate();
754 }
755 return handle;
756 }
757
Create(const GpuBufferDesc & desc,const array_view<const uint8_t> data)758 RenderHandleReference GpuResourceManager::Create(const GpuBufferDesc& desc, const array_view<const uint8_t> data)
759 {
760 // this is a fwd-method, desc is validated inside the called method
761 return Create({}, desc, data);
762 }
763
Create(const string_view name,const BackendSpecificBufferDesc & backendSpecificData)764 RenderHandleReference GpuResourceManager::Create(
765 const string_view name, const BackendSpecificBufferDesc& backendSpecificData)
766 {
767 device_.Activate();
768 PerManagerStore& store = bufferStore_;
769 const auto lock = std::lock_guard(store.clientMutex);
770
771 // replace immediate created if still pending (i.e. not usable on the GPU)
772 const auto emplaceResourceIndex = static_cast<uint32_t>(store.pendingData.buffers.size());
773 const uint32_t optionalResourceIndex =
774 Math::min(emplaceResourceIndex, GetPendingOptionalResourceIndex(store, {}, name));
775
776 // additional handle flags provide information if platform conversion is needed
777 uint32_t additionalHandleFlags = 0u;
778
779 if (unique_ptr<GpuBuffer> gpuBuffer = [this](const BackendSpecificBufferDesc& backendSpecificData) {
780 // protect GPU memory allocations
781 auto lock = std::lock_guard(allocationMutex_);
782 return device_.CreateGpuBuffer(backendSpecificData);
783 }(backendSpecificData)) {
784 // safety checks
785 if ((optionalResourceIndex < emplaceResourceIndex) &&
786 (optionalResourceIndex < store.pendingData.buffers.size())) {
787 store.pendingData.buffers[optionalResourceIndex] = move(gpuBuffer);
788 } else {
789 store.pendingData.buffers.push_back(move(gpuBuffer));
790 }
791 }
792 device_.Deactivate();
793
794 const auto& buffers = store.pendingData.buffers;
795 const auto& finalDesc = (optionalResourceIndex < buffers.size() && buffers[optionalResourceIndex])
796 ? buffers[optionalResourceIndex]->GetDesc()
797 : GpuBufferDesc {};
798 auto handle = StoreAllocation(store, { ResourceDescriptor { finalDesc }, name, {}, RenderHandleType::GPU_BUFFER,
799 optionalResourceIndex, additionalHandleFlags })
800 .handle;
801
802 return handle;
803 }
804
805 // needs to be locked when called
CreateImage(const string_view name,const RenderHandle & replacedHandle,const GpuImageDesc & desc)806 GpuResourceManager::StoreAllocationData GpuResourceManager::CreateImage(
807 const string_view name, const RenderHandle& replacedHandle, const GpuImageDesc& desc)
808 {
809 #if (RENDER_VALIDATION_ENABLED == 1)
810 ValidateGpuImageDesc(desc, name);
811 #endif
812
813 PerManagerStore& store = imageStore_;
814 const StoreAllocationInfo info {
815 ResourceDescriptor { GpuImageDesc {
816 desc.imageType,
817 desc.imageViewType,
818 device_.GetFormatOrFallback(desc.format),
819 (desc.imageTiling > ImageTiling::CORE_IMAGE_TILING_LINEAR) ? ImageTiling::CORE_IMAGE_TILING_OPTIMAL
820 : desc.imageTiling,
821 (desc.usageFlags & GpuResourceDescFlagValidation::ALL_GPU_IMAGE_USAGE_FLAGS) | defaultImageUsageFlags_,
822 ((desc.memoryPropertyFlags != 0) ? desc.memoryPropertyFlags
823 : MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) &
824 GpuResourceDescFlagValidation::ALL_MEMORY_PROPERTY_FLAGS,
825 desc.createFlags & GpuResourceDescFlagValidation::ALL_IMAGE_CREATE_FLAGS,
826 desc.engineCreationFlags,
827 Math::min(MAX_IMAGE_EXTENT, Math::max(1u, desc.width)),
828 Math::min(MAX_IMAGE_EXTENT, Math::max(1u, desc.height)),
829 Math::min(MAX_IMAGE_EXTENT, Math::max(1u, desc.depth)),
830 Math::max(1u,
831 Math::min(desc.mipCount,
832 static_cast<uint32_t>(std::log2f(static_cast<float>(Math::max(desc.width, desc.height)))) + 1u)),
833 Math::max(1u, desc.layerCount),
834 Math::max(1u, desc.sampleCountFlags),
835 desc.componentMapping,
836 } },
837 name, replacedHandle, RenderHandleType::GPU_IMAGE, ~0u, 0u
838 };
839 if (info.descriptor.imageDescriptor.format == Format::BASE_FORMAT_UNDEFINED) {
840 PLUGIN_LOG_E("Undefined image BASE_FORMAT_UNDEFINED (input format %u)", static_cast<uint32_t>(desc.format));
841 return {};
842 }
843
844 return StoreAllocation(store, info);
845 }
846
Create(const RenderHandleReference & replacedHandle,const GpuImageDesc & desc)847 RenderHandleReference GpuResourceManager::Create(const RenderHandleReference& replacedHandle, const GpuImageDesc& desc)
848 {
849 const RenderHandle rawHandle = replacedHandle.GetHandle();
850 #if (RENDER_VALIDATION_ENABLED == 1)
851 const bool valid = RenderHandleUtil::IsValid(rawHandle);
852 const RenderHandleType type = RenderHandleUtil::GetHandleType(rawHandle);
853 if (valid && (type != RenderHandleType::GPU_IMAGE)) {
854 PLUGIN_LOG_E(
855 "RENDER_VALIDATION: trying to replace a non GPU image handle (type: %u) with GpuImageDesc", (uint32_t)type);
856 }
857 #endif
858 PerManagerStore& store = imageStore_;
859 const auto lock = std::lock_guard(store.clientMutex);
860
861 return CreateImage({}, rawHandle, desc).handle;
862 }
863
GetOrCreate(const string_view name,const GpuImageDesc & desc)864 RenderHandleReference GpuResourceManager::GetOrCreate(const string_view name, const GpuImageDesc& desc)
865 {
866 RenderHandleReference handle;
867 {
868 PerManagerStore& store = imageStore_;
869 const auto lock = std::lock_guard(store.clientMutex);
870
871 // check if present (not locked inside)
872 handle = GetHandleNoLock(store, name);
873 if (!handle) {
874 handle = CreateImage(name, {}, desc).handle;
875 }
876 }
877 return handle;
878 }
879
Create(const string_view name,const GpuImageDesc & desc)880 RenderHandleReference GpuResourceManager::Create(const string_view name, const GpuImageDesc& desc)
881 {
882 PerManagerStore& store = imageStore_;
883 const auto lock = std::lock_guard(store.clientMutex);
884
885 return CreateImage(name, {}, desc).handle;
886 }
887
Create(const GpuImageDesc & desc)888 RenderHandleReference GpuResourceManager::Create(const GpuImageDesc& desc)
889 {
890 PerManagerStore& store = imageStore_;
891 const auto lock = std::lock_guard(store.clientMutex);
892
893 return CreateImage({}, {}, desc).handle;
894 }
895
RemapGpuImageHandle(const RenderHandle & clientHandle,const RenderHandle & clientHandleGpuResource)896 void GpuResourceManager::RemapGpuImageHandle(
897 const RenderHandle& clientHandle, const RenderHandle& clientHandleGpuResource)
898 {
899 bool validClientHandles = (RenderHandleUtil::GetHandleType(clientHandle) == RenderHandleType::GPU_IMAGE) ||
900 (RenderHandleUtil::GetHandleType(clientHandleGpuResource) == RenderHandleType::GPU_IMAGE);
901 if (validClientHandles) {
902 PerManagerStore& store = imageStore_;
903 auto const lock = std::lock_guard(store.clientMutex);
904
905 const uint32_t clientArrayIndex = RenderHandleUtil::GetIndexPart(clientHandle);
906 const uint32_t clientResourceArrayIndex = RenderHandleUtil::GetIndexPart(clientHandleGpuResource);
907 validClientHandles =
908 validClientHandles && ((clientArrayIndex < (uint32_t)store.clientHandles.size()) &&
909 (clientResourceArrayIndex < (uint32_t)store.clientHandles.size()));
910 if (validClientHandles) {
911 store.descriptions[clientArrayIndex] = store.descriptions[clientResourceArrayIndex];
912 store.pendingData.remaps.push_back(RemapDescription { clientHandle, clientHandleGpuResource });
913 }
914 }
915
916 if (!validClientHandles) {
917 PLUGIN_LOG_E("invalid client handles given to RemapGpuImageHandle()");
918 }
919 }
920
Create(const string_view name,const GpuImageDesc & desc,IImageContainer::Ptr image)921 RenderHandleReference GpuResourceManager::Create(
922 const string_view name, const GpuImageDesc& desc, IImageContainer::Ptr image)
923 {
924 StoreAllocationData sad;
925 if (image) {
926 PerManagerStore& store = imageStore_;
927 auto const lockImg = std::lock_guard(store.clientMutex);
928
929 sad = CreateImage(name, {}, desc);
930 if (IsGpuImage(sad.handle)) {
931 auto const lockStag = std::lock_guard(stagingMutex_);
932
933 const auto& copies = image->GetBufferImageCopies();
934 const bool scaleImage = GetScalingImageNeed(desc, copies);
935
936 const uint32_t stagingBufferByteOffset = stagingOperations_.stagingByteSize;
937 Format format = Format::BASE_FORMAT_UNDEFINED;
938 if (scaleImage) { // needs to be locked
939 UpdateStagingScaling(desc.format, copies, stagingOperations_.scalingImageData);
940 format = desc.format;
941 }
942 for (const auto& copyRef : copies) {
943 stagingOperations_.bufferImageCopies.push_back(
944 ConvertCoreBufferImageCopy(copyRef, stagingBufferByteOffset));
945 }
946
947 // add staging handle to resource handle in pending allocations
948 PLUGIN_ASSERT(sad.allocationIndex < store.pendingData.allocations.size());
949 auto& allocRef = store.pendingData.allocations[sad.allocationIndex];
950 allocRef.optionalStagingVectorIndex = static_cast<uint32_t>(stagingOperations_.bufferToImage.size());
951 allocRef.optionalStagingCopyType = StagingCopyStruct::CopyType::BUFFER_TO_IMAGE;
952
953 const uint32_t stagingBufferByteSize = static_cast<uint32_t>(image->GetData().size_bytes());
954 stagingOperations_.stagingByteSize =
955 Align(stagingOperations_.stagingByteSize + stagingBufferByteSize, BUFFER_ALIGNMENT);
956 const auto count = static_cast<uint32_t>(copies.size());
957 const uint32_t beginIndex = static_cast<uint32_t>(stagingOperations_.bufferImageCopies.size()) - count;
958 stagingOperations_.bufferToImage.push_back(
959 StagingCopyStruct { StagingCopyStruct::DataType::DATA_TYPE_IMAGE_CONTAINER, {}, sad.handle, beginIndex,
960 count, {}, move(image), format, stagingBufferByteOffset, stagingBufferByteSize, false });
961 }
962 } else {
963 PLUGIN_LOG_E("invalid image pointer to Create GPU image");
964 }
965 return sad.handle;
966 }
967
Create(const string_view name,const GpuImageDesc & desc,const array_view<const uint8_t> data,const array_view<const BufferImageCopy> bufferImageCopies)968 RenderHandleReference GpuResourceManager::Create(const string_view name, const GpuImageDesc& desc,
969 const array_view<const uint8_t> data, const array_view<const BufferImageCopy> bufferImageCopies)
970 {
971 StoreAllocationData sad;
972 {
973 PerManagerStore& store = imageStore_;
974 auto const lockImg = std::lock_guard(store.clientMutex);
975
976 sad = CreateImage(name, {}, desc);
977 if (IsGpuImage(sad.handle)) {
978 auto const lockStag = std::lock_guard(stagingMutex_);
979
980 const uint32_t stagingBufferByteOffset = stagingOperations_.stagingByteSize;
981 Format format = Format::BASE_FORMAT_UNDEFINED;
982 if (GetScalingImageNeed(desc, bufferImageCopies)) { // needs to be locked
983 UpdateStagingScaling(desc.format, bufferImageCopies, stagingOperations_.scalingImageData);
984 format = desc.format;
985 }
986 for (const auto& copyRef : bufferImageCopies) {
987 #if (RENDER_VALIDATION_ENABLED == 1)
988 ValidateGpuImageCopy(desc, copyRef, name);
989 #endif
990 BufferImageCopy bic = copyRef;
991 bic.bufferOffset += stagingBufferByteOffset; // add additional staging offset
992 stagingOperations_.bufferImageCopies.push_back(bic);
993 }
994 // add staging vector index to resource alloc in pending allocations
995 PLUGIN_ASSERT(sad.allocationIndex < store.pendingData.allocations.size());
996 auto& allocRef = store.pendingData.allocations[sad.allocationIndex];
997 allocRef.optionalStagingVectorIndex = static_cast<uint32_t>(stagingOperations_.bufferToImage.size());
998 allocRef.optionalStagingCopyType = StagingCopyStruct::CopyType::BUFFER_TO_IMAGE;
999
1000 const auto stagingBufferByteSize = static_cast<uint32_t>(data.size_bytes());
1001 stagingOperations_.stagingByteSize =
1002 Align(stagingOperations_.stagingByteSize + stagingBufferByteSize, BUFFER_ALIGNMENT);
1003 const auto count = (uint32_t)bufferImageCopies.size();
1004 const uint32_t beginIndex = (uint32_t)stagingOperations_.bufferImageCopies.size() - count;
1005
1006 vector<uint8_t> copiedData(data.cbegin().ptr(), data.cend().ptr());
1007 stagingOperations_.bufferToImage.push_back(
1008 StagingCopyStruct { StagingCopyStruct::DataType::DATA_TYPE_VECTOR, {}, sad.handle, beginIndex, count,
1009 move(copiedData), nullptr, format, stagingBufferByteOffset, stagingBufferByteSize, false });
1010 }
1011 }
1012 return sad.handle;
1013 }
1014
Create(const string_view name,const GpuImageDesc & desc,const array_view<const uint8_t> data)1015 RenderHandleReference GpuResourceManager::Create(
1016 const string_view name, const GpuImageDesc& desc, const array_view<const uint8_t> data)
1017 {
1018 BufferImageCopy bufferImageCopy {
1019 0,
1020 desc.width,
1021 desc.height,
1022 { ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT, 0, 0, desc.layerCount },
1023 { 0, 0, 0 },
1024 { desc.width, desc.height, desc.depth },
1025 };
1026
1027 const array_view<const BufferImageCopy> av(&bufferImageCopy, 1);
1028 return Create(name, desc, data, av);
1029 }
1030
CreateView(const string_view name,const GpuImageDesc & desc,const GpuImagePlatformData & gpuImagePlatformData)1031 RenderHandleReference GpuResourceManager::CreateView(
1032 const string_view name, const GpuImageDesc& desc, const GpuImagePlatformData& gpuImagePlatformData)
1033 {
1034 device_.Activate();
1035 PerManagerStore& store = imageStore_;
1036 const auto lock = std::lock_guard(store.clientMutex);
1037
1038 // replace immediate created if still pending (i.e. not usable on the GPU)
1039 const auto emplaceResourceIndex = static_cast<uint32_t>(store.pendingData.images.size());
1040 const uint32_t optionalResourceIndex =
1041 Math::min(emplaceResourceIndex, GetPendingOptionalResourceIndex(store, {}, name));
1042
1043 if (unique_ptr<GpuImage> gpuImage = [this](const GpuImageDesc& desc,
1044 const GpuImagePlatformData& gpuImagePlatformData) {
1045 // protect GPU memory allocations
1046 auto lock = std::lock_guard(allocationMutex_);
1047 return device_.CreateGpuImageView(desc, gpuImagePlatformData);
1048 }(desc, gpuImagePlatformData)) {
1049 // safety checks
1050 if ((optionalResourceIndex < emplaceResourceIndex) &&
1051 (optionalResourceIndex < store.pendingData.images.size())) {
1052 store.pendingData.images[optionalResourceIndex] = move(gpuImage);
1053 } else {
1054 store.pendingData.images.push_back(move(gpuImage));
1055 }
1056 }
1057 device_.Deactivate();
1058
1059 return StoreAllocation(
1060 store, { ResourceDescriptor { desc }, name, {}, RenderHandleType::GPU_IMAGE, optionalResourceIndex, 0u })
1061 .handle;
1062 }
1063
CreateView(const string_view name,const GpuImageDesc & desc,const BackendSpecificImageDesc & backendSpecificData)1064 RenderHandleReference GpuResourceManager::CreateView(
1065 const string_view name, const GpuImageDesc& desc, const BackendSpecificImageDesc& backendSpecificData)
1066 {
1067 device_.Activate();
1068 PerManagerStore& store = imageStore_;
1069 const auto lock = std::lock_guard(store.clientMutex);
1070
1071 // replace immediate created if still pending (i.e. not usable on the GPU)
1072 const auto emplaceResourceIndex = static_cast<uint32_t>(store.pendingData.images.size());
1073 const uint32_t optionalResourceIndex =
1074 Math::min(emplaceResourceIndex, GetPendingOptionalResourceIndex(store, {}, name));
1075
1076 // additional handle flags provide information if platform conversion is needed
1077 uint32_t additionalHandleFlags = 0u;
1078
1079 if (unique_ptr<GpuImage> gpuImage = [this](const GpuImageDesc& desc,
1080 const BackendSpecificImageDesc& backendSpecificData) {
1081 // protect GPU memory allocations
1082 auto lock = std::lock_guard(allocationMutex_);
1083 return device_.CreateGpuImageView(desc, backendSpecificData);
1084 }(desc, backendSpecificData)) {
1085 const auto additionalImageFlags = gpuImage->GetAdditionalFlags();
1086 additionalHandleFlags =
1087 (additionalImageFlags & GpuImage::AdditionalFlagBits::ADDITIONAL_PLATFORM_CONVERSION_BIT)
1088 ? CORE_RESOURCE_HANDLE_PLATFORM_CONVERSION
1089 : 0u;
1090 // safety checks
1091 if ((optionalResourceIndex < emplaceResourceIndex) &&
1092 (optionalResourceIndex < store.pendingData.images.size())) {
1093 store.pendingData.images[optionalResourceIndex] = move(gpuImage);
1094 } else {
1095 store.pendingData.images.push_back(move(gpuImage));
1096 }
1097 }
1098 device_.Deactivate();
1099
1100 const auto& images = store.pendingData.images;
1101 const auto& finalDesc = (optionalResourceIndex < images.size() && images[optionalResourceIndex])
1102 ? images[optionalResourceIndex]->GetDesc()
1103 : desc;
1104 auto handle = StoreAllocation(store, { ResourceDescriptor { finalDesc }, name, {}, RenderHandleType::GPU_IMAGE,
1105 optionalResourceIndex, additionalHandleFlags })
1106 .handle;
1107 #if (RENDER_VALIDATION_ENABLED == 1)
1108 if ((additionalHandleFlags & CORE_RESOURCE_HANDLE_PLATFORM_CONVERSION) &&
1109 !RenderHandleUtil::IsPlatformConversionResource(handle.GetHandle())) {
1110 PLUGIN_LOG_ONCE_W("core_validation_create_view_plat_conversion",
1111 "RENDER_VALIDATION: platform conversion needing resource cannot replace existing resource handle (name: "
1112 "%s)",
1113 name.data());
1114 }
1115 #endif
1116 return handle;
1117 }
1118
Create(const GpuImageDesc & desc,const array_view<const uint8_t> data,const array_view<const BufferImageCopy> bufferImageCopies)1119 RenderHandleReference GpuResourceManager::Create(const GpuImageDesc& desc, const array_view<const uint8_t> data,
1120 const array_view<const BufferImageCopy> bufferImageCopies)
1121 {
1122 return Create({}, desc, data, bufferImageCopies);
1123 }
1124
Create(const GpuImageDesc & desc,const array_view<const uint8_t> data)1125 RenderHandleReference GpuResourceManager::Create(const GpuImageDesc& desc, const array_view<const uint8_t> data)
1126 {
1127 return Create({}, desc, data);
1128 }
1129
Create(const GpuImageDesc & desc,IImageContainer::Ptr image)1130 RenderHandleReference GpuResourceManager::Create(const GpuImageDesc& desc, IImageContainer::Ptr image)
1131 {
1132 return Create({}, desc, move(image));
1133 }
1134
GetOrCreate(const string_view name,const GpuSamplerDesc & desc)1135 RenderHandleReference GpuResourceManager::GetOrCreate(const string_view name, const GpuSamplerDesc& desc)
1136 {
1137 RenderHandleReference handle;
1138 {
1139 PerManagerStore& store = samplerStore_;
1140 const auto lock = std::lock_guard(store.clientMutex);
1141
1142 // check if present (not locked inside)
1143 handle = GetHandleNoLock(store, name);
1144 if (!handle) {
1145 handle = StoreAllocation(
1146 store, { ResourceDescriptor { desc }, name, {}, RenderHandleType::GPU_SAMPLER, ~0u, 0u })
1147 .handle;
1148 }
1149 }
1150 return handle;
1151 }
1152
Create(const string_view name,const GpuSamplerDesc & desc)1153 RenderHandleReference GpuResourceManager::Create(const string_view name, const GpuSamplerDesc& desc)
1154 {
1155 PerManagerStore& store = samplerStore_;
1156 const auto lock = std::lock_guard(store.clientMutex);
1157
1158 return StoreAllocation(store, { ResourceDescriptor { desc }, name, {}, RenderHandleType::GPU_SAMPLER, ~0u, 0u })
1159 .handle;
1160 }
1161
Create(const RenderHandleReference & replacedHandle,const GpuSamplerDesc & desc)1162 RenderHandleReference GpuResourceManager::Create(
1163 const RenderHandleReference& replacedHandle, const GpuSamplerDesc& desc)
1164 {
1165 const RenderHandle rawHandle = replacedHandle.GetHandle();
1166 #if (RENDER_VALIDATION_ENABLED == 1)
1167 const bool valid = RenderHandleUtil::IsValid(rawHandle);
1168 const RenderHandleType type = RenderHandleUtil::GetHandleType(rawHandle);
1169 if (valid && (type != RenderHandleType::GPU_SAMPLER)) {
1170 PLUGIN_LOG_E("RENDER_VALIDATION: trying to replace a non GPU sampler handle (type: %u) with GpuSamplerDesc",
1171 (uint32_t)type);
1172 }
1173 #endif
1174 PerManagerStore& store = samplerStore_;
1175 const auto lock = std::lock_guard(store.clientMutex);
1176
1177 return StoreAllocation(
1178 store, { ResourceDescriptor { desc }, {}, rawHandle, RenderHandleType::GPU_SAMPLER, ~0u, 0u })
1179 .handle;
1180 }
1181
Create(const GpuSamplerDesc & desc)1182 RenderHandleReference GpuResourceManager::Create(const GpuSamplerDesc& desc)
1183 {
1184 return Create("", desc);
1185 }
1186
CreateAccelerationStructure(const BASE_NS::string_view name,const RenderHandle & replacedHandle,const GpuAccelerationStructureDesc & desc)1187 GpuResourceManager::StoreAllocationData GpuResourceManager::CreateAccelerationStructure(
1188 const BASE_NS::string_view name, const RenderHandle& replacedHandle, const GpuAccelerationStructureDesc& desc)
1189 {
1190 PerManagerStore& store = bufferStore_;
1191 #if (RENDER_VALIDATION_ENABLED == 1)
1192 ValidateGpuAccelerationStructureDesc(desc);
1193 #endif
1194 GpuAccelerationStructureDesc validatedDesc = desc;
1195 validatedDesc.bufferDesc.usageFlags |= defaultBufferUsageFlags_;
1196 validatedDesc.bufferDesc.byteSize = Math::max(validatedDesc.bufferDesc.byteSize, 1u),
1197 validatedDesc.bufferDesc.usageFlags |=
1198 CORE_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT | CORE_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
1199
1200 constexpr auto additionalBufferFlags = CORE_RESOURCE_HANDLE_ACCELERATION_STRUCTURE;
1201 return StoreAllocation(store, { ResourceDescriptor { validatedDesc }, name, replacedHandle,
1202 RenderHandleType::GPU_BUFFER, ~0u, additionalBufferFlags });
1203 }
1204
Create(const GpuAccelerationStructureDesc & desc)1205 RenderHandleReference GpuResourceManager::Create(const GpuAccelerationStructureDesc& desc)
1206 {
1207 PerManagerStore& store = bufferStore_;
1208 const auto lock = std::lock_guard(store.clientMutex);
1209
1210 return CreateAccelerationStructure("", {}, desc).handle;
1211 }
1212
Create(const string_view name,const GpuAccelerationStructureDesc & desc)1213 RenderHandleReference GpuResourceManager::Create(const string_view name, const GpuAccelerationStructureDesc& desc)
1214 {
1215 PerManagerStore& store = bufferStore_;
1216 const auto lock = std::lock_guard(store.clientMutex);
1217
1218 return CreateAccelerationStructure(name, {}, desc).handle;
1219 }
1220
Create(const RenderHandleReference & replacedHandle,const GpuAccelerationStructureDesc & desc)1221 RenderHandleReference GpuResourceManager::Create(
1222 const RenderHandleReference& replacedHandle, const GpuAccelerationStructureDesc& desc)
1223 {
1224 const RenderHandle rawHandle = replacedHandle.GetHandle();
1225 #if (RENDER_VALIDATION_ENABLED == 1)
1226 const bool valid = RenderHandleUtil::IsValid(rawHandle);
1227 const RenderHandleType type = RenderHandleUtil::GetHandleType(rawHandle);
1228 if (valid &&
1229 ((type != RenderHandleType::GPU_BUFFER) || (!RenderHandleUtil::IsGpuAccelerationStructure(rawHandle)))) {
1230 PLUGIN_LOG_E("RENDER_VALIDATION: trying to replace a non GPU acceleration structure handle (type: %u) with "
1231 "GpuAccelerationStructureDesc",
1232 (uint32_t)type);
1233 }
1234 #endif
1235 PerManagerStore& store = bufferStore_;
1236 const auto lock = std::lock_guard(store.clientMutex);
1237
1238 return CreateAccelerationStructure("", rawHandle, desc).handle;
1239 }
1240
CreateSwapchainImage(const RenderHandleReference & replacedHandle,const BASE_NS::string_view name,const GpuImageDesc & desc)1241 RenderHandleReference GpuResourceManager::CreateSwapchainImage(
1242 const RenderHandleReference& replacedHandle, const BASE_NS::string_view name, const GpuImageDesc& desc)
1243 {
1244 #if (RENDER_VALIDATION_ENABLED == 1)
1245 ValidateGpuImageDesc(desc, "");
1246 #endif
1247
1248 PerManagerStore& store = imageStore_;
1249 const auto lock = std::lock_guard(store.clientMutex);
1250
1251 const uint32_t addFlags = RenderHandleInfoFlagBits::CORE_RESOURCE_HANDLE_SWAPCHAIN_RESOURCE;
1252 // NOTE: no mips for swapchains
1253 // NOTE: allocation type is undefined
1254 const StoreAllocationInfo info {
1255 ResourceDescriptor { GpuImageDesc {
1256 desc.imageType,
1257 desc.imageViewType,
1258 device_.GetFormatOrFallback(desc.format),
1259 (desc.imageTiling > ImageTiling::CORE_IMAGE_TILING_LINEAR) ? ImageTiling::CORE_IMAGE_TILING_OPTIMAL
1260 : desc.imageTiling,
1261 (desc.usageFlags & GpuResourceDescFlagValidation::ALL_GPU_IMAGE_USAGE_FLAGS) | defaultImageUsageFlags_,
1262 ((desc.memoryPropertyFlags != 0) ? desc.memoryPropertyFlags
1263 : MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) &
1264 GpuResourceDescFlagValidation::ALL_MEMORY_PROPERTY_FLAGS,
1265 desc.createFlags & GpuResourceDescFlagValidation::ALL_IMAGE_CREATE_FLAGS,
1266 desc.engineCreationFlags,
1267 Math::min(MAX_IMAGE_EXTENT, Math::max(1u, desc.width)),
1268 Math::min(MAX_IMAGE_EXTENT, Math::max(1u, desc.height)),
1269 Math::min(MAX_IMAGE_EXTENT, Math::max(1u, desc.depth)),
1270 1u, // hard-coded mip count
1271 Math::max(1u, desc.layerCount),
1272 Math::max(1u, desc.sampleCountFlags),
1273 desc.componentMapping,
1274 } },
1275 name, replacedHandle.GetHandle(), RenderHandleType::GPU_IMAGE, ~0u, addFlags, AllocType::UNDEFINED
1276 };
1277 if (info.descriptor.imageDescriptor.format == Format::BASE_FORMAT_UNDEFINED) {
1278 PLUGIN_LOG_E("Undefined image BASE_FORMAT_UNDEFINED (input format %u)", static_cast<uint32_t>(desc.format));
1279 return {};
1280 }
1281
1282 return StoreAllocation(store, info).handle;
1283 }
1284
CreateShallowHandle(const GpuImageDesc & desc)1285 RenderHandleReference GpuResourceManager::CreateShallowHandle(const GpuImageDesc& desc)
1286 {
1287 PerManagerStore& store = imageStore_;
1288 const auto lock = std::lock_guard(store.clientMutex);
1289
1290 #if (RENDER_VALIDATION_ENABLED == 1)
1291 ValidateGpuImageDesc(desc, "");
1292 #endif
1293
1294 const uint32_t addFlags = RenderHandleInfoFlagBits::CORE_RESOURCE_HANDLE_SHALLOW_RESOURCE;
1295 const StoreAllocationInfo info {
1296 ResourceDescriptor { GpuImageDesc {
1297 desc.imageType,
1298 desc.imageViewType,
1299 device_.GetFormatOrFallback(desc.format),
1300 (desc.imageTiling > ImageTiling::CORE_IMAGE_TILING_LINEAR) ? ImageTiling::CORE_IMAGE_TILING_OPTIMAL
1301 : desc.imageTiling,
1302 (desc.usageFlags & GpuResourceDescFlagValidation::ALL_GPU_IMAGE_USAGE_FLAGS) | defaultImageUsageFlags_,
1303 ((desc.memoryPropertyFlags != 0) ? desc.memoryPropertyFlags
1304 : MemoryPropertyFlagBits::CORE_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) &
1305 GpuResourceDescFlagValidation::ALL_MEMORY_PROPERTY_FLAGS,
1306 desc.createFlags & GpuResourceDescFlagValidation::ALL_IMAGE_CREATE_FLAGS,
1307 desc.engineCreationFlags,
1308 Math::min(MAX_IMAGE_EXTENT, Math::max(1u, desc.width)),
1309 Math::min(MAX_IMAGE_EXTENT, Math::max(1u, desc.height)),
1310 Math::min(MAX_IMAGE_EXTENT, Math::max(1u, desc.depth)),
1311 Math::max(1u,
1312 Math::min(desc.mipCount,
1313 static_cast<uint32_t>(std::log2f(static_cast<float>(Math::max(desc.width, desc.height)))) + 1u)),
1314 Math::max(1u, desc.layerCount),
1315 Math::max(1u, desc.sampleCountFlags),
1316 desc.componentMapping,
1317 } },
1318 "", {}, RenderHandleType::GPU_IMAGE, ~0u, addFlags, AllocType::UNDEFINED
1319 };
1320 if (info.descriptor.imageDescriptor.format == Format::BASE_FORMAT_UNDEFINED) {
1321 PLUGIN_LOG_E("Undefined image BASE_FORMAT_UNDEFINED (input format %u)", static_cast<uint32_t>(desc.format));
1322 return {};
1323 }
1324
1325 return StoreAllocation(store, info).handle;
1326 }
1327
1328 // has staging lock and possible gpu buffer lock inside
RemoveStagingOperations(const OperationDescription & destroyAlloc)1329 void GpuResourceManager::RemoveStagingOperations(const OperationDescription& destroyAlloc)
1330 {
1331 // remove possible stagings
1332 const RenderHandleType handleType = RenderHandleUtil::GetHandleType(destroyAlloc.handle);
1333 if (((handleType == RenderHandleType::GPU_BUFFER) || (handleType == RenderHandleType::GPU_IMAGE)) &&
1334 (destroyAlloc.optionalStagingCopyType != StagingCopyStruct::CopyType::UNDEFINED)) {
1335 auto const lockStaging = std::lock_guard(stagingMutex_);
1336
1337 auto invalidateStagingCopy = [](const OperationDescription& alloc, vector<StagingCopyStruct>& vecRef) {
1338 PLUGIN_ASSERT(alloc.optionalStagingVectorIndex < vecRef.size());
1339 vecRef[alloc.optionalStagingVectorIndex].srcHandle = {};
1340 vecRef[alloc.optionalStagingVectorIndex].dstHandle = {};
1341 vecRef[alloc.optionalStagingVectorIndex].invalidOperation = true;
1342 };
1343
1344 if (destroyAlloc.optionalStagingCopyType == StagingCopyStruct::CopyType::BUFFER_TO_BUFFER) {
1345 invalidateStagingCopy(destroyAlloc, stagingOperations_.bufferToBuffer);
1346 } else if (destroyAlloc.optionalStagingCopyType == StagingCopyStruct::CopyType::BUFFER_TO_IMAGE) {
1347 invalidateStagingCopy(destroyAlloc, stagingOperations_.bufferToImage);
1348 } else if (destroyAlloc.optionalStagingCopyType == StagingCopyStruct::CopyType::IMAGE_TO_BUFFER) {
1349 invalidateStagingCopy(destroyAlloc, stagingOperations_.imageToBuffer);
1350 } else if (destroyAlloc.optionalStagingCopyType == StagingCopyStruct::CopyType::IMAGE_TO_IMAGE) {
1351 invalidateStagingCopy(destroyAlloc, stagingOperations_.imageToImage);
1352 } else if (destroyAlloc.optionalStagingCopyType == StagingCopyStruct::CopyType::CPU_TO_BUFFER) {
1353 invalidateStagingCopy(destroyAlloc, stagingOperations_.cpuToBuffer);
1354 } else {
1355 PLUGIN_ASSERT(false);
1356 }
1357 }
1358
1359 // NOTE: we do not clean-up/invalidate copy operations stagingOperations_.cpuToBuffer ATM
1360 // it is user's responsibility do not use handle that you've destroyed
1361 }
1362
1363 // needs to be locked from outside
1364 // staging cannot be locked when called
Destroy(PerManagerStore & store,const RenderHandle & handle)1365 void GpuResourceManager::Destroy(PerManagerStore& store, const RenderHandle& handle)
1366 {
1367 #if (RENDER_DEBUG_GPU_RESOURCE_IDS == 1)
1368 PLUGIN_LOG_E("gpu resource deallocation %" PRIx64, handle.id);
1369 #endif
1370
1371 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
1372 if (arrayIndex >= store.clientHandles.size()) {
1373 return;
1374 }
1375 if (!(store.clientHandles[arrayIndex])) {
1376 return; // early out if re-destroying the same handle
1377 }
1378 #if (RENDER_VALIDATION_ENABLED == 1)
1379 const uint32_t currGeneration =
1380 RenderHandleUtil::GetGenerationIndexPart(store.clientHandles[arrayIndex].GetHandle());
1381 const uint32_t destroyHandleGeneration = RenderHandleUtil::GetGenerationIndexPart(handle);
1382 if (currGeneration != destroyHandleGeneration) {
1383 PLUGIN_LOG_W("RENDER_VALIDATION: destroy handle is not the current generation (destroy:%u != current:%u)",
1384 currGeneration, destroyHandleGeneration);
1385 }
1386 #endif
1387
1388 if (const uint32_t hasNameId = RenderHandleUtil::GetHasNamePart(handle); hasNameId != 0U) {
1389 // remove name if present
1390 if (auto const pos =
1391 std::find_if(store.nameToClientIndex.begin(), store.nameToClientIndex.end(),
1392 [arrayIndex](auto const &nameToHandle) { return nameToHandle.second == arrayIndex; });
1393 pos != store.nameToClientIndex.end()) {
1394 store.nameToClientIndex.erase(pos);
1395 }
1396 }
1397
1398 // we do not set default values to GpuXDesc (we leave the old data, it won't be used)
1399 // old code: store.descriptions[arrayIndex] = {};
1400 // invalidate handle, noticed when trying to re-destroy (early-out in the beginning of the if)
1401 store.clientHandles[arrayIndex] = {};
1402
1403 // if the handle is already found and it's an alloc we do not want to allocate and then deallocate
1404 if (const uint32_t pendingArrIndex = store.additionalData[arrayIndex].indexToPendingData;
1405 pendingArrIndex != INVALID_PENDING_INDEX) {
1406 // NOTE: check valid assert here (if called before resources are created)
1407 if (pendingArrIndex < store.pendingData.allocations.size()) {
1408 auto& ref = store.pendingData.allocations[pendingArrIndex];
1409 if (ref.allocType == AllocType::ALLOC) {
1410 ref.allocType = AllocType::REMOVED;
1411 RemoveStagingOperations(ref);
1412 }
1413 }
1414 } else {
1415 PLUGIN_ASSERT(store.additionalData[arrayIndex].indexToPendingData == INVALID_PENDING_INDEX);
1416 store.additionalData[arrayIndex] = { 0, static_cast<uint32_t>(store.pendingData.allocations.size()) };
1417 store.pendingData.allocations.emplace_back(
1418 handle, ResourceDescriptor { GpuBufferDesc {} }, AllocType::DEALLOC, static_cast<uint32_t>(~0u));
1419 // there cannot be staging operations because pendingData was not found
1420 // all other operations to destroyable handle are user's responsibility
1421 }
1422 }
1423
1424 // needs to be locked when called
DestroyImmediate(PerManagerStore & store,const RenderHandle & handle)1425 void GpuResourceManager::DestroyImmediate(PerManagerStore& store, const RenderHandle& handle)
1426 {
1427 if (RenderHandleUtil::IsValid(handle)) { // found, Destroy immediate
1428 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
1429 if (arrayIndex < (uint32_t)store.gpuHandles.size()) {
1430 store.mgr->DestroyImmediate(arrayIndex);
1431 store.clientHandles[arrayIndex] = {};
1432 store.additionalData[arrayIndex] = {};
1433 store.gpuHandles[arrayIndex] = InvalidateWithGeneration(store.gpuHandles[arrayIndex]);
1434 }
1435 }
1436 }
1437
Destroy(const RenderHandle & handle)1438 void GpuResourceManager::Destroy(const RenderHandle& handle)
1439 {
1440 if (RenderHandleUtil::IsValid(handle)) {
1441 const RenderHandleType handleType = RenderHandleUtil::GetHandleType(handle);
1442 if (handleType == RenderHandleType::GPU_BUFFER) {
1443 const auto clientLock = std::lock_guard(bufferStore_.clientMutex);
1444 Destroy(bufferStore_, handle);
1445 } else if (handleType == RenderHandleType::GPU_IMAGE) {
1446 const auto clientLock = std::lock_guard(imageStore_.clientMutex);
1447 Destroy(imageStore_, handle);
1448 } else if (handleType == RenderHandleType::GPU_SAMPLER) {
1449 const auto clientLock = std::lock_guard(samplerStore_.clientMutex);
1450 Destroy(samplerStore_, handle);
1451 } else {
1452 PLUGIN_LOG_I("invalid gpu resource handle : %" PRIu64, handle.id);
1453 }
1454 }
1455 }
1456
1457 // needs to be locked outside
GetHandleNoLock(const PerManagerStore & store,const string_view name)1458 RenderHandleReference GpuResourceManager::GetHandleNoLock(const PerManagerStore& store, const string_view name)
1459 {
1460 if (name.empty()) { // early out before lock
1461 return RenderHandleReference {};
1462 }
1463
1464 if (auto const pos = store.nameToClientIndex.find(name); pos != store.nameToClientIndex.end()) {
1465 PLUGIN_ASSERT(pos->second < static_cast<uint32_t>(store.clientHandles.size()));
1466 return store.clientHandles[pos->second];
1467 }
1468 // NOTE: This is used in some places to check if the GPU resource is found
1469 // therefore no error prints here
1470 return RenderHandleReference {};
1471 }
1472
GetHandle(const PerManagerStore & store,const string_view name)1473 RenderHandleReference GpuResourceManager::GetHandle(const PerManagerStore& store, const string_view name)
1474 {
1475 if (name.empty()) { // early out before lock
1476 return RenderHandleReference {};
1477 }
1478
1479 auto const clientLock = std::shared_lock(store.clientMutex);
1480 if (auto const pos = store.nameToClientIndex.find(name); pos != store.nameToClientIndex.end()) {
1481 PLUGIN_ASSERT(pos->second < static_cast<uint32_t>(store.clientHandles.size()));
1482 return store.clientHandles[pos->second];
1483 }
1484 // NOTE: This is used in some places to check if the GPU resource is found
1485 // therefore no error prints here
1486 return RenderHandleReference {};
1487 }
1488
GetRawHandle(const PerManagerStore & store,const string_view name)1489 RenderHandle GpuResourceManager::GetRawHandle(const PerManagerStore& store, const string_view name)
1490 {
1491 if (name.empty()) { // early out before lock
1492 return RenderHandle {};
1493 }
1494
1495 auto const clientLock = std::shared_lock(store.clientMutex);
1496 if (auto const pos = store.nameToClientIndex.find(name); pos != store.nameToClientIndex.end()) {
1497 PLUGIN_ASSERT(pos->second < static_cast<uint32_t>(store.clientHandles.size()));
1498 return store.clientHandles[pos->second].GetHandle();
1499 }
1500 // NOTE: This is used in some places to check if the GPU resource is found
1501 // therefore no error prints here
1502 return RenderHandle {};
1503 }
1504
HasNamedResource(const PerManagerStore & store,const string_view name)1505 bool GpuResourceManager::HasNamedResource(const PerManagerStore& store, const string_view name)
1506 {
1507 if (name.empty()) { // early out before lock
1508 return false;
1509 }
1510
1511 auto const clientLock = std::shared_lock(store.clientMutex);
1512 if (auto const pos = store.nameToClientIndex.find(name); pos != store.nameToClientIndex.end()) {
1513 PLUGIN_ASSERT(pos->second < static_cast<uint32_t>(store.clientHandles.size()));
1514 return true;
1515 }
1516 return false;
1517 }
1518
GetBufferHandle(const string_view name) const1519 RenderHandleReference GpuResourceManager::GetBufferHandle(const string_view name) const
1520 {
1521 return GetHandle(bufferStore_, name);
1522 }
1523
GetImageHandle(const string_view name) const1524 RenderHandleReference GpuResourceManager::GetImageHandle(const string_view name) const
1525 {
1526 return GetHandle(imageStore_, name);
1527 }
1528
GetSamplerHandle(const string_view name) const1529 RenderHandleReference GpuResourceManager::GetSamplerHandle(const string_view name) const
1530 {
1531 return GetHandle(samplerStore_, name);
1532 }
1533
HasBuffer(const BASE_NS::string_view name) const1534 bool GpuResourceManager::HasBuffer(const BASE_NS::string_view name) const
1535 {
1536 return HasNamedResource(bufferStore_, name);
1537 }
1538
HasImage(const BASE_NS::string_view name) const1539 bool GpuResourceManager::HasImage(const BASE_NS::string_view name) const
1540 {
1541 return HasNamedResource(imageStore_, name);
1542 }
1543
HasSampler(const BASE_NS::string_view name) const1544 bool GpuResourceManager::HasSampler(const BASE_NS::string_view name) const
1545 {
1546 return HasNamedResource(samplerStore_, name);
1547 }
1548
GetBufferRawHandle(const string_view name) const1549 RenderHandle GpuResourceManager::GetBufferRawHandle(const string_view name) const
1550 {
1551 return GetRawHandle(bufferStore_, name);
1552 }
1553
GetImageRawHandle(const string_view name) const1554 RenderHandle GpuResourceManager::GetImageRawHandle(const string_view name) const
1555 {
1556 return GetRawHandle(imageStore_, name);
1557 }
1558
GetSamplerRawHandle(const string_view name) const1559 RenderHandle GpuResourceManager::GetSamplerRawHandle(const string_view name) const
1560 {
1561 return GetRawHandle(samplerStore_, name);
1562 }
1563
GetBufferDescriptor(const RenderHandle & handle) const1564 GpuBufferDesc GpuResourceManager::GetBufferDescriptor(const RenderHandle& handle) const
1565 {
1566 if (!IsGpuBuffer(handle)) {
1567 return GpuBufferDesc {};
1568 }
1569 {
1570 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
1571 const PerManagerStore& store = bufferStore_;
1572 auto const clientLock = std::shared_lock(store.clientMutex);
1573 if (arrayIndex < (uint32_t)store.descriptions.size()) {
1574 return store.descriptions[arrayIndex].combinedBufDescriptor.bufferDesc;
1575 }
1576 }
1577 return GpuBufferDesc {};
1578 }
1579
GetBufferDescriptor(const RenderHandleReference & handle) const1580 GpuBufferDesc GpuResourceManager::GetBufferDescriptor(const RenderHandleReference& handle) const
1581 {
1582 return GetBufferDescriptor(handle.GetHandle());
1583 }
1584
GetImageDescriptor(const RenderHandle & handle) const1585 GpuImageDesc GpuResourceManager::GetImageDescriptor(const RenderHandle& handle) const
1586 {
1587 if (!IsGpuImage(handle)) {
1588 return GpuImageDesc {};
1589 }
1590 {
1591 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
1592 const PerManagerStore& store = imageStore_;
1593 auto const clientLock = std::shared_lock(store.clientMutex);
1594 if (arrayIndex < (uint32_t)store.descriptions.size()) {
1595 return store.descriptions[arrayIndex].imageDescriptor;
1596 }
1597 }
1598 return GpuImageDesc {};
1599 }
1600
GetImageDescriptor(const RenderHandleReference & handle) const1601 GpuImageDesc GpuResourceManager::GetImageDescriptor(const RenderHandleReference& handle) const
1602 {
1603 return GetImageDescriptor(handle.GetHandle());
1604 }
1605
GetSamplerDescriptor(const RenderHandle & handle) const1606 GpuSamplerDesc GpuResourceManager::GetSamplerDescriptor(const RenderHandle& handle) const
1607 {
1608 if (!IsGpuSampler(handle)) {
1609 return GpuSamplerDesc {};
1610 }
1611 {
1612 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
1613 const PerManagerStore& store = samplerStore_;
1614 auto const clientLock = std::shared_lock(store.clientMutex);
1615 if (arrayIndex < (uint32_t)store.descriptions.size()) {
1616 return store.descriptions[arrayIndex].samplerDescriptor;
1617 }
1618 }
1619 return GpuSamplerDesc {};
1620 }
1621
GetSamplerDescriptor(const RenderHandleReference & handle) const1622 GpuSamplerDesc GpuResourceManager::GetSamplerDescriptor(const RenderHandleReference& handle) const
1623 {
1624 return GetSamplerDescriptor(handle.GetHandle());
1625 }
1626
GetAccelerationStructureDescriptor(const RenderHandle & handle) const1627 GpuAccelerationStructureDesc GpuResourceManager::GetAccelerationStructureDescriptor(const RenderHandle& handle) const
1628 {
1629 if (!IsGpuAccelerationStructure(handle)) {
1630 return GpuAccelerationStructureDesc {};
1631 }
1632 {
1633 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
1634 const PerManagerStore& store = bufferStore_;
1635 auto const clientLock = std::shared_lock(store.clientMutex);
1636 if (arrayIndex < (uint32_t)store.descriptions.size()) {
1637 return store.descriptions[arrayIndex].combinedBufDescriptor;
1638 }
1639 }
1640 return GpuAccelerationStructureDesc {};
1641 }
1642
GetAccelerationStructureDescriptor(const RenderHandleReference & handle) const1643 GpuAccelerationStructureDesc GpuResourceManager::GetAccelerationStructureDescriptor(
1644 const RenderHandleReference& handle) const
1645 {
1646 return GetAccelerationStructureDescriptor(handle.GetHandle());
1647 }
1648
GetName(const RenderHandle & handle) const1649 string GpuResourceManager::GetName(const RenderHandle& handle) const
1650 {
1651 if (RenderHandleUtil::GetHasNamePart(handle) != 0) {
1652 const PerManagerStore* store = nullptr;
1653 const RenderHandleType handleType = RenderHandleUtil::GetHandleType(handle);
1654 if (handleType == RenderHandleType::GPU_BUFFER) {
1655 store = &bufferStore_;
1656 } else if (handleType == RenderHandleType::GPU_IMAGE) {
1657 store = &imageStore_;
1658 } else if (handleType == RenderHandleType::GPU_SAMPLER) {
1659 store = &samplerStore_;
1660 }
1661 if (store) {
1662 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
1663 for (const auto& iter : store->nameToClientIndex) {
1664 if (arrayIndex == iter.second) {
1665 return iter.first;
1666 }
1667 }
1668 }
1669 }
1670 return {};
1671 }
1672
GetName(const RenderHandleReference & handle) const1673 string GpuResourceManager::GetName(const RenderHandleReference& handle) const
1674 {
1675 return GetName(handle.GetHandle());
1676 }
1677
GetHandles(const PerManagerStore & store)1678 vector<RenderHandleReference> GpuResourceManager::GetHandles(const PerManagerStore& store)
1679 {
1680 const auto lock = std::shared_lock(store.clientMutex);
1681
1682 vector<RenderHandleReference> res;
1683 res.reserve(store.clientHandles.size());
1684 for (const auto& ref : store.clientHandles) {
1685 if (ref) {
1686 res.push_back(ref);
1687 }
1688 }
1689 return res;
1690 }
1691
GetRawHandles(const PerManagerStore & store)1692 vector<RenderHandle> GpuResourceManager::GetRawHandles(const PerManagerStore& store)
1693 {
1694 const auto lock = std::shared_lock(store.clientMutex);
1695
1696 vector<RenderHandle> res;
1697 res.reserve(store.clientHandles.size());
1698 for (const auto& ref : store.clientHandles) {
1699 if (ref) {
1700 res.push_back(ref.GetHandle());
1701 }
1702 }
1703 return res;
1704 }
1705
GetBufferHandles() const1706 vector<RenderHandleReference> GpuResourceManager::GetBufferHandles() const
1707 {
1708 return GetHandles(bufferStore_);
1709 }
1710
GetImageHandles() const1711 vector<RenderHandleReference> GpuResourceManager::GetImageHandles() const
1712 {
1713 return GetHandles(imageStore_);
1714 }
1715
GetSamplerHandles() const1716 vector<RenderHandleReference> GpuResourceManager::GetSamplerHandles() const
1717 {
1718 return GetHandles(samplerStore_);
1719 }
1720
GetRawBufferHandles() const1721 vector<RenderHandle> GpuResourceManager::GetRawBufferHandles() const
1722 {
1723 return GetRawHandles(bufferStore_);
1724 }
1725
GetRawImageHandles() const1726 vector<RenderHandle> GpuResourceManager::GetRawImageHandles() const
1727 {
1728 return GetRawHandles(imageStore_);
1729 }
1730
GetRawSamplerHandles() const1731 vector<RenderHandle> GpuResourceManager::GetRawSamplerHandles() const
1732 {
1733 return GetRawHandles(samplerStore_);
1734 }
1735
SetDefaultGpuBufferCreationFlags(const BufferUsageFlags usageFlags)1736 void GpuResourceManager::SetDefaultGpuBufferCreationFlags(const BufferUsageFlags usageFlags)
1737 {
1738 defaultBufferUsageFlags_ = usageFlags;
1739 }
1740
SetDefaultGpuImageCreationFlags(const ImageUsageFlags usageFlags)1741 void GpuResourceManager::SetDefaultGpuImageCreationFlags(const ImageUsageFlags usageFlags)
1742 {
1743 defaultImageUsageFlags_ = usageFlags;
1744 }
1745
CreateGpuResource(const OperationDescription & op,const uint32_t arrayIndex,const RenderHandleType resourceType,const uintptr_t preCreatedResVec)1746 void GpuResourceManager::CreateGpuResource(const OperationDescription& op, const uint32_t arrayIndex,
1747 const RenderHandleType resourceType, const uintptr_t preCreatedResVec)
1748 {
1749 if (resourceType == RenderHandleType::GPU_BUFFER) {
1750 PLUGIN_ASSERT(preCreatedResVec);
1751 if (RenderHandleUtil::IsGpuAccelerationStructure(op.handle)) {
1752 PLUGIN_ASSERT(op.optionalResourceIndex == ~0u);
1753 gpuBufferMgr_->Create<GpuAccelerationStructureDesc>(arrayIndex,
1754 op.descriptor.combinedBufDescriptor.bufferDesc, {}, true, op.descriptor.combinedBufDescriptor);
1755 } else {
1756 if (op.optionalResourceIndex != ~0u) {
1757 BufferVector& res = *(reinterpret_cast<BufferVector*>(reinterpret_cast<void*>(preCreatedResVec)));
1758 PLUGIN_ASSERT(op.optionalResourceIndex < res.size());
1759 gpuBufferMgr_->Create<GpuAccelerationStructureDesc>(arrayIndex,
1760 op.descriptor.combinedBufDescriptor.bufferDesc, move(res[op.optionalResourceIndex]), false,
1761 op.descriptor.combinedBufDescriptor);
1762 } else {
1763 gpuBufferMgr_->Create<GpuAccelerationStructureDesc>(arrayIndex,
1764 op.descriptor.combinedBufDescriptor.bufferDesc, {}, false, op.descriptor.combinedBufDescriptor);
1765 }
1766 }
1767 } else if (resourceType == RenderHandleType::GPU_IMAGE) {
1768 PLUGIN_ASSERT(preCreatedResVec);
1769 if (op.optionalResourceIndex != ~0u) {
1770 ImageVector& images = *(reinterpret_cast<ImageVector*>(reinterpret_cast<void*>(preCreatedResVec)));
1771 PLUGIN_ASSERT(op.optionalResourceIndex < images.size());
1772 gpuImageMgr_->Create<uint32_t>(
1773 arrayIndex, op.descriptor.imageDescriptor, move(images[op.optionalResourceIndex]), false, 0);
1774 } else {
1775 gpuImageMgr_->Create<uint32_t>(arrayIndex, op.descriptor.imageDescriptor, {}, false, 0);
1776 }
1777 } else if (resourceType == RenderHandleType::GPU_SAMPLER) {
1778 PLUGIN_ASSERT(preCreatedResVec == 0);
1779 PLUGIN_ASSERT(op.optionalResourceIndex == ~0u);
1780 gpuSamplerMgr_->Create<uint32_t>(arrayIndex, op.descriptor.samplerDescriptor, {}, false, 0);
1781 }
1782 }
1783
1784 // needs to be locked when called, and call only for valid gpu handles
DestroyGpuResource(const OperationDescription & operation,const uint32_t arrayIndex,const RenderHandleType resourceType,PerManagerStore & store)1785 void GpuResourceManager::DestroyGpuResource(const OperationDescription& operation, const uint32_t arrayIndex,
1786 const RenderHandleType resourceType, PerManagerStore& store)
1787 {
1788 store.mgr->Destroy(arrayIndex);
1789 PLUGIN_ASSERT(arrayIndex < static_cast<uint32_t>(store.gpuHandles.size()));
1790 store.clientHandles[arrayIndex] = {};
1791 store.additionalData[arrayIndex] = {};
1792 store.gpuHandles[arrayIndex] = InvalidateWithGeneration(store.gpuHandles[arrayIndex]);
1793 }
1794
HandlePendingRemappings(const vector<RemapDescription> & pendingShallowRemappings,vector<EngineResourceHandle> & gpuHandles)1795 void GpuResourceManager::HandlePendingRemappings(
1796 const vector<RemapDescription>& pendingShallowRemappings, vector<EngineResourceHandle>& gpuHandles)
1797 {
1798 for (auto const& shallowRemap : pendingShallowRemappings) {
1799 // find the gpu handle where the client handle wants to point to
1800 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(shallowRemap.resourceClientHandle);
1801 if (arrayIndex < gpuHandles.size()) {
1802 const EngineResourceHandle gpuHandle = gpuHandles[arrayIndex];
1803
1804 const bool validGpuHandle = RenderHandleUtil::IsValid(gpuHandle);
1805 const uint32_t remapArrayIndex = RenderHandleUtil::GetIndexPart(shallowRemap.shallowClientHandle);
1806 if (validGpuHandle && (remapArrayIndex < (uint32_t)gpuHandles.size())) {
1807 gpuHandles[remapArrayIndex] = gpuHandle;
1808 } else {
1809 PLUGIN_LOG_E("gpuimage handle remapping failed; client handle not found");
1810 }
1811 }
1812 }
1813 }
1814
HandlePendingAllocationsImpl(const bool isFrameEnd,const bool allowDestruction)1815 void GpuResourceManager::HandlePendingAllocationsImpl(const bool isFrameEnd, const bool allowDestruction)
1816 {
1817 HandleStorePendingAllocations(isFrameEnd, allowDestruction, bufferStore_);
1818 HandleStorePendingAllocations(isFrameEnd, allowDestruction, imageStore_);
1819 HandleStorePendingAllocations(isFrameEnd, allowDestruction, samplerStore_);
1820
1821 #if (RENDER_VULKAN_VALIDATION_ENABLED == 1)
1822 ProcessDebugTags();
1823 #endif
1824 }
1825
HandleRemoved(uint32_t arrayIndex,const OperationDescription & operation)1826 void GpuResourceManager::PerManagerStore::HandleRemoved(uint32_t arrayIndex, const OperationDescription& operation)
1827 {
1828 PLUGIN_ASSERT(arrayIndex < static_cast<uint32_t>(clientHandles.size()));
1829 availableHandleIds.push_back(operation.handle.id);
1830 clientHandles[arrayIndex] = {};
1831 additionalData[arrayIndex] = {};
1832 }
1833
HandleAlloc(uint32_t arrayIndex,const OperationDescription & operation)1834 void GpuResourceManager::PerManagerStore::HandleAlloc(uint32_t arrayIndex, const OperationDescription& operation)
1835 {
1836 PLUGIN_ASSERT(arrayIndex < static_cast<uint32_t>(gpuHandles.size()));
1837 // NOTE: this is essential to get correct, this maps render pass etc. hashing
1838 // if the generation counter is old there might vulkan image layout issues etc.
1839 const EngineResourceHandle gpuHandle = UnpackNewHandle(gpuHandles[arrayIndex], handleType, arrayIndex);
1840 #if (RENDER_DEBUG_GPU_RESOURCE_IDS == 1)
1841 LogGpuResource(operation.handle, gpuHandle);
1842 #endif
1843 gpuHandles[arrayIndex] = gpuHandle;
1844 #if (RENDER_VULKAN_VALIDATION_ENABLED == 1)
1845 debugTagAllocations.push_back(operation.handle);
1846 #endif
1847 }
1848
HandleDealloc(uint32_t arrayIndex,const OperationDescription & operation,const bool isFrameEnd,const bool allowDestruction)1849 bool GpuResourceManager::PerManagerStore::HandleDealloc(
1850 uint32_t arrayIndex, const OperationDescription& operation, const bool isFrameEnd, const bool allowDestruction)
1851 {
1852 if ((!allowDestruction) || (RenderHandleUtil::IsDeferredDestroy(operation.handle) && (!isFrameEnd))) {
1853 // push deferred destroys back to wait for the end of the frame destruction
1854 pendingData.allocations.push_back(operation);
1855 return false;
1856 }
1857 availableHandleIds.push_back(operation.handle.id);
1858 if (RenderHandleUtil::IsShallowResource(operation.handle)) {
1859 PLUGIN_ASSERT(arrayIndex < static_cast<uint32_t>(clientHandles.size()));
1860 PLUGIN_ASSERT(arrayIndex < static_cast<uint32_t>(gpuHandles.size()));
1861 // shallow handles do not destroy the actual GPU resource
1862 // there's always the real handle which handles the destruction based on references
1863 clientHandles[arrayIndex] = {};
1864 additionalData[arrayIndex] = {};
1865 // shallow handle invalidated with default (the generation is not correct anyhow)
1866 gpuHandles[arrayIndex] = {};
1867 return false;
1868 }
1869 return true;
1870 }
1871
HandleStorePendingAllocations(const bool isFrameEnd,const bool allowDestruction,PerManagerStore & store)1872 void GpuResourceManager::HandleStorePendingAllocations(
1873 const bool isFrameEnd, const bool allowDestruction, PerManagerStore& store)
1874 {
1875 store.clientMutex.lock();
1876 // protect GPU memory allocations
1877 allocationMutex_.lock();
1878 // check for handle destructions
1879 for (const auto& handleRef : store.clientHandles) {
1880 if (handleRef && (handleRef.GetRefCount() <= 1)) {
1881 Destroy(store, handleRef.GetHandle());
1882 }
1883 }
1884
1885 const auto pendingAllocations = move(store.pendingData.allocations);
1886 auto pendingBuffers = move(store.pendingData.buffers);
1887 auto pendingImages = move(store.pendingData.images);
1888 const auto pendingRemaps = move(store.pendingData.remaps);
1889 uintptr_t pendingRes = 0; // ptr to pending resource vector
1890 if (store.handleType == RenderHandleType::GPU_BUFFER) {
1891 pendingRes = reinterpret_cast<uintptr_t>(static_cast<void*>(&pendingBuffers));
1892 } else if (store.handleType == RenderHandleType::GPU_IMAGE) {
1893 pendingRes = reinterpret_cast<uintptr_t>(static_cast<void*>(&pendingImages));
1894 }
1895
1896 // increase the gpu handle vector sizes if needed
1897 if (store.clientHandles.size() > store.gpuHandles.size()) {
1898 store.gpuHandles.resize(store.clientHandles.size());
1899 store.mgr->Resize(store.clientHandles.size());
1900 }
1901
1902 for (auto const& allocation : pendingAllocations) {
1903 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(allocation.handle);
1904 // NOTE: needs to be cleared here
1905 store.additionalData[arrayIndex].indexToPendingData = ~0u;
1906 // NOTE: immediately created resource replacing should be prevented (ptr might be present still)
1907 if (allocation.allocType == AllocType::REMOVED) {
1908 store.HandleRemoved(arrayIndex, allocation);
1909 continue;
1910 }
1911
1912 // first allocation, then dealloc, with dealloc we need to check for deferred destruction
1913 if (allocation.allocType == AllocType::ALLOC) {
1914 store.HandleAlloc(arrayIndex, allocation);
1915 CreateGpuResource(allocation, arrayIndex, store.handleType, pendingRes);
1916 } else if (allocation.allocType == AllocType::DEALLOC) {
1917 if (store.HandleDealloc(arrayIndex, allocation, isFrameEnd, allowDestruction)) {
1918 DestroyGpuResource(allocation, arrayIndex, store.handleType, store);
1919 }
1920 }
1921 // there might be undefined type, e.g. for shallow handles
1922
1923 // render graph frame state reset for trackable and not auto reset frame states
1924 // with alloc there might be a replace which needs this as well as with dealloc
1925 if (RenderHandleUtil::IsDynamicResource(allocation.handle) &&
1926 (!RenderHandleUtil::IsResetOnFrameBorders(allocation.handle))) {
1927 PLUGIN_ASSERT((store.handleType == RenderHandleType::GPU_BUFFER) ||
1928 (store.handleType == RenderHandleType::GPU_IMAGE));
1929 clientHandleStateDestroy_.resources.push_back(allocation.handle);
1930 }
1931 }
1932
1933 // inside mutex (calls device)
1934 store.mgr->HandlePendingDeallocations();
1935
1936 allocationMutex_.unlock();
1937 // although the pendingData was moved and doesn't need locks anymore, other parts of the store are modified
1938 // as well so we need to hold the lock until here.
1939 store.clientMutex.unlock();
1940
1941 if (store.handleType == RenderHandleType::GPU_IMAGE) {
1942 // no lock needed for gpuHandles access
1943 HandlePendingRemappings(pendingRemaps, store.gpuHandles);
1944 }
1945 }
1946
HandlePendingAllocations(const bool allowDestruction)1947 void GpuResourceManager::HandlePendingAllocations(const bool allowDestruction)
1948 {
1949 HandlePendingAllocationsImpl(false, allowDestruction);
1950 }
1951
EndFrame()1952 void GpuResourceManager::EndFrame()
1953 {
1954 DestroyFrameStaging();
1955 // end frame, allow destruction
1956 HandlePendingAllocationsImpl(true, true);
1957 }
1958
RenderBackendImmediateRemapGpuImageHandle(const RenderHandle & clientHandle,const RenderHandle & clientHandleGpuResource)1959 void GpuResourceManager::RenderBackendImmediateRemapGpuImageHandle(
1960 const RenderHandle& clientHandle, const RenderHandle& clientHandleGpuResource)
1961 {
1962 const uint32_t clientArrayIndex = RenderHandleUtil::GetIndexPart(clientHandle);
1963 const uint32_t clientResourceArrayIndex = RenderHandleUtil::GetIndexPart(clientHandleGpuResource);
1964 const bool areGpuImages = (RenderHandleUtil::GetHandleType(clientHandle) == RenderHandleType::GPU_IMAGE) &&
1965 (RenderHandleUtil::GetHandleType(clientHandleGpuResource) == RenderHandleType::GPU_IMAGE);
1966 if (areGpuImages) {
1967 PerManagerStore& store = imageStore_;
1968 auto const clientLock = std::lock_guard(store.clientMutex);
1969
1970 if ((clientArrayIndex < store.gpuHandles.size()) && (clientResourceArrayIndex < store.gpuHandles.size())) {
1971 // NOTE: old owned gpu resource should be destroyed
1972 PLUGIN_ASSERT(RenderHandleUtil::IsValid(store.gpuHandles[clientResourceArrayIndex]));
1973
1974 store.gpuHandles[clientArrayIndex] = store.gpuHandles[clientResourceArrayIndex];
1975 store.descriptions[clientArrayIndex] = store.descriptions[clientResourceArrayIndex];
1976 } else {
1977 PLUGIN_LOG_E("invalid backend gpu image remapping indices");
1978 }
1979 } else {
1980 PLUGIN_LOG_E("invalid backend gpu image remapping handles");
1981 }
1982 }
1983
LockFrameStagingData()1984 void GpuResourceManager::LockFrameStagingData()
1985 {
1986 {
1987 std::lock_guard lock(stagingMutex_);
1988 perFrameStagingData_ = move(stagingOperations_);
1989 stagingOperations_ = {};
1990 }
1991
1992 // create frame staging buffers and set handles for staging
1993 {
1994 std::lock_guard lock(bufferStore_.clientMutex);
1995
1996 // handle render time reserved gpu buffer
1997 {
1998 auto& rtrgb = renderTimeReservedGpuBuffer_;
1999 if (rtrgb.neededByteSize > 0U) {
2000 if ((!rtrgb.handle) || (rtrgb.storedByteSize < rtrgb.neededByteSize)) {
2001 // over-allocate
2002 rtrgb.storedByteSize = rtrgb.neededByteSize + Align(rtrgb.neededByteSize / 8U, BUFFER_ALIGNMENT);
2003 // re-create
2004 rtrgb.handle = CreateBuffer({}, rtrgb.baseHandle, GetMapBufferDesc(rtrgb.storedByteSize)).handle;
2005 rtrgb.baseHandle = rtrgb.handle.GetHandle();
2006 }
2007 rtrgb.neededByteSize = 0U;
2008 rtrgb.executeValues.swap(rtrgb.values);
2009 rtrgb.mappedData = nullptr;
2010 } else {
2011 // clear always the resource if no usage
2012 rtrgb = {};
2013 }
2014 }
2015
2016 if (perFrameStagingData_.stagingByteSize > 0U) {
2017 perFrameStagingBuffers_.push_back(
2018 CreateStagingBuffer(GetStagingBufferDesc(perFrameStagingData_.stagingByteSize)));
2019 perFrameStagingData_.stagingBuffer = perFrameStagingBuffers_.back().GetHandle();
2020 }
2021
2022 perFrameStagingBuffers_.reserve(
2023 perFrameStagingData_.bufferToBuffer.size() + perFrameStagingData_.bufferToImage.size());
2024 for (auto& ref : perFrameStagingData_.bufferToBuffer) {
2025 if ((!ref.invalidOperation) && (ref.stagingBufferByteSize > 0)) {
2026 ref.srcHandle = perFrameStagingBuffers_.back();
2027 }
2028 }
2029 for (auto& ref : perFrameStagingData_.bufferToImage) {
2030 if ((!ref.invalidOperation) && (ref.stagingBufferByteSize > 0)) {
2031 ref.srcHandle = perFrameStagingBuffers_.back();
2032 }
2033 }
2034 }
2035 {
2036 auto const clientLock = std::lock_guard(imageStore_.clientMutex);
2037 // create image scaling targets and set handles
2038 perFrameStagingScalingImages_.resize(perFrameStagingData_.scalingImageData.scalingImages.size());
2039 for (size_t idx = 0; idx < perFrameStagingData_.scalingImageData.scalingImages.size(); ++idx) {
2040 auto& scalingImageRef = perFrameStagingData_.scalingImageData.scalingImages[idx];
2041 perFrameStagingScalingImages_[idx] = CreateImage({}, {},
2042 GetStagingScalingImageDesc(scalingImageRef.format, scalingImageRef.maxWidth, scalingImageRef.maxHeight))
2043 .handle;
2044 scalingImageRef.handle = perFrameStagingScalingImages_[idx];
2045 }
2046 }
2047 }
2048
DestroyFrameStaging()2049 void GpuResourceManager::DestroyFrameStaging()
2050 {
2051 // explicit destruction of staging resources
2052 {
2053 PerManagerStore& store = bufferStore_;
2054 auto const clientLock = std::lock_guard(store.clientMutex);
2055 for (const auto& handleRef : perFrameStagingBuffers_) {
2056 Destroy(store, handleRef.GetHandle());
2057 }
2058 perFrameStagingBuffers_.clear();
2059 }
2060 {
2061 PerManagerStore& store = imageStore_;
2062 auto const clientLock = std::lock_guard(store.clientMutex);
2063 for (const auto& handleRef : perFrameStagingScalingImages_) {
2064 Destroy(store, handleRef.GetHandle());
2065 }
2066 perFrameStagingScalingImages_.clear();
2067 }
2068 }
2069
HasStagingData() const2070 bool GpuResourceManager::HasStagingData() const
2071 {
2072 if (perFrameStagingData_.bufferToBuffer.empty() && perFrameStagingData_.bufferToImage.empty() &&
2073 perFrameStagingData_.imageToBuffer.empty() && perFrameStagingData_.imageToImage.empty() &&
2074 perFrameStagingData_.cpuToBuffer.empty() && perFrameStagingData_.bufferCopies.empty() &&
2075 perFrameStagingData_.bufferImageCopies.empty() && perFrameStagingData_.imageCopies.empty()) {
2076 return false;
2077 } else {
2078 return true;
2079 }
2080 }
2081
ConsumeStagingData()2082 StagingConsumeStruct GpuResourceManager::ConsumeStagingData()
2083 {
2084 StagingConsumeStruct staging = move(perFrameStagingData_);
2085 return staging;
2086 }
2087
ConsumeStateDestroyData()2088 GpuResourceManager::StateDestroyConsumeStruct GpuResourceManager::ConsumeStateDestroyData()
2089 {
2090 StateDestroyConsumeStruct srcs = move(clientHandleStateDestroy_);
2091 return srcs;
2092 }
2093
MapBuffer(const RenderHandle & handle) const2094 void* GpuResourceManager::MapBuffer(const RenderHandle& handle) const
2095 {
2096 if (GpuBuffer* buffer = GetBuffer(handle); buffer) {
2097 return buffer->Map();
2098 }
2099 return nullptr;
2100 }
2101
MapBuffer(const RenderHandleReference & handle) const2102 void* GpuResourceManager::MapBuffer(const RenderHandleReference& handle) const
2103 {
2104 return MapBuffer(handle.GetHandle());
2105 }
2106
MapBufferMemory(const RenderHandle & handle) const2107 void* GpuResourceManager::MapBufferMemory(const RenderHandle& handle) const
2108 {
2109 const bool isOutsideRendererMappable = RenderHandleUtil::IsMappableOutsideRenderer(handle);
2110 void* data = nullptr;
2111 if (isOutsideRendererMappable) {
2112 const bool isCreatedImmediate = RenderHandleUtil::IsImmediatelyCreated(handle);
2113 #if (RENDER_VALIDATION_ENABLED == 1)
2114 if (!isCreatedImmediate) {
2115 const auto tmpString = "map_buffer_memory" + to_string(handle.id);
2116 PLUGIN_LOG_ONCE_W(tmpString,
2117 "RENDER_VALIDATION: cannot map buffer memory (missing CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE?)");
2118 }
2119 #endif
2120 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
2121 // with gl MapMemory may require context to be active. Activate grabs a mutex which must be before clientMutex.
2122 const bool isOpenGl = (device_.GetBackendType() != DeviceBackendType::VULKAN);
2123 if (isOpenGl) {
2124 device_.Activate();
2125 }
2126 const auto clientLock = std::lock_guard(bufferStore_.clientMutex);
2127 if (isCreatedImmediate && (arrayIndex < static_cast<uint32_t>(bufferStore_.clientHandles.size()))) {
2128 #if (RENDER_VALIDATION_ENABLED == 1)
2129 if (!bufferStore_.additionalData[arrayIndex].resourcePtr) {
2130 PLUGIN_LOG_E("RENDER_VALIDATION: invalid pointer with mappable GPU buffer MapBufferMemory");
2131 }
2132 #endif
2133 if (bufferStore_.additionalData[arrayIndex].resourcePtr) {
2134 data = (reinterpret_cast<GpuBuffer*>(bufferStore_.additionalData[arrayIndex].resourcePtr))->MapMemory();
2135 }
2136 }
2137 if (isOpenGl) {
2138 device_.Deactivate();
2139 }
2140 } else if (GpuBuffer* buffer = GetBuffer(handle); buffer) {
2141 data = buffer->MapMemory();
2142 }
2143 return data;
2144 }
2145
MapBufferMemory(const RenderHandleReference & handle) const2146 void* GpuResourceManager::MapBufferMemory(const RenderHandleReference& handle) const
2147 {
2148 return MapBufferMemory(handle.GetHandle());
2149 }
2150
UnmapBuffer(const RenderHandle & handle) const2151 void GpuResourceManager::UnmapBuffer(const RenderHandle& handle) const
2152 {
2153 const bool isOutsideRendererMappable = RenderHandleUtil::IsMappableOutsideRenderer(handle);
2154 if (isOutsideRendererMappable) {
2155 const bool isCreatedImmediate = RenderHandleUtil::IsImmediatelyCreated(handle);
2156 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
2157 // with gl Unmap may require context to be active. Activate grabs a mutex which must be before clientMutex.
2158 const bool isOpenGl = (device_.GetBackendType() != DeviceBackendType::VULKAN);
2159 if (isOpenGl) {
2160 device_.Activate();
2161 }
2162 const auto clientLock = std::lock_guard(bufferStore_.clientMutex);
2163 if (isCreatedImmediate && (arrayIndex < static_cast<uint32_t>(bufferStore_.clientHandles.size()))) {
2164 #if (RENDER_VALIDATION_ENABLED == 1)
2165 if (!bufferStore_.additionalData[arrayIndex].resourcePtr) {
2166 PLUGIN_LOG_E("RENDER_VALIDATION: invalid pointer with mappable GPU buffer UnmapBuffer");
2167 }
2168 #endif
2169 if (bufferStore_.additionalData[arrayIndex].resourcePtr) {
2170 (reinterpret_cast<GpuBuffer*>(bufferStore_.additionalData[arrayIndex].resourcePtr))->Unmap();
2171 }
2172 }
2173 if (isOpenGl) {
2174 device_.Deactivate();
2175 }
2176 } else if (const GpuBuffer* buffer = GetBuffer(handle); buffer) {
2177 buffer->Unmap();
2178 }
2179 }
2180
UnmapBuffer(const RenderHandleReference & handle) const2181 void GpuResourceManager::UnmapBuffer(const RenderHandleReference& handle) const
2182 {
2183 return UnmapBuffer(handle.GetHandle());
2184 }
2185
2186 // must be locked when called
CommitPendingData(PerManagerStore & store)2187 GpuResourceManager::PendingData GpuResourceManager::CommitPendingData(PerManagerStore& store)
2188 {
2189 return { move(store.pendingData.allocations), move(store.pendingData.buffers), move(store.pendingData.images),
2190 move(store.pendingData.remaps) };
2191 }
2192
DebugPrintValidCounts()2193 void GpuResourceManager::DebugPrintValidCounts()
2194 {
2195 #if (RENDER_VALIDATION_ENABLED == 1)
2196 PLUGIN_LOG_D("GPU buffer count: %u", static_cast<uint32_t>(gpuBufferMgr_->GetValidResourceCount()));
2197 PLUGIN_LOG_D("GPU image count: %u", static_cast<uint32_t>(gpuImageMgr_->GetValidResourceCount()));
2198 PLUGIN_LOG_D("GPU sampler count: %u", static_cast<uint32_t>(gpuSamplerMgr_->GetValidResourceCount()));
2199 #endif
2200 }
2201
WaitForIdleAndDestroyGpuResources()2202 void GpuResourceManager::WaitForIdleAndDestroyGpuResources()
2203 {
2204 PLUGIN_LOG_D("WFIADGR thread id: %" PRIu64, (uint64_t)std::hash<std::thread::id> {}(std::this_thread::get_id()));
2205 device_.Activate();
2206 device_.WaitForIdle();
2207
2208 // 1. immediate Destroy all the handles that are to be destroyed
2209 // 2. throw away everything else that's pending
2210 auto DestroyPendingData = [this](PerManagerStore& store) {
2211 const auto lock = std::lock_guard(store.clientMutex);
2212 const auto allocLock = std::lock_guard(allocationMutex_);
2213
2214 auto pd = CommitPendingData(store);
2215 if (store.clientHandles.size() > store.gpuHandles.size()) {
2216 store.gpuHandles.resize(store.clientHandles.size());
2217 store.mgr->Resize(store.clientHandles.size());
2218 }
2219
2220 #if (RENDER_VALIDATION_ENABLED == 1)
2221 uint32_t dc = 0;
2222 #endif
2223 for (const auto& ref : pd.allocations) {
2224 if (ref.allocType == AllocType::DEALLOC) {
2225 store.availableHandleIds.push_back(ref.handle.id);
2226 DestroyImmediate(store, ref.handle);
2227 // render graph frame state reset for trackable and not auto reset frame states
2228 if (RenderHandleUtil::IsDynamicResource(ref.handle) &&
2229 (!RenderHandleUtil::IsResetOnFrameBorders(ref.handle))) {
2230 PLUGIN_ASSERT((store.handleType == RenderHandleType::GPU_BUFFER) ||
2231 (store.handleType == RenderHandleType::GPU_IMAGE));
2232 clientHandleStateDestroy_.resources.push_back(ref.handle);
2233 }
2234 #if (RENDER_VALIDATION_ENABLED == 1)
2235 dc++;
2236 } else if (ref.allocType == AllocType::REMOVED) {
2237 dc++;
2238 #endif
2239 }
2240 }
2241 #if (RENDER_VALIDATION_ENABLED == 1)
2242 PLUGIN_LOG_D("WFIADGR: d: %u r (type:%u)", dc, uint32_t(store.handleType));
2243 PLUGIN_LOG_D("WFIADGR: pa cl: %u (t:%u)", (uint32_t)pd.allocations.size() - dc, uint32_t(store.handleType));
2244 #endif
2245
2246 // inside mutex (calls device)
2247 store.mgr->HandlePendingDeallocationsImmediate();
2248 };
2249
2250 DestroyPendingData(bufferStore_);
2251 DestroyPendingData(imageStore_);
2252 DestroyPendingData(samplerStore_);
2253
2254 // make sure that all staging resources are forcefully destroyed
2255 LockFrameStagingData();
2256 ConsumeStagingData(); // consume cpu data
2257 DestroyFrameStaging();
2258
2259 {
2260 // additional possible staging buffer clean-up
2261 auto& store = bufferStore_;
2262 const auto lock = std::lock_guard(store.clientMutex);
2263 const auto allocLock = std::lock_guard(allocationMutex_);
2264 store.mgr->HandlePendingDeallocationsImmediate();
2265 }
2266
2267 DebugPrintValidCounts();
2268
2269 device_.Deactivate();
2270 }
2271
GetGpuHandle(const PerManagerStore & store,const RenderHandle & clientHandle)2272 EngineResourceHandle GpuResourceManager::GetGpuHandle(const PerManagerStore& store, const RenderHandle& clientHandle)
2273 {
2274 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(clientHandle);
2275 if (arrayIndex < (uint32_t)store.gpuHandles.size()) {
2276 // NOTE: separate GPU handle generation counter might not
2277 // be exact when using shallow handles (i.e. counter is zero always after shallow handles)
2278 #if (RENDER_VALIDATION_ENABLED == 1)
2279 if (!RenderHandleUtil::IsValid(store.gpuHandles[arrayIndex])) {
2280 PLUGIN_LOG_E("RENDER_VALIDATION : invalid gpu handle %" PRIx64, clientHandle.id);
2281 }
2282 #endif
2283 return store.gpuHandles[arrayIndex];
2284 } else {
2285 const auto tmpName = to_string(clientHandle.id) + "GpuResourceManager::GetGpuHandle";
2286 PLUGIN_LOG_ONCE_E(tmpName.c_str(), "No gpu resource handle for client handle : %" PRIx64, clientHandle.id);
2287 return EngineResourceHandle {};
2288 }
2289 }
2290
GetGpuHandle(const RenderHandle & clientHandle) const2291 EngineResourceHandle GpuResourceManager::GetGpuHandle(const RenderHandle& clientHandle) const
2292 {
2293 const RenderHandleType handleType = RenderHandleUtil::GetHandleType(clientHandle);
2294 if (handleType == RenderHandleType::GPU_BUFFER) {
2295 return GetGpuHandle(bufferStore_, clientHandle);
2296 } else if (handleType == RenderHandleType::GPU_IMAGE) {
2297 return GetGpuHandle(imageStore_, clientHandle);
2298 } else if (handleType == RenderHandleType::GPU_SAMPLER) {
2299 return GetGpuHandle(samplerStore_, clientHandle);
2300 } else {
2301 return {};
2302 }
2303 }
2304
GetBuffer(const RenderHandle & handle) const2305 GpuBuffer* GpuResourceManager::GetBuffer(const RenderHandle& handle) const
2306 {
2307 const EngineResourceHandle resHandle = GetGpuHandle(bufferStore_, handle);
2308 return gpuBufferMgr_->Get(RenderHandleUtil::GetIndexPart(resHandle));
2309 }
2310
GetImage(const RenderHandle & handle) const2311 GpuImage* GpuResourceManager::GetImage(const RenderHandle& handle) const
2312 {
2313 const EngineResourceHandle resHandle = GetGpuHandle(imageStore_, handle);
2314 return gpuImageMgr_->Get(RenderHandleUtil::GetIndexPart(resHandle));
2315 }
2316
GetSampler(const RenderHandle & handle) const2317 GpuSampler* GpuResourceManager::GetSampler(const RenderHandle& handle) const
2318 {
2319 const EngineResourceHandle resHandle = GetGpuHandle(samplerStore_, handle);
2320 return gpuSamplerMgr_->Get(RenderHandleUtil::GetIndexPart(resHandle));
2321 }
2322
GetBufferHandleCount() const2323 uint32_t GpuResourceManager::GetBufferHandleCount() const
2324 {
2325 return static_cast<uint32_t>(bufferStore_.gpuHandles.size());
2326 }
2327
GetImageHandleCount() const2328 uint32_t GpuResourceManager::GetImageHandleCount() const
2329 {
2330 return static_cast<uint32_t>(imageStore_.gpuHandles.size());
2331 }
2332
CreateClientHandle(const RenderHandleType type,const ResourceDescriptor & resourceDescriptor,const uint64_t handleId,const uint32_t hasNameId,const uint32_t additionalInfoFlags)2333 RenderHandle GpuResourceManager::CreateClientHandle(const RenderHandleType type,
2334 const ResourceDescriptor& resourceDescriptor, const uint64_t handleId, const uint32_t hasNameId,
2335 const uint32_t additionalInfoFlags)
2336 {
2337 RenderHandle handle;
2338
2339 const uint32_t index = RenderHandleUtil::GetIndexPart(handleId);
2340 const uint32_t generationIndex = RenderHandleUtil::GetGenerationIndexPart(handleId) + 1; // next gen
2341 if (type == RenderHandleType::GPU_BUFFER) {
2342 // NOTE: additional flags for GPU acceleration structure might be needed
2343 const auto& rd = resourceDescriptor.combinedBufDescriptor.bufferDesc;
2344 RenderHandleInfoFlags infoFlags = (rd.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_DYNAMIC_BARRIERS)
2345 ? CORE_RESOURCE_HANDLE_DYNAMIC_TRACK
2346 : 0u;
2347 infoFlags |= (rd.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_CREATE_IMMEDIATE)
2348 ? CORE_RESOURCE_HANDLE_IMMEDIATELY_CREATED
2349 : 0;
2350 infoFlags |= (rd.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_DEFERRED_DESTROY)
2351 ? CORE_RESOURCE_HANDLE_DEFERRED_DESTROY
2352 : 0;
2353 infoFlags |= (rd.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_MAP_OUTSIDE_RENDERER)
2354 ? CORE_RESOURCE_HANDLE_MAP_OUTSIDE_RENDERER
2355 : 0;
2356 // NOTE: currently tagged as shallow resource with dynamic ring buffer (offsets)
2357 infoFlags |= (rd.engineCreationFlags & CORE_ENGINE_BUFFER_CREATION_DYNAMIC_RING_BUFFER)
2358 ? CORE_RESOURCE_HANDLE_SHALLOW_RESOURCE
2359 : 0;
2360 infoFlags |= additionalInfoFlags;
2361 handle = RenderHandleUtil::CreateGpuResourceHandle(type, infoFlags, index, generationIndex, hasNameId);
2362 } else if (type == RenderHandleType::GPU_IMAGE) {
2363 const auto& rd = resourceDescriptor.imageDescriptor;
2364 RenderHandleInfoFlags infoFlags {};
2365 infoFlags |= (rd.engineCreationFlags & CORE_ENGINE_IMAGE_CREATION_DYNAMIC_BARRIERS)
2366 ? CORE_RESOURCE_HANDLE_DYNAMIC_TRACK
2367 : 0u;
2368 infoFlags |= (rd.engineCreationFlags & CORE_ENGINE_IMAGE_CREATION_DEFERRED_DESTROY)
2369 ? CORE_RESOURCE_HANDLE_DEFERRED_DESTROY
2370 : 0;
2371 infoFlags |= ((rd.engineCreationFlags & CORE_ENGINE_IMAGE_CREATION_DYNAMIC_BARRIERS) &&
2372 (resourceDescriptor.imageDescriptor.mipCount > 1U))
2373 ? CORE_RESOURCE_HANDLE_DYNAMIC_ADDITIONAL_STATE
2374 : 0u;
2375 // force transient attachments to be state reset on frame borders
2376 infoFlags |= ((rd.engineCreationFlags & CORE_ENGINE_IMAGE_CREATION_RESET_STATE_ON_FRAME_BORDERS) ||
2377 (rd.usageFlags & CORE_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT))
2378 ? CORE_RESOURCE_HANDLE_RESET_ON_FRAME_BORDERS
2379 : 0u;
2380 infoFlags |= GetAdditionalImageFlagsFromFormat(rd.format);
2381 infoFlags |= additionalInfoFlags;
2382 handle = RenderHandleUtil::CreateGpuResourceHandle(type, infoFlags, index, generationIndex, hasNameId);
2383 } else if (type == RenderHandleType::GPU_SAMPLER) {
2384 handle = RenderHandleUtil::CreateGpuResourceHandle(type, 0, index, generationIndex, hasNameId);
2385 }
2386 return handle;
2387 }
2388
2389 // needs to be locked when called
GetNextAvailableHandleId(PerManagerStore & store)2390 uint64_t GpuResourceManager::GetNextAvailableHandleId(PerManagerStore& store)
2391 {
2392 uint64_t handleId = INVALID_RESOURCE_HANDLE;
2393 auto& availableHandleIds = store.availableHandleIds;
2394 if (!availableHandleIds.empty()) {
2395 handleId = availableHandleIds.back();
2396 availableHandleIds.pop_back();
2397 } else {
2398 handleId = static_cast<uint32_t>(store.clientHandles.size())
2399 << RenderHandleUtil::RES_HANDLE_ID_SHIFT; // next idx
2400 }
2401 return handleId;
2402 }
2403
2404 // needs to be locked when called
2405 // staging cannot be locked when called
StoreAllocation(PerManagerStore & store,const StoreAllocationInfo & info)2406 GpuResourceManager::StoreAllocationData GpuResourceManager::StoreAllocation(
2407 PerManagerStore& store, const StoreAllocationInfo& info)
2408 {
2409 // NOTE: PerManagerStore gpu handles cannot be touched here
2410 StoreAllocationData data;
2411
2412 // there cannot be both (valid name and a valid replaced handle)
2413 const uint32_t replaceArrayIndex = RenderHandleUtil::GetIndexPart(info.replacedHandle);
2414 bool hasReplaceHandle = (replaceArrayIndex < (uint32_t)store.clientHandles.size());
2415 uint32_t hasNameId = (!info.name.empty()) ? 1u : 0u;
2416 if (hasReplaceHandle) {
2417 data.handle = store.clientHandles[replaceArrayIndex];
2418 // re-use the name if it was present
2419 hasNameId = RenderHandleUtil::GetHasNamePart(data.handle.GetHandle());
2420 // NOTE: should be documented better, and prevented
2421 PLUGIN_ASSERT_MSG(!RenderHandleUtil::IsDeferredDestroy(data.handle.GetHandle()),
2422 "deferred desctruction resources cannot be replaced");
2423 if (RenderHandleUtil::IsValid(data.handle.GetHandle())) {
2424 // valid handle and reference counter, re-use the same ref count object, CreateClientHandle increases gen
2425 PLUGIN_ASSERT(data.handle.GetCounter());
2426 data.handle =
2427 RenderHandleReference(CreateClientHandle(info.type, info.descriptor, data.handle.GetHandle().id,
2428 hasNameId, info.addHandleFlags),
2429 data.handle.GetCounter());
2430 } else {
2431 #if (RENDER_VALIDATION_ENABLED == 1)
2432 PLUGIN_LOG_E("RENDER_VALIDATION: invalid replaced handle given to GPU resource manager, creating new");
2433 #endif
2434 const uint64_t handleId = GetNextAvailableHandleId(store);
2435 data.handle = RenderHandleReference(
2436 CreateClientHandle(info.type, info.descriptor, handleId, hasNameId, info.addHandleFlags),
2437 IRenderReferenceCounter::Ptr(new RenderReferenceCounter()));
2438 hasReplaceHandle = false;
2439 if (hasNameId) {
2440 // NOTE: should remove old name if it was in use
2441 store.nameToClientIndex[info.name] = RenderHandleUtil::GetIndexPart(data.handle.GetHandle());
2442 }
2443 }
2444 } else if (hasNameId != 0u) {
2445 // in some cases the handle might be invalid even though it is found
2446 if (auto const iter = store.nameToClientIndex.find(info.name); iter != store.nameToClientIndex.cend()) {
2447 PLUGIN_ASSERT(iter->second < static_cast<uint32_t>(store.clientHandles.size()));
2448 data.handle = store.clientHandles[iter->second];
2449 PLUGIN_ASSERT_MSG(!RenderHandleUtil::IsDeferredDestroy(data.handle.GetHandle()),
2450 "deferred desctruction resources cannot be replaced");
2451 data.handle =
2452 RenderHandleReference(CreateClientHandle(info.type, info.descriptor, data.handle.GetHandle().id,
2453 hasNameId, info.addHandleFlags),
2454 data.handle.GetCounter());
2455 }
2456 if (!data.handle) {
2457 const uint64_t handleId = GetNextAvailableHandleId(store);
2458 data.handle = RenderHandleReference(
2459 CreateClientHandle(info.type, info.descriptor, handleId, hasNameId, info.addHandleFlags),
2460 IRenderReferenceCounter::Ptr(new RenderReferenceCounter()));
2461 store.nameToClientIndex[info.name] = RenderHandleUtil::GetIndexPart(data.handle.GetHandle());
2462 }
2463 } else {
2464 const uint64_t handleId = GetNextAvailableHandleId(store);
2465 data.handle = RenderHandleReference(
2466 CreateClientHandle(info.type, info.descriptor, handleId, hasNameId, info.addHandleFlags),
2467 IRenderReferenceCounter::Ptr(new RenderReferenceCounter()));
2468 }
2469
2470 PLUGIN_ASSERT(data.handle);
2471 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(data.handle.GetHandle());
2472 PLUGIN_ASSERT(store.clientHandles.size() == store.descriptions.size());
2473 PLUGIN_ASSERT(store.clientHandles.size() == store.additionalData.size());
2474 if (arrayIndex >= (uint32_t)store.clientHandles.size()) {
2475 store.clientHandles.push_back(data.handle);
2476 store.additionalData.push_back({});
2477 store.descriptions.push_back(info.descriptor);
2478 } else {
2479 store.clientHandles[arrayIndex] = data.handle;
2480 // store.additionalData[arrayIndex] cannot be cleared here (staging cleaned based on this)
2481 store.descriptions[arrayIndex] = info.descriptor;
2482 }
2483
2484 #if (RENDER_DEBUG_GPU_RESOURCE_IDS == 1)
2485 PLUGIN_LOG_E("gpu resource allocation %" PRIx64 " (name: %s)", data.handle.GetHandle().id, info.name.data());
2486 #endif
2487
2488 // store allocation for GPU allocation
2489
2490 // needs to find the allocation and replace
2491 if (hasReplaceHandle || (hasNameId != 0)) {
2492 if (const uint32_t pendingArrIndex = store.additionalData[arrayIndex].indexToPendingData;
2493 (pendingArrIndex != INVALID_PENDING_INDEX) && (pendingArrIndex < store.pendingData.allocations.size())) {
2494 data.allocationIndex = pendingArrIndex;
2495 }
2496 }
2497 if (data.allocationIndex == ~0u) {
2498 data.allocationIndex = store.pendingData.allocations.size();
2499 store.additionalData[arrayIndex].indexToPendingData = static_cast<uint32_t>(data.allocationIndex);
2500 store.pendingData.allocations.emplace_back(
2501 data.handle.GetHandle(), info.descriptor, info.allocType, info.optResourceIndex);
2502 } else { // quite rare case and slow path
2503 // replace this frame's allocation
2504 auto& allocOp = store.pendingData.allocations[data.allocationIndex];
2505 // invalid flag should be only needed and all the allocations would be done later
2506 // i.e. create staging buffers and fetch based on index in render node staging
2507 RemoveStagingOperations(allocOp);
2508 store.pendingData.allocations[data.allocationIndex] = { data.handle.GetHandle(), info.descriptor,
2509 info.allocType, info.optResourceIndex };
2510 }
2511
2512 return data;
2513 }
2514
IsGpuBuffer(const RenderHandleReference & handle) const2515 bool GpuResourceManager::IsGpuBuffer(const RenderHandleReference& handle) const
2516 {
2517 return IsGpuBuffer(handle.GetHandle());
2518 }
2519
IsGpuImage(const RenderHandleReference & handle) const2520 bool GpuResourceManager::IsGpuImage(const RenderHandleReference& handle) const
2521 {
2522 return IsGpuImage(handle.GetHandle());
2523 }
2524
IsGpuSampler(const RenderHandleReference & handle) const2525 bool GpuResourceManager::IsGpuSampler(const RenderHandleReference& handle) const
2526 {
2527 return IsGpuSampler(handle.GetHandle());
2528 }
2529
IsGpuAccelerationStructure(const RenderHandleReference & handle) const2530 bool GpuResourceManager::IsGpuAccelerationStructure(const RenderHandleReference& handle) const
2531 {
2532 return IsGpuAccelerationStructure(handle.GetHandle());
2533 }
2534
IsSwapchain(const RenderHandleReference & handle) const2535 bool GpuResourceManager::IsSwapchain(const RenderHandleReference& handle) const
2536 {
2537 return IsSwapchain(handle.GetHandle());
2538 }
2539
IsMappableOutsideRender(const RenderHandleReference & handle) const2540 bool GpuResourceManager::IsMappableOutsideRender(const RenderHandleReference& handle) const
2541 {
2542 return RenderHandleUtil::IsMappableOutsideRenderer(handle.GetHandle());
2543 }
2544
IsGpuBuffer(const RenderHandle & handle) const2545 bool GpuResourceManager::IsGpuBuffer(const RenderHandle& handle) const
2546 {
2547 return RenderHandleUtil::IsGpuBuffer(handle);
2548 }
2549
IsGpuImage(const RenderHandle & handle) const2550 bool GpuResourceManager::IsGpuImage(const RenderHandle& handle) const
2551 {
2552 return RenderHandleUtil::IsGpuImage(handle);
2553 }
2554
IsGpuSampler(const RenderHandle & handle) const2555 bool GpuResourceManager::IsGpuSampler(const RenderHandle& handle) const
2556 {
2557 return RenderHandleUtil::IsGpuSampler(handle);
2558 }
2559
IsGpuAccelerationStructure(const RenderHandle & handle) const2560 bool GpuResourceManager::IsGpuAccelerationStructure(const RenderHandle& handle) const
2561 {
2562 return RenderHandleUtil::IsGpuAccelerationStructure(handle);
2563 }
2564
IsSwapchain(const RenderHandle & handle) const2565 bool GpuResourceManager::IsSwapchain(const RenderHandle& handle) const
2566 {
2567 return RenderHandleUtil::IsSwapchain(handle);
2568 }
2569
IsValid(const RenderHandle & handle) const2570 bool GpuResourceManager::IsValid(const RenderHandle& handle) const
2571 {
2572 return RenderHandleUtil::IsValid(handle);
2573 }
2574
GetFormatProperties(const RenderHandle & handle) const2575 FormatProperties GpuResourceManager::GetFormatProperties(const RenderHandle& handle) const
2576 {
2577 const RenderHandleType type = RenderHandleUtil::GetHandleType(handle);
2578 Format format = Format::BASE_FORMAT_UNDEFINED;
2579 if (type == RenderHandleType::GPU_BUFFER) {
2580 const GpuBufferDesc desc = GetBufferDescriptor(handle);
2581 format = desc.format;
2582 } else if (type == RenderHandleType::GPU_IMAGE) {
2583 const GpuImageDesc desc = GetImageDescriptor(handle);
2584 format = desc.format;
2585 }
2586 return device_.GetFormatProperties(format);
2587 }
2588
GetFormatProperties(const RenderHandleReference & handle) const2589 FormatProperties GpuResourceManager::GetFormatProperties(const RenderHandleReference& handle) const
2590 {
2591 return GetFormatProperties(handle.GetHandle());
2592 }
2593
GetFormatProperties(const Format format) const2594 FormatProperties GpuResourceManager::GetFormatProperties(const Format format) const
2595 {
2596 return device_.GetFormatProperties(format);
2597 }
2598
GetImageAspectFlags(const RenderHandleReference & handle) const2599 ImageAspectFlags GpuResourceManager::GetImageAspectFlags(const RenderHandleReference& handle) const
2600 {
2601 return GetImageAspectFlags(handle.GetHandle());
2602 }
2603
GetImageAspectFlags(const RenderHandle & handle) const2604 ImageAspectFlags GpuResourceManager::GetImageAspectFlags(const RenderHandle& handle) const
2605 {
2606 const RenderHandleType type = RenderHandleUtil::GetHandleType(handle);
2607 Format format = Format::BASE_FORMAT_UNDEFINED;
2608 if (type == RenderHandleType::GPU_BUFFER) {
2609 const GpuBufferDesc desc = GetBufferDescriptor(handle);
2610 format = desc.format;
2611 } else if (type == RenderHandleType::GPU_IMAGE) {
2612 const GpuImageDesc desc = GetImageDescriptor(handle);
2613 format = desc.format;
2614 }
2615 return GetImageAspectFlags(format);
2616 }
2617
GetImageAspectFlags(const BASE_NS::Format format) const2618 ImageAspectFlags GpuResourceManager::GetImageAspectFlags(const BASE_NS::Format format) const
2619 {
2620 ImageAspectFlags flags {};
2621 const bool isDepthFormat = ((format == BASE_FORMAT_D16_UNORM) || (format == BASE_FORMAT_X8_D24_UNORM_PACK32) ||
2622 (format == BASE_FORMAT_D32_SFLOAT) || (format == BASE_FORMAT_D16_UNORM_S8_UINT) ||
2623 (format == BASE_FORMAT_D24_UNORM_S8_UINT));
2624 if (isDepthFormat) {
2625 flags |= ImageAspectFlagBits::CORE_IMAGE_ASPECT_DEPTH_BIT;
2626
2627 const bool isStencilFormat =
2628 ((format == BASE_FORMAT_S8_UINT) || (format == BASE_FORMAT_D16_UNORM_S8_UINT) ||
2629 (format == BASE_FORMAT_D24_UNORM_S8_UINT) || (format == BASE_FORMAT_D32_SFLOAT_S8_UINT));
2630 if (isStencilFormat) {
2631 flags |= ImageAspectFlagBits::CORE_IMAGE_ASPECT_STENCIL_BIT;
2632 }
2633
2634 } else if (format == BASE_FORMAT_S8_UINT) {
2635 flags |= ImageAspectFlagBits::CORE_IMAGE_ASPECT_STENCIL_BIT;
2636 } else if (format != BASE_FORMAT_UNDEFINED) {
2637 flags |= ImageAspectFlagBits::CORE_IMAGE_ASPECT_COLOR_BIT;
2638 }
2639
2640 return flags;
2641 }
2642
GetColorSpaceFlags() const2643 ColorSpaceFlags GpuResourceManager::GetColorSpaceFlags() const
2644 {
2645 return device_.GetColorSpaceFlags();
2646 }
2647
GetColorSpaceFormat(const Format format,const ColorSpaceFlags colorSpaceFlags) const2648 Format GpuResourceManager::GetColorSpaceFormat(const Format format, const ColorSpaceFlags colorSpaceFlags) const
2649 {
2650 return device_.GetColorSpaceFormat(format, colorSpaceFlags);
2651 }
2652
GetSurfaceTransformFlags(const RenderHandleReference & handle) const2653 SurfaceTransformFlags GpuResourceManager::GetSurfaceTransformFlags(const RenderHandleReference& handle) const
2654 {
2655 return GetSurfaceTransformFlags(handle.GetHandle());
2656 }
2657
GetSurfaceTransformFlags(const RenderHandle & handle) const2658 SurfaceTransformFlags GpuResourceManager::GetSurfaceTransformFlags(const RenderHandle& handle) const
2659 {
2660 if (IsSwapchain(handle)) {
2661 return device_.GetSurfaceTransformFlags(handle);
2662 } else {
2663 return 0u;
2664 }
2665 }
2666
MapRenderTimeGpuBuffers()2667 void GpuResourceManager::MapRenderTimeGpuBuffers()
2668 {
2669 if (RenderHandleUtil::IsValid(renderTimeReservedGpuBuffer_.baseHandle)) {
2670 renderTimeReservedGpuBuffer_.mappedData = MapBuffer(renderTimeReservedGpuBuffer_.baseHandle);
2671 }
2672 }
2673
UnmapRenderTimeGpuBuffers() const2674 void GpuResourceManager::UnmapRenderTimeGpuBuffers() const
2675 {
2676 if (RenderHandleUtil::IsValid(renderTimeReservedGpuBuffer_.baseHandle)) {
2677 UnmapBuffer(renderTimeReservedGpuBuffer_.baseHandle);
2678 }
2679 }
2680
ReserveRenderTimeGpuBuffer(const uint32_t byteSize)2681 RenderHandle GpuResourceManager::ReserveRenderTimeGpuBuffer(const uint32_t byteSize)
2682 {
2683 if (renderTimeState_ != RenderTimeState::RENDER_PRE_EXECUTE) {
2684 #if (RENDER_VALIDATION_ENABLED == 1)
2685 PLUGIN_LOG_ONCE_W(
2686 "ReserveRenderTimeGpuBuffer_RENDER", "ReserveRenderTimeGpuBuffer can only be called in PreExecuteFrame()");
2687 #endif
2688 return {};
2689 }
2690
2691 const uint32_t currOffset = renderTimeReservedGpuBuffer_.neededByteSize;
2692 const auto currIndex = static_cast<uint32_t>(renderTimeReservedGpuBuffer_.values.size());
2693
2694 const uint32_t fullByteSize = Align(byteSize, BUFFER_ALIGNMENT);
2695
2696 renderTimeReservedGpuBuffer_.neededByteSize += fullByteSize;
2697 // store only the needed bytesize
2698 renderTimeReservedGpuBuffer_.values.push_back({ currOffset, byteSize });
2699
2700 constexpr RenderHandleInfoFlags infoFlags {
2701 RenderHandleInfoFlagBits::CORE_RESOURCE_HANDLE_RENDER_TIME_MAPPED_GPU_BUFFER
2702 };
2703 return RenderHandleUtil::CreateGpuResourceHandle(RenderHandleType::GPU_BUFFER, infoFlags, currIndex, 0xffU, 0U);
2704 }
2705
AcquireRenderTimeGpuBuffer(const RenderHandle handle) const2706 IRenderNodeGpuResourceManager::MappedGpuBufferData GpuResourceManager::AcquireRenderTimeGpuBuffer(
2707 const RenderHandle handle) const
2708 {
2709 IRenderNodeGpuResourceManager::MappedGpuBufferData mgbd;
2710 if (renderTimeState_ != RenderTimeState::RENDER_EXECUTE) {
2711 #if (RENDER_VALIDATION_ENABLED == 1)
2712 PLUGIN_LOG_ONCE_W(
2713 "AcquireRenderTimeGpuBuffer_RENDER", "AcquireRenderTimeGpuBuffer can only be called in ExecuteFrame()");
2714 #endif
2715 return mgbd;
2716 } else if (RenderHandleUtil::GetAdditionalData(handle) &
2717 RenderHandleInfoFlagBits::CORE_RESOURCE_HANDLE_RENDER_TIME_MAPPED_GPU_BUFFER) {
2718 const uint32_t index = RenderHandleUtil::GetIndexPart(handle);
2719 if (index < static_cast<uint32_t>(renderTimeReservedGpuBuffer_.executeValues.size())) {
2720 const auto& ref = renderTimeReservedGpuBuffer_.executeValues[index];
2721 mgbd.bindingByteOffset = ref.byteOffset;
2722 mgbd.byteSize = ref.byteSize;
2723 mgbd.data = static_cast<uint8_t*>(renderTimeReservedGpuBuffer_.mappedData) + ref.byteOffset;
2724 mgbd.handle = renderTimeReservedGpuBuffer_.baseHandle;
2725 }
2726 }
2727 return mgbd;
2728 }
2729
SetState(const RenderTimeState rts)2730 void GpuResourceManager::SetState(const RenderTimeState rts)
2731 {
2732 renderTimeState_ = rts;
2733 }
2734
CreateGpuImageDesc(const CORE_NS::IImageContainer::ImageDesc & desc) const2735 GpuImageDesc GpuResourceManager::CreateGpuImageDesc(const CORE_NS::IImageContainer::ImageDesc& desc) const
2736 {
2737 GpuImageDesc gpuImageDesc;
2738 // default values for loaded images
2739 gpuImageDesc.imageTiling = CORE_IMAGE_TILING_OPTIMAL;
2740 gpuImageDesc.usageFlags |= CORE_IMAGE_USAGE_SAMPLED_BIT | CORE_IMAGE_USAGE_TRANSFER_DST_BIT;
2741 gpuImageDesc.memoryPropertyFlags = CORE_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
2742
2743 if (desc.imageFlags & IImageContainer::ImageFlags::FLAGS_CUBEMAP_BIT) {
2744 gpuImageDesc.createFlags |= ImageCreateFlagBits::CORE_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
2745 }
2746 if ((desc.imageFlags & IImageContainer::ImageFlags::FLAGS_REQUESTING_MIPMAPS_BIT) && (desc.mipCount > 1)) {
2747 gpuImageDesc.engineCreationFlags |= EngineImageCreationFlagBits::CORE_ENGINE_IMAGE_CREATION_GENERATE_MIPS;
2748 gpuImageDesc.usageFlags |= CORE_IMAGE_USAGE_TRANSFER_SRC_BIT;
2749 }
2750 gpuImageDesc.imageType = static_cast<RENDER_NS::ImageType>(desc.imageType);
2751 gpuImageDesc.imageViewType = static_cast<RENDER_NS::ImageViewType>(desc.imageViewType);
2752 gpuImageDesc.format = desc.format;
2753 gpuImageDesc.width = desc.width;
2754 gpuImageDesc.height = desc.height;
2755 gpuImageDesc.depth = desc.depth;
2756 gpuImageDesc.mipCount = desc.mipCount;
2757 gpuImageDesc.layerCount = desc.layerCount;
2758 return gpuImageDesc;
2759 }
2760
GetGpuResourceCache() const2761 IGpuResourceCache& GpuResourceManager::GetGpuResourceCache() const
2762 {
2763 return *gpuResourceCache_;
2764 }
2765
2766 #if (RENDER_VULKAN_VALIDATION_ENABLED == 1)
ProcessDebugTags()2767 void GpuResourceManager::ProcessDebugTags()
2768 {
2769 // NOTE: there's a minor possibility that client data has changed before these are locked
2770 // but the GPU resource is the correct one
2771 // define CORE_EXTENT_DEBUG_GPU_RESOURCE_MGR_NAMES for more precise debugging
2772 #if defined(CORE_EXTENT_DEBUG_GPU_RESOURCE_MGR_NAMES)
2773 const auto frameIdx = to_string(device_.GetFrameCount());
2774 #endif
2775 auto AddDebugTags = [&](PerManagerStore& store, const RenderHandleType handleType) {
2776 const auto lock = std::lock_guard(store.clientMutex);
2777 vector<RenderHandle> allocs = move(store.debugTagAllocations);
2778 for (const auto& handle : allocs) {
2779 if (RenderHandleUtil::GetHasNamePart(handle) == 0) {
2780 continue;
2781 }
2782 const uint32_t arrayIndex = RenderHandleUtil::GetIndexPart(handle);
2783 string name = GetName(store.clientHandles[arrayIndex].GetHandle());
2784 #if defined(CORE_EXTENT_DEBUG_GPU_RESOURCE_MGR_NAMES)
2785 const auto extentName =
2786 "_chandle:" + to_string(handle.id) + "_idx:" + to_string(arrayIndex) + "_fr:" + frameIdx;
2787 name += extentName;
2788 #endif
2789 if (handleType == RenderHandleType::GPU_BUFFER) {
2790 GpuResourceUtil::DebugBufferName(device_, *gpuBufferMgr_->Get(arrayIndex), name);
2791 } else if (handleType == RenderHandleType::GPU_IMAGE) {
2792 GpuResourceUtil::DebugImageName(device_, *gpuImageMgr_->Get(arrayIndex), name);
2793 } else if (handleType == RenderHandleType::GPU_SAMPLER) {
2794 GpuResourceUtil::DebugSamplerName(device_, *gpuSamplerMgr_->Get(arrayIndex), name);
2795 }
2796 }
2797 };
2798 AddDebugTags(bufferStore_, RenderHandleType::GPU_BUFFER);
2799 AddDebugTags(imageStore_, RenderHandleType::GPU_IMAGE);
2800 AddDebugTags(samplerStore_, RenderHandleType::GPU_SAMPLER);
2801 }
2802 #endif
2803
RenderNodeGpuResourceManager(GpuResourceManager & gpuResourceManager)2804 RenderNodeGpuResourceManager::RenderNodeGpuResourceManager(GpuResourceManager& gpuResourceManager)
2805 : gpuResourceMgr_(gpuResourceManager)
2806 {}
2807
2808 RenderNodeGpuResourceManager::~RenderNodeGpuResourceManager() = default;
2809
Get(const RenderHandle & handle) const2810 RenderHandleReference RenderNodeGpuResourceManager::Get(const RenderHandle& handle) const
2811 {
2812 return gpuResourceMgr_.Get(handle);
2813 }
2814
GetRawHandle(const RenderHandle & handle) const2815 RenderHandle RenderNodeGpuResourceManager::GetRawHandle(const RenderHandle& handle) const
2816 {
2817 return gpuResourceMgr_.GetRawHandle(handle);
2818 }
2819
Create(const GpuBufferDesc & desc)2820 RenderHandleReference RenderNodeGpuResourceManager::Create(const GpuBufferDesc& desc)
2821 {
2822 return gpuResourceMgr_.Create(desc);
2823 }
2824
Create(const string_view name,const GpuBufferDesc & desc)2825 RenderHandleReference RenderNodeGpuResourceManager::Create(const string_view name, const GpuBufferDesc& desc)
2826 {
2827 return gpuResourceMgr_.Create(name, desc);
2828 }
2829
Create(const RenderHandleReference & handle,const GpuBufferDesc & desc)2830 RenderHandleReference RenderNodeGpuResourceManager::Create(
2831 const RenderHandleReference& handle, const GpuBufferDesc& desc)
2832 {
2833 return gpuResourceMgr_.Create(handle, desc);
2834 }
2835
Create(const string_view name,const GpuBufferDesc & desc,const array_view<const uint8_t> data)2836 RenderHandleReference RenderNodeGpuResourceManager::Create(
2837 const string_view name, const GpuBufferDesc& desc, const array_view<const uint8_t> data)
2838 {
2839 return gpuResourceMgr_.Create(name, desc, data);
2840 }
2841
Create(const GpuImageDesc & desc)2842 RenderHandleReference RenderNodeGpuResourceManager::Create(const GpuImageDesc& desc)
2843 {
2844 return gpuResourceMgr_.Create(desc);
2845 }
2846
Create(const string_view name,const GpuImageDesc & desc)2847 RenderHandleReference RenderNodeGpuResourceManager::Create(const string_view name, const GpuImageDesc& desc)
2848 {
2849 return gpuResourceMgr_.Create(name, desc);
2850 }
2851
Create(const RenderHandleReference & handle,const GpuImageDesc & desc)2852 RenderHandleReference RenderNodeGpuResourceManager::Create(
2853 const RenderHandleReference& handle, const GpuImageDesc& desc)
2854 {
2855 return gpuResourceMgr_.Create(handle, desc);
2856 }
2857
Create(const string_view name,const GpuImageDesc & desc,const array_view<const uint8_t> data)2858 RenderHandleReference RenderNodeGpuResourceManager::Create(
2859 const string_view name, const GpuImageDesc& desc, const array_view<const uint8_t> data)
2860 {
2861 return gpuResourceMgr_.Create(name, desc, data);
2862 }
2863
Create(const string_view name,const GpuSamplerDesc & desc)2864 RenderHandleReference RenderNodeGpuResourceManager::Create(const string_view name, const GpuSamplerDesc& desc)
2865 {
2866 return gpuResourceMgr_.Create(name, desc);
2867 }
2868
Create(const RenderHandleReference & handle,const GpuSamplerDesc & desc)2869 RenderHandleReference RenderNodeGpuResourceManager::Create(
2870 const RenderHandleReference& handle, const GpuSamplerDesc& desc)
2871 {
2872 return gpuResourceMgr_.Create(handle, desc);
2873 }
2874
Create(const GpuSamplerDesc & desc)2875 RenderHandleReference RenderNodeGpuResourceManager::Create(const GpuSamplerDesc& desc)
2876 {
2877 return gpuResourceMgr_.Create(desc);
2878 }
2879
Create(const string_view name,const GpuAccelerationStructureDesc & desc)2880 RenderHandleReference RenderNodeGpuResourceManager::Create(
2881 const string_view name, const GpuAccelerationStructureDesc& desc)
2882 {
2883 return gpuResourceMgr_.Create(name, desc);
2884 }
2885
Create(const RenderHandleReference & handle,const GpuAccelerationStructureDesc & desc)2886 RenderHandleReference RenderNodeGpuResourceManager::Create(
2887 const RenderHandleReference& handle, const GpuAccelerationStructureDesc& desc)
2888 {
2889 return gpuResourceMgr_.Create(handle, desc);
2890 }
2891
Create(const GpuAccelerationStructureDesc & desc)2892 RenderHandleReference RenderNodeGpuResourceManager::Create(const GpuAccelerationStructureDesc& desc)
2893 {
2894 return gpuResourceMgr_.Create(desc);
2895 }
2896
GetBufferHandle(const string_view name) const2897 RenderHandle RenderNodeGpuResourceManager::GetBufferHandle(const string_view name) const
2898 {
2899 return gpuResourceMgr_.GetBufferRawHandle(name);
2900 }
2901
GetImageHandle(const string_view name) const2902 RenderHandle RenderNodeGpuResourceManager::GetImageHandle(const string_view name) const
2903 {
2904 return gpuResourceMgr_.GetImageRawHandle(name);
2905 }
2906
GetSamplerHandle(const string_view name) const2907 RenderHandle RenderNodeGpuResourceManager::GetSamplerHandle(const string_view name) const
2908 {
2909 return gpuResourceMgr_.GetSamplerRawHandle(name);
2910 }
2911
GetBufferDescriptor(const RenderHandle & handle) const2912 GpuBufferDesc RenderNodeGpuResourceManager::GetBufferDescriptor(const RenderHandle& handle) const
2913 {
2914 return gpuResourceMgr_.GetBufferDescriptor(handle);
2915 }
2916
GetImageDescriptor(const RenderHandle & handle) const2917 GpuImageDesc RenderNodeGpuResourceManager::GetImageDescriptor(const RenderHandle& handle) const
2918 {
2919 return gpuResourceMgr_.GetImageDescriptor(handle);
2920 }
2921
GetSamplerDescriptor(const RenderHandle & handle) const2922 GpuSamplerDesc RenderNodeGpuResourceManager::GetSamplerDescriptor(const RenderHandle& handle) const
2923 {
2924 return gpuResourceMgr_.GetSamplerDescriptor(handle);
2925 }
2926
GetAccelerationStructureDescriptor(const RenderHandle & handle) const2927 GpuAccelerationStructureDesc RenderNodeGpuResourceManager::GetAccelerationStructureDescriptor(
2928 const RenderHandle& handle) const
2929 {
2930 return gpuResourceMgr_.GetAccelerationStructureDescriptor(handle);
2931 }
2932
HasStagingData() const2933 bool RenderNodeGpuResourceManager::HasStagingData() const
2934 {
2935 return gpuResourceMgr_.HasStagingData();
2936 }
2937
ConsumeStagingData()2938 StagingConsumeStruct RenderNodeGpuResourceManager::ConsumeStagingData()
2939 {
2940 return gpuResourceMgr_.ConsumeStagingData();
2941 }
2942
MapBuffer(const RenderHandle & handle) const2943 void* RenderNodeGpuResourceManager::MapBuffer(const RenderHandle& handle) const
2944 {
2945 return gpuResourceMgr_.MapBuffer(handle);
2946 }
2947
MapBufferMemory(const RenderHandle & handle) const2948 void* RenderNodeGpuResourceManager::MapBufferMemory(const RenderHandle& handle) const
2949 {
2950 return gpuResourceMgr_.MapBufferMemory(handle);
2951 }
2952
UnmapBuffer(const RenderHandle & handle) const2953 void RenderNodeGpuResourceManager::UnmapBuffer(const RenderHandle& handle) const
2954 {
2955 gpuResourceMgr_.UnmapBuffer(handle);
2956 }
2957
GetGpuResourceManager()2958 GpuResourceManager& RenderNodeGpuResourceManager::GetGpuResourceManager()
2959 {
2960 return gpuResourceMgr_;
2961 }
2962
IsValid(const RenderHandle & handle) const2963 bool RenderNodeGpuResourceManager::IsValid(const RenderHandle& handle) const
2964 {
2965 return RenderHandleUtil::IsValid(handle);
2966 }
2967
IsGpuBuffer(const RenderHandle & handle) const2968 bool RenderNodeGpuResourceManager::IsGpuBuffer(const RenderHandle& handle) const
2969 {
2970 return RenderHandleUtil::IsGpuBuffer(handle);
2971 }
2972
IsGpuImage(const RenderHandle & handle) const2973 bool RenderNodeGpuResourceManager::IsGpuImage(const RenderHandle& handle) const
2974 {
2975 return RenderHandleUtil::IsGpuImage(handle);
2976 }
2977
IsGpuSampler(const RenderHandle & handle) const2978 bool RenderNodeGpuResourceManager::IsGpuSampler(const RenderHandle& handle) const
2979 {
2980 return RenderHandleUtil::IsGpuSampler(handle);
2981 }
2982
IsGpuAccelerationStructure(const RenderHandle & handle) const2983 bool RenderNodeGpuResourceManager::IsGpuAccelerationStructure(const RenderHandle& handle) const
2984 {
2985 return RenderHandleUtil::IsGpuAccelerationStructure(handle);
2986 }
2987
IsSwapchain(const RenderHandle & handle) const2988 bool RenderNodeGpuResourceManager::IsSwapchain(const RenderHandle& handle) const
2989 {
2990 return RenderHandleUtil::IsSwapchain(handle);
2991 }
2992
GetFormatProperties(const RenderHandle & handle) const2993 FormatProperties RenderNodeGpuResourceManager::GetFormatProperties(const RenderHandle& handle) const
2994 {
2995 return gpuResourceMgr_.GetFormatProperties(handle);
2996 }
2997
GetFormatProperties(const Format format) const2998 FormatProperties RenderNodeGpuResourceManager::GetFormatProperties(const Format format) const
2999 {
3000 return gpuResourceMgr_.GetFormatProperties(format);
3001 }
3002
GetImageAspectFlags(const RenderHandle & handle) const3003 ImageAspectFlags RenderNodeGpuResourceManager::GetImageAspectFlags(const RenderHandle& handle) const
3004 {
3005 return gpuResourceMgr_.GetImageAspectFlags(handle);
3006 }
3007
GetImageAspectFlags(const Format format) const3008 ImageAspectFlags RenderNodeGpuResourceManager::GetImageAspectFlags(const Format format) const
3009 {
3010 return gpuResourceMgr_.GetImageAspectFlags(format);
3011 }
3012
GetColorSpaceFlags() const3013 ColorSpaceFlags RenderNodeGpuResourceManager::GetColorSpaceFlags() const
3014 {
3015 return gpuResourceMgr_.GetColorSpaceFlags();
3016 }
3017
GetColorSpaceFormat(const Format format,const ColorSpaceFlags colorSpaceFlags) const3018 Format RenderNodeGpuResourceManager::GetColorSpaceFormat(
3019 const Format format, const ColorSpaceFlags colorSpaceFlags) const
3020 {
3021 return gpuResourceMgr_.GetColorSpaceFormat(format, colorSpaceFlags);
3022 }
3023
GetSurfaceTransformFlags(const RenderHandle & handle) const3024 SurfaceTransformFlags RenderNodeGpuResourceManager::GetSurfaceTransformFlags(const RenderHandle& handle) const
3025 {
3026 return gpuResourceMgr_.GetSurfaceTransformFlags(handle);
3027 }
3028
GetName(const RenderHandle & handle) const3029 string RenderNodeGpuResourceManager::GetName(const RenderHandle& handle) const
3030 {
3031 return gpuResourceMgr_.GetName(handle);
3032 }
3033
GetBufferHandles() const3034 vector<RenderHandle> RenderNodeGpuResourceManager::GetBufferHandles() const
3035 {
3036 return gpuResourceMgr_.GetRawBufferHandles();
3037 }
3038
GetImageHandles() const3039 vector<RenderHandle> RenderNodeGpuResourceManager::GetImageHandles() const
3040 {
3041 return gpuResourceMgr_.GetRawImageHandles();
3042 }
3043
GetSamplerHandles() const3044 vector<RenderHandle> RenderNodeGpuResourceManager::GetSamplerHandles() const
3045 {
3046 return gpuResourceMgr_.GetRawSamplerHandles();
3047 }
3048
GetGpuResourceCache() const3049 IGpuResourceCache& RenderNodeGpuResourceManager::GetGpuResourceCache() const
3050 {
3051 return gpuResourceMgr_.GetGpuResourceCache();
3052 }
3053
ReserveGpuBuffer(const uint32_t byteSize)3054 RenderHandle RenderNodeGpuResourceManager::ReserveGpuBuffer(const uint32_t byteSize)
3055 {
3056 return gpuResourceMgr_.ReserveRenderTimeGpuBuffer(byteSize);
3057 }
3058
AcquireGpubuffer(const RenderHandle handle)3059 RenderNodeGpuResourceManager::MappedGpuBufferData RenderNodeGpuResourceManager::AcquireGpubuffer(
3060 const RenderHandle handle)
3061 {
3062 return gpuResourceMgr_.AcquireRenderTimeGpuBuffer(handle);
3063 }
3064
3065 RENDER_END_NAMESPACE()
3066