virtio/vulkan/vn_buffer.c

/*
 * Copyright 2019 Google LLC
 * SPDX-License-Identifier: MIT
 *
 * based in part on anv and radv which are:
 * Copyright © 2015 Intel Corporation
 * Copyright © 2016 Red Hat.
 * Copyright © 2016 Bas Nieuwenhuizen
 */

#include "vn_buffer.h"

#include "venus-protocol/vn_protocol_driver_buffer.h"
#include "venus-protocol/vn_protocol_driver_buffer_view.h"

#include "vn_android.h"
#include "vn_device.h"
#include "vn_device_memory.h"
#include "vn_physical_device.h"

/* buffer commands */

static inline uint64_t
vn_buffer_get_cache_index(const VkBufferCreateInfo *create_info,
                          struct vn_buffer_reqs_cache *cache)
{
   /* For simplicity, cache only when below conditions are met:
    * - pNext is NULL
    * - VK_SHARING_MODE_EXCLUSIVE or VK_SHARING_MODE_CONCURRENT across all
    *
    * Combine sharing mode, flags and usage bits to form a unique index.
    *
    * Btw, we assume VkBufferCreateFlagBits won't exhaust all 32bits, at least
    * no earlier than VkBufferUsageFlagBits.
    *
    * TODO: extend cache to cover VkBufferUsageFlags2CreateInfo (introduced in
    * VK_KHR_maintenance5 and promoted to 1.4).
    */
   assert(!(create_info->flags & 0x80000000));

   const bool is_exclusive =
      create_info->sharingMode == VK_SHARING_MODE_EXCLUSIVE;
   const bool is_concurrent =
      create_info->sharingMode == VK_SHARING_MODE_CONCURRENT &&
      create_info->queueFamilyIndexCount == cache->queue_family_count;
   if (create_info->size <= cache->max_buffer_size &&
       create_info->pNext == NULL && (is_exclusive || is_concurrent)) {
      return (uint64_t)is_concurrent << 63 |
             (uint64_t)create_info->flags << 32 | create_info->usage;
   }

   /* index being zero suggests uncachable since usage must not be zero */
   return 0;
}

static inline uint64_t
vn_buffer_get_max_buffer_size(struct vn_physical_device *physical_dev)
{
   /* Without maintenance4, hardcode the min of supported drivers:
    * - anv:  1ull << 30
    * - radv: UINT32_MAX - 4
    * - tu:   UINT32_MAX + 1
    * - lvp:  UINT32_MAX
    * - mali: UINT32_MAX
    */
   static const uint64_t safe_max_buffer_size = 1ULL << 30;
   return physical_dev->base.base.supported_features.maintenance4
             ? physical_dev->base.base.properties.maxBufferSize
             : safe_max_buffer_size;
}

void
vn_buffer_reqs_cache_init(struct vn_device *dev)
{
   assert(dev->physical_device->queue_family_count);

   dev->buffer_reqs_cache.max_buffer_size =
      vn_buffer_get_max_buffer_size(dev->physical_device);
   dev->buffer_reqs_cache.queue_family_count =
      dev->physical_device->queue_family_count;

   simple_mtx_init(&dev->buffer_reqs_cache.mutex, mtx_plain);
   util_sparse_array_init(&dev->buffer_reqs_cache.entries,
                          sizeof(struct vn_buffer_reqs_cache_entry), 64);
}

static void
vn_buffer_reqs_cache_debug_dump(struct vn_buffer_reqs_cache *cache)
{
   vn_log(NULL, "dumping buffer cache statistics");
   vn_log(NULL, "  cache hit: %d", cache->debug.cache_hit_count);
   vn_log(NULL, "  cache miss: %d", cache->debug.cache_miss_count);
   vn_log(NULL, "  cache skip: %d", cache->debug.cache_skip_count);
}

void
vn_buffer_reqs_cache_fini(struct vn_device *dev)
{
   util_sparse_array_finish(&dev->buffer_reqs_cache.entries);
   simple_mtx_destroy(&dev->buffer_reqs_cache.mutex);

   if (VN_DEBUG(CACHE))
      vn_buffer_reqs_cache_debug_dump(&dev->buffer_reqs_cache);
}

static inline uint32_t
vn_buffer_get_ahb_memory_type_bits(struct vn_device *dev)
{
   struct vn_buffer_reqs_cache *cache = &dev->buffer_reqs_cache;
   if (unlikely(!cache->ahb_mem_type_bits_valid)) {
      simple_mtx_lock(&cache->mutex);
      if (!cache->ahb_mem_type_bits_valid) {
         cache->ahb_mem_type_bits =
            vn_android_get_ahb_buffer_memory_type_bits(dev);
         cache->ahb_mem_type_bits_valid = true;
      }
      simple_mtx_unlock(&cache->mutex);
   }

   return cache->ahb_mem_type_bits;
}

static inline VkDeviceSize
vn_buffer_get_aligned_memory_requirement_size(VkDeviceSize size,
                                              const VkMemoryRequirements *req)
{
   /* TODO remove comment after mandating VK_KHR_maintenance4
    *
    * This is based on below implementation defined behavior:
    *    req.size <= align64(info.size, req.alignment)
    */
   return align64(size, req->alignment);
}

static struct vn_buffer_reqs_cache_entry *
vn_buffer_get_cached_memory_requirements(
   struct vn_buffer_reqs_cache *cache,
   const VkBufferCreateInfo *create_info,
   struct vn_buffer_memory_requirements *out)
{
   if (VN_PERF(NO_ASYNC_BUFFER_CREATE))
      return NULL;

   /* 12.7. Resource Memory Association
    *
    * The memoryTypeBits member is identical for all VkBuffer objects created
    * with the same value for the flags and usage members in the
    * VkBufferCreateInfo structure and the handleTypes member of the
    * VkExternalMemoryBufferCreateInfo structure passed to vkCreateBuffer.
    */
   const uint64_t idx = vn_buffer_get_cache_index(create_info, cache);
   if (idx) {
      struct vn_buffer_reqs_cache_entry *entry =
         util_sparse_array_get(&cache->entries, idx);

      if (entry->valid) {
         *out = entry->requirements;

         out->memory.memoryRequirements.size =
            vn_buffer_get_aligned_memory_requirement_size(
               create_info->size, &out->memory.memoryRequirements);

         p_atomic_inc(&cache->debug.cache_hit_count);
      } else {
         p_atomic_inc(&cache->debug.cache_miss_count);
      }

      return entry;
   }

   p_atomic_inc(&cache->debug.cache_skip_count);

   return NULL;
}

static void
vn_buffer_reqs_cache_entry_init(struct vn_buffer_reqs_cache *cache,
                                struct vn_buffer_reqs_cache_entry *entry,
                                VkMemoryRequirements2 *req)
{
   simple_mtx_lock(&cache->mutex);

   /* Entry might have already been initialized by another thread
    * before the lock
    */
   if (entry->valid)
      goto unlock;

   entry->requirements.memory = *req;

   const VkMemoryDedicatedRequirements *dedicated_req =
      vk_find_struct_const(req->pNext, MEMORY_DEDICATED_REQUIREMENTS);
   if (dedicated_req)
      entry->requirements.dedicated = *dedicated_req;

   entry->valid = true;

unlock:
   simple_mtx_unlock(&cache->mutex);

   /* ensure invariance of the memory requirement size */
   req->memoryRequirements.size =
      vn_buffer_get_aligned_memory_requirement_size(
         req->memoryRequirements.size,
         &entry->requirements.memory.memoryRequirements);
}

static void
vn_copy_cached_memory_requirements(
   const struct vn_buffer_memory_requirements *cached,
   VkMemoryRequirements2 *out_mem_req)
{
   union {
      VkBaseOutStructure *pnext;
      VkMemoryRequirements2 *two;
      VkMemoryDedicatedRequirements *dedicated;
   } u = { .two = out_mem_req };

   while (u.pnext) {
      switch (u.pnext->sType) {
      case VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2:
         u.two->memoryRequirements = cached->memory.memoryRequirements;
         break;
      case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS:
         u.dedicated->prefersDedicatedAllocation =
            cached->dedicated.prefersDedicatedAllocation;
         u.dedicated->requiresDedicatedAllocation =
            cached->dedicated.requiresDedicatedAllocation;
         break;
      default:
         break;
      }
      u.pnext = u.pnext->pNext;
   }
}

static VkResult
vn_buffer_init(struct vn_device *dev,
               const VkBufferCreateInfo *create_info,
               struct vn_buffer *buf)
{
   VkDevice dev_handle = vn_device_to_handle(dev);
   VkBuffer buf_handle = vn_buffer_to_handle(buf);
   struct vn_buffer_reqs_cache *cache = &dev->buffer_reqs_cache;
   VkResult result;

   /* If cacheable and mem requirements found in cache, make async call */
   struct vn_buffer_reqs_cache_entry *entry =
      vn_buffer_get_cached_memory_requirements(cache, create_info,
                                               &buf->requirements);

   /* Check size instead of entry->valid to be lock free */
   if (buf->requirements.memory.memoryRequirements.size) {
      vn_async_vkCreateBuffer(dev->primary_ring, dev_handle, create_info,
                              NULL, &buf_handle);
      return VK_SUCCESS;
   }

   /* If cache miss or not cacheable, make synchronous call */
   result = vn_call_vkCreateBuffer(dev->primary_ring, dev_handle, create_info,
                                   NULL, &buf_handle);
   if (result != VK_SUCCESS)
      return result;

   buf->requirements.memory.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2;
   buf->requirements.memory.pNext = &buf->requirements.dedicated;
   buf->requirements.dedicated.sType =
      VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS;
   buf->requirements.dedicated.pNext = NULL;

   vn_call_vkGetBufferMemoryRequirements2(
      dev->primary_ring, dev_handle,
      &(VkBufferMemoryRequirementsInfo2){
         .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2,
         .buffer = buf_handle,
      },
      &buf->requirements.memory);

   /* If cacheable, store mem requirements from the synchronous call */
   if (entry) {
      vn_buffer_reqs_cache_entry_init(cache, entry,
                                      &buf->requirements.memory);
   }

   return VK_SUCCESS;
}

VkResult
vn_buffer_create(struct vn_device *dev,
                 const VkBufferCreateInfo *create_info,
                 const VkAllocationCallbacks *alloc,
                 struct vn_buffer **out_buf)
{
   struct vn_buffer *buf = NULL;
   VkResult result;

   buf = vk_zalloc(alloc, sizeof(*buf), VN_DEFAULT_ALIGN,
                   VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
   if (!buf)
      return VK_ERROR_OUT_OF_HOST_MEMORY;

   vn_object_base_init(&buf->base, VK_OBJECT_TYPE_BUFFER, &dev->base);

   result = vn_buffer_init(dev, create_info, buf);
   if (result != VK_SUCCESS) {
      vn_object_base_fini(&buf->base);
      vk_free(alloc, buf);
      return result;
   }

   *out_buf = buf;

   return VK_SUCCESS;
}

struct vn_buffer_create_info {
   VkBufferCreateInfo create;
   VkExternalMemoryBufferCreateInfo external;
   VkBufferOpaqueCaptureAddressCreateInfo capture;
};

static const VkBufferCreateInfo *
vn_buffer_fix_create_info(
   const VkBufferCreateInfo *create_info,
   const VkExternalMemoryHandleTypeFlagBits renderer_handle_type,
   struct vn_buffer_create_info *local_info)
{
   local_info->create = *create_info;
   VkBaseOutStructure *cur = (void *)&local_info->create;

   vk_foreach_struct_const(src, create_info->pNext) {
      void *next = NULL;
      switch (src->sType) {
      case VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO:
         memcpy(&local_info->external, src, sizeof(local_info->external));
         local_info->external.handleTypes = renderer_handle_type;
         next = &local_info->external;
         break;
      case VK_STRUCTURE_TYPE_BUFFER_OPAQUE_CAPTURE_ADDRESS_CREATE_INFO:
         memcpy(&local_info->capture, src, sizeof(local_info->capture));
         next = &local_info->capture;
         break;
      default:
         break;
      }

      if (next) {
         cur->pNext = next;
         cur = next;
      }
   }

   cur->pNext = NULL;

   return &local_info->create;
}

VkResult
vn_CreateBuffer(VkDevice device,
                const VkBufferCreateInfo *pCreateInfo,
                const VkAllocationCallbacks *pAllocator,
                VkBuffer *pBuffer)
{
   struct vn_device *dev = vn_device_from_handle(device);
   const VkAllocationCallbacks *alloc =
      pAllocator ? pAllocator : &dev->base.base.alloc;
   const VkExternalMemoryHandleTypeFlagBits renderer_handle_type =
      dev->physical_device->external_memory.renderer_handle_type;

   struct vn_buffer_create_info local_info;
   const VkExternalMemoryBufferCreateInfo *external_info =
      vk_find_struct_const(pCreateInfo->pNext,
                           EXTERNAL_MEMORY_BUFFER_CREATE_INFO);
   if (external_info && external_info->handleTypes &&
       external_info->handleTypes != renderer_handle_type) {
      pCreateInfo = vn_buffer_fix_create_info(
         pCreateInfo, renderer_handle_type, &local_info);
   }

   struct vn_buffer *buf;
   VkResult result = vn_buffer_create(dev, pCreateInfo, alloc, &buf);
   if (result != VK_SUCCESS)
      return vn_error(dev->instance, result);

   if (external_info &&
       external_info->handleTypes ==
          VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID) {
      /* AHB backed buffer layers on top of renderer external memory, so here
       * we combine the queried type bits from both buffer memory requirement
       * and renderer external memory properties.
       */
      buf->requirements.memory.memoryRequirements.memoryTypeBits &=
         vn_buffer_get_ahb_memory_type_bits(dev);

      assert(buf->requirements.memory.memoryRequirements.memoryTypeBits);
   }

   *pBuffer = vn_buffer_to_handle(buf);

   return VK_SUCCESS;
}

void
vn_DestroyBuffer(VkDevice device,
                 VkBuffer buffer,
                 const VkAllocationCallbacks *pAllocator)
{
   struct vn_device *dev = vn_device_from_handle(device);
   struct vn_buffer *buf = vn_buffer_from_handle(buffer);
   const VkAllocationCallbacks *alloc =
      pAllocator ? pAllocator : &dev->base.base.alloc;

   if (!buf)
      return;

   vn_async_vkDestroyBuffer(dev->primary_ring, device, buffer, NULL);

   vn_object_base_fini(&buf->base);
   vk_free(alloc, buf);
}

VkDeviceAddress
vn_GetBufferDeviceAddress(VkDevice device,
                          const VkBufferDeviceAddressInfo *pInfo)
{
   struct vn_device *dev = vn_device_from_handle(device);

   return vn_call_vkGetBufferDeviceAddress(dev->primary_ring, device, pInfo);
}

uint64_t
vn_GetBufferOpaqueCaptureAddress(VkDevice device,
                                 const VkBufferDeviceAddressInfo *pInfo)
{
   struct vn_device *dev = vn_device_from_handle(device);

   return vn_call_vkGetBufferOpaqueCaptureAddress(dev->primary_ring, device,
                                                  pInfo);
}

void
vn_GetBufferMemoryRequirements2(VkDevice device,
                                const VkBufferMemoryRequirementsInfo2 *pInfo,
                                VkMemoryRequirements2 *pMemoryRequirements)
{
   const struct vn_buffer *buf = vn_buffer_from_handle(pInfo->buffer);

   vn_copy_cached_memory_requirements(&buf->requirements,
                                      pMemoryRequirements);
}

VkResult
vn_BindBufferMemory2(VkDevice device,
                     uint32_t bindInfoCount,
                     const VkBindBufferMemoryInfo *pBindInfos)
{
   struct vn_device *dev = vn_device_from_handle(device);
   vn_async_vkBindBufferMemory2(dev->primary_ring, device, bindInfoCount,
                                pBindInfos);

   return VK_SUCCESS;
}

/* buffer view commands */

VkResult
vn_CreateBufferView(VkDevice device,
                    const VkBufferViewCreateInfo *pCreateInfo,
                    const VkAllocationCallbacks *pAllocator,
                    VkBufferView *pView)
{
   struct vn_device *dev = vn_device_from_handle(device);
   const VkAllocationCallbacks *alloc =
      pAllocator ? pAllocator : &dev->base.base.alloc;

   struct vn_buffer_view *view =
      vk_zalloc(alloc, sizeof(*view), VN_DEFAULT_ALIGN,
                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
   if (!view)
      return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);

   vn_object_base_init(&view->base, VK_OBJECT_TYPE_BUFFER_VIEW, &dev->base);

   VkBufferView view_handle = vn_buffer_view_to_handle(view);
   vn_async_vkCreateBufferView(dev->primary_ring, device, pCreateInfo, NULL,
                               &view_handle);

   *pView = view_handle;

   return VK_SUCCESS;
}

void
vn_DestroyBufferView(VkDevice device,
                     VkBufferView bufferView,
                     const VkAllocationCallbacks *pAllocator)
{
   struct vn_device *dev = vn_device_from_handle(device);
   struct vn_buffer_view *view = vn_buffer_view_from_handle(bufferView);
   const VkAllocationCallbacks *alloc =
      pAllocator ? pAllocator : &dev->base.base.alloc;

   if (!view)
      return;

   vn_async_vkDestroyBufferView(dev->primary_ring, device, bufferView, NULL);

   vn_object_base_fini(&view->base);
   vk_free(alloc, view);
}

void
vn_GetDeviceBufferMemoryRequirements(
   VkDevice device,
   const VkDeviceBufferMemoryRequirements *pInfo,
   VkMemoryRequirements2 *pMemoryRequirements)
{
   struct vn_device *dev = vn_device_from_handle(device);
   struct vn_buffer_reqs_cache *cache = &dev->buffer_reqs_cache;
   struct vn_buffer_memory_requirements reqs = { 0 };

   /* If cacheable and mem requirements found in cache, skip host call */
   struct vn_buffer_reqs_cache_entry *entry =
      vn_buffer_get_cached_memory_requirements(cache, pInfo->pCreateInfo,
                                               &reqs);

   /* Check size instead of entry->valid to be lock free */
   if (reqs.memory.memoryRequirements.size) {
      vn_copy_cached_memory_requirements(&reqs, pMemoryRequirements);
      return;
   }

   /* Make the host call if not found in cache or not cacheable */
   vn_call_vkGetDeviceBufferMemoryRequirements(dev->primary_ring, device,
                                               pInfo, pMemoryRequirements);

   /* If cacheable, store mem requirements from the host call */
   if (entry)
      vn_buffer_reqs_cache_entry_init(cache, entry, pMemoryRequirements);
}