// Copyright (c) 2018-2022 NVIDIA Corporation
//
// SPDX-License-Identifier: CC-BY-4.0

include::{generated}/meta/{refprefix}VK_NV_optical_flow.adoc[]

=== Other Extension Metadata

*Last Modified Date*::
    2022-09-26
*Contributors*::
  - Carsten Rohde, NVIDIA
  - Vipul Parashar, NVIDIA
  - Jeff Bolz, NVIDIA
  - Eric Werness, NVIDIA

=== Description

Optical flow are fundamental algorithms in computer vision (CV) area.
This extension allows applications to estimate 2D displacement of pixels
between two frames.

[NOTE]
.Note
====
This extension is designed to be used with upcoming NVIDIA Optical Flow SDK
Version 5 which will be available on NVIDIA Developer webpage.
====

include::{generated}/interfaces/VK_NV_optical_flow.adoc[]

=== Examples

[source,cpp]
----
// Example querying available input formats
VkOpticalFlowImageFormatInfoNV ofFormatInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_INFO_NV };
ofFormatInfo.usage = VK_OPTICAL_FLOW_USAGE_INPUT_BIT_NV;

uint32_t count = 0;
vkGetPhysicalDeviceOpticalFlowImageFormatsNV(physicalDevice, &ofFormatInfo, &count, NULL);
VkOpticalFlowImageFormatPropertiesNV* fmt = new VkOpticalFlowImageFormatPropertiesNV[count];
memset(fmt, 0, count  * sizeof(VkOpticalFlowImageFormatPropertiesNV));
for (uint32_t i = 0; i < count; i++) {
    fmt[i].sType = VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_PROPERTIES_NV;
}
vkGetPhysicalDeviceOpticalFlowImageFormatsNV(physicalDevice, &ofFormatInfo, &count, fmt);

// Pick one of the available formats
VkFormat inputFormat = fmt[0].format;

// Check feature support for optimal tiling
VkFormatProperties3 formatProperties3 = { VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3 };
VkFormatProperties2 formatProperties2 = { VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2, &formatProperties3 };
vkGetPhysicalDeviceFormatProperties2(physicalDevice, inputFormat, &formatProperties2);
if (!(formatProperties3.optimalTilingFeatures & VK_FORMAT_FEATURE_2_OPTICAL_FLOW_IMAGE_BIT_NV)) {
    return false;
}

// Check support for image creation parameters
VkPhysicalDeviceImageFormatInfo2 imageFormatInfo2 = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, &ofFormatInfo };
imageFormatInfo2.format = inputFormat;
imageFormatInfo2.type = VK_IMAGE_TYPE_2D;
imageFormatInfo2.tiling = VK_IMAGE_TILING_OPTIMAL;
imageFormatInfo2.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;

VkImageFormatProperties2 imageFormatProperties2 = { VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2 };
if (vkGetPhysicalDeviceImageFormatProperties2(physicalDevice, &imageFormatInfo2, &imageFormatProperties2) != VK_SUCCESS) {
    return false;
}

VkImageCreateInfo imageCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, &ofFormatInfo };
imageCreateInfo.imageType = VK_IMAGE_TYPE_2D;
imageCreateInfo.format = inputFormat;
imageCreateInfo.extent = { width, height, (uint32_t)1};
imageCreateInfo.mipLevels = 1;
imageCreateInfo.arrayLayers = 1;
imageCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageCreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;;
imageCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL;

vkCreateImage(device, &imageCreateInfo, NULL, &input);
"allocate memory, bind image, create view"

"do the same for reference and output"

// Create optical flow session
VkOpticalFlowSessionCreateInfoNV sessionCreateInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_SESSION_CREATE_INFO_NV };
sessionCreateInfo.width = width;
sessionCreateInfo.height = height;
sessionCreateInfo.imageFormat = inputFormat;
sessionCreateInfo.flowVectorFormat = outputFormat;
sessionCreateInfo.outputGridSize = VK_OPTICAL_FLOW_GRID_SIZE_4X4_BIT_NV;
sessionCreateInfo.performanceLevel = VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_SLOW_NV;
VkOpticalFlowSessionNV session;
vkCreateOpticalFlowSessionNV(device, &sessionCreateInfo, NULL, &session);

"allocate command buffer"

"transfer images to VK_PIPELINE_STAGE_2_OPTICAL_FLOW_BIT_NV"
"transfer input images to VK_ACCESS_2_OPTICAL_FLOW_READ_BIT_NV and output image to VK_ACCESS_2_OPTICAL_FLOW_WRITE_BIT_NV"

vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_INPUT_NV, inputView, VK_IMAGE_LAYOUT_GENERAL);
vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_REFERENCE_NV, refView, VK_IMAGE_LAYOUT_GENERAL);
vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_FLOW_VECTOR_NV, outputView, VK_IMAGE_LAYOUT_GENERAL);

VkOpticalFlowExecuteInfoNV opticalFlowExecuteInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_EXECUTE_INFO_NV };
vkCmdOpticalFlowExecuteNV(cmd, session, &opticalFlowExecuteInfo);

"submit command buffer"
----

=== Version History

  * Revision 1, 2022-09-26 (Carsten Rohde)
  ** Internal revisions