• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright (c) 2018-2022 NVIDIA Corporation
2//
3// SPDX-License-Identifier: CC-BY-4.0
4
5include::{generated}/meta/{refprefix}VK_NV_optical_flow.adoc[]
6
7=== Other Extension Metadata
8
9*Last Modified Date*::
10    2022-09-26
11*Contributors*::
12  - Carsten Rohde, NVIDIA
13  - Vipul Parashar, NVIDIA
14  - Jeff Bolz, NVIDIA
15  - Eric Werness, NVIDIA
16
17=== Description
18
19Optical flow are fundamental algorithms in computer vision (CV) area.
20This extension allows applications to estimate 2D displacement of pixels
21between two frames.
22
23[NOTE]
24.Note
25====
26This extension is designed to be used with upcoming NVIDIA Optical Flow SDK
27Version 5 which will be available on NVIDIA Developer webpage.
28====
29
30include::{generated}/interfaces/VK_NV_optical_flow.adoc[]
31
32=== Examples
33
34[source,cpp]
35----
36// Example querying available input formats
37VkOpticalFlowImageFormatInfoNV ofFormatInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_INFO_NV };
38ofFormatInfo.usage = VK_OPTICAL_FLOW_USAGE_INPUT_BIT_NV;
39
40uint32_t count = 0;
41vkGetPhysicalDeviceOpticalFlowImageFormatsNV(physicalDevice, &ofFormatInfo, &count, NULL);
42VkOpticalFlowImageFormatPropertiesNV* fmt = new VkOpticalFlowImageFormatPropertiesNV[count];
43memset(fmt, 0, count  * sizeof(VkOpticalFlowImageFormatPropertiesNV));
44for (uint32_t i = 0; i < count; i++) {
45    fmt[i].sType = VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_PROPERTIES_NV;
46}
47vkGetPhysicalDeviceOpticalFlowImageFormatsNV(physicalDevice, &ofFormatInfo, &count, fmt);
48
49// Pick one of the available formats
50VkFormat inputFormat = fmt[0].format;
51
52// Check feature support for optimal tiling
53VkFormatProperties3 formatProperties3 = { VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3 };
54VkFormatProperties2 formatProperties2 = { VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2, &formatProperties3 };
55vkGetPhysicalDeviceFormatProperties2(physicalDevice, inputFormat, &formatProperties2);
56if (!(formatProperties3.optimalTilingFeatures & VK_FORMAT_FEATURE_2_OPTICAL_FLOW_IMAGE_BIT_NV)) {
57    return false;
58}
59
60// Check support for image creation parameters
61VkPhysicalDeviceImageFormatInfo2 imageFormatInfo2 = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, &ofFormatInfo };
62imageFormatInfo2.format = inputFormat;
63imageFormatInfo2.type = VK_IMAGE_TYPE_2D;
64imageFormatInfo2.tiling = VK_IMAGE_TILING_OPTIMAL;
65imageFormatInfo2.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
66
67VkImageFormatProperties2 imageFormatProperties2 = { VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2 };
68if (vkGetPhysicalDeviceImageFormatProperties2(physicalDevice, &imageFormatInfo2, &imageFormatProperties2) != VK_SUCCESS) {
69    return false;
70}
71
72VkImageCreateInfo imageCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, &ofFormatInfo };
73imageCreateInfo.imageType = VK_IMAGE_TYPE_2D;
74imageCreateInfo.format = inputFormat;
75imageCreateInfo.extent = { width, height, (uint32_t)1};
76imageCreateInfo.mipLevels = 1;
77imageCreateInfo.arrayLayers = 1;
78imageCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT;
79imageCreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;;
80imageCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
81
82vkCreateImage(device, &imageCreateInfo, NULL, &input);
83"allocate memory, bind image, create view"
84
85"do the same for reference and output"
86
87// Create optical flow session
88VkOpticalFlowSessionCreateInfoNV sessionCreateInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_SESSION_CREATE_INFO_NV };
89sessionCreateInfo.width = width;
90sessionCreateInfo.height = height;
91sessionCreateInfo.imageFormat = inputFormat;
92sessionCreateInfo.flowVectorFormat = outputFormat;
93sessionCreateInfo.outputGridSize = VK_OPTICAL_FLOW_GRID_SIZE_4X4_BIT_NV;
94sessionCreateInfo.performanceLevel = VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_SLOW_NV;
95VkOpticalFlowSessionNV session;
96vkCreateOpticalFlowSessionNV(device, &sessionCreateInfo, NULL, &session);
97
98"allocate command buffer"
99
100"transfer images to VK_PIPELINE_STAGE_2_OPTICAL_FLOW_BIT_NV"
101"transfer input images to VK_ACCESS_2_OPTICAL_FLOW_READ_BIT_NV and output image to VK_ACCESS_2_OPTICAL_FLOW_WRITE_BIT_NV"
102
103vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_INPUT_NV, inputView, VK_IMAGE_LAYOUT_GENERAL);
104vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_REFERENCE_NV, refView, VK_IMAGE_LAYOUT_GENERAL);
105vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_FLOW_VECTOR_NV, outputView, VK_IMAGE_LAYOUT_GENERAL);
106
107VkOpticalFlowExecuteInfoNV opticalFlowExecuteInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_EXECUTE_INFO_NV };
108vkCmdOpticalFlowExecuteNV(cmd, session, &opticalFlowExecuteInfo);
109
110"submit command buffer"
111----
112
113=== Version History
114
115  * Revision 1, 2022-09-26 (Carsten Rohde)
116  ** Internal revisions
117