1 /* 2 * This file is part of FFmpeg. 3 * 4 * FFmpeg is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU Lesser General Public 6 * License as published by the Free Software Foundation; either 7 * version 2.1 of the License, or (at your option) any later version. 8 * 9 * FFmpeg is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 * Lesser General Public License for more details. 13 * 14 * You should have received a copy of the GNU Lesser General Public 15 * License along with FFmpeg; if not, write to the Free Software 16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 19 #ifndef AVFILTER_VULKAN_H 20 #define AVFILTER_VULKAN_H 21 22 #include "avfilter.h" 23 #include "libavutil/pixdesc.h" 24 #include "libavutil/bprint.h" 25 #include "libavutil/hwcontext.h" 26 #include "libavutil/hwcontext_vulkan.h" 27 28 /* GLSL management macros */ 29 #define INDENT(N) INDENT_##N 30 #define INDENT_0 31 #define INDENT_1 INDENT_0 " " 32 #define INDENT_2 INDENT_1 INDENT_1 33 #define INDENT_3 INDENT_2 INDENT_1 34 #define INDENT_4 INDENT_3 INDENT_1 35 #define INDENT_5 INDENT_4 INDENT_1 36 #define INDENT_6 INDENT_5 INDENT_1 37 #define C(N, S) INDENT(N) #S "\n" 38 #define GLSLC(N, S) av_bprintf(&shd->src, C(N, S)) 39 #define GLSLA(...) av_bprintf(&shd->src, __VA_ARGS__) 40 #define GLSLF(N, S, ...) av_bprintf(&shd->src, C(N, S), __VA_ARGS__) 41 #define GLSLD(D) GLSLC(0, ); \ 42 av_bprint_append_data(&shd->src, D, strlen(D)); \ 43 GLSLC(0, ) 44 45 /* Helper, pretty much every Vulkan return value needs to be checked */ 46 #define RET(x) \ 47 do { \ 48 if ((err = (x)) < 0) \ 49 goto fail; \ 50 } while (0) 51 52 /* Gets the queues count for a single queue family */ 53 #define GET_QUEUE_COUNT(hwctx, graph, comp, tx) ( \ 54 graph ? hwctx->nb_graphics_queues : \ 55 comp ? (hwctx->nb_comp_queues ? \ 56 hwctx->nb_comp_queues : hwctx->nb_graphics_queues) : \ 57 tx ? (hwctx->nb_tx_queues ? hwctx->nb_tx_queues : \ 58 (hwctx->nb_comp_queues ? \ 59 hwctx->nb_comp_queues : hwctx->nb_graphics_queues)) : \ 60 0 \ 61 ) 62 63 /* Useful for attaching immutable samplers to arrays */ 64 #define DUP_SAMPLER_ARRAY4(x) (VkSampler []){ x, x, x, x, } 65 66 typedef struct SPIRVShader { 67 const char *name; /* Name for id/debugging purposes */ 68 AVBPrint src; 69 int local_size[3]; /* Compute shader workgroup sizes */ 70 VkPipelineShaderStageCreateInfo shader; 71 } SPIRVShader; 72 73 typedef struct VulkanDescriptorSetBinding { 74 const char *name; 75 VkDescriptorType type; 76 const char *mem_layout; /* Storage images (rgba8, etc.) and buffers (std430, etc.) */ 77 const char *mem_quali; /* readonly, writeonly, etc. */ 78 const char *buf_content; /* For buffers */ 79 uint32_t dimensions; /* Needed for e.g. sampler%iD */ 80 uint32_t elems; /* 0 - scalar, 1 or more - vector */ 81 VkShaderStageFlags stages; 82 const VkSampler *samplers; /* Immutable samplers, length - #elems */ 83 void *updater; /* Pointer to VkDescriptor*Info */ 84 } VulkanDescriptorSetBinding; 85 86 typedef struct FFVkBuffer { 87 VkBuffer buf; 88 VkDeviceMemory mem; 89 VkMemoryPropertyFlagBits flags; 90 } FFVkBuffer; 91 92 typedef struct VulkanPipeline { 93 VkPipelineBindPoint bind_point; 94 95 /* Contexts */ 96 VkPipelineLayout pipeline_layout; 97 VkPipeline pipeline; 98 99 /* Shaders */ 100 SPIRVShader **shaders; 101 int shaders_num; 102 103 /* Push consts */ 104 VkPushConstantRange *push_consts; 105 int push_consts_num; 106 107 /* Descriptors */ 108 VkDescriptorSetLayout *desc_layout; 109 VkDescriptorPool desc_pool; 110 VkDescriptorSet *desc_set; 111 VkDescriptorUpdateTemplate *desc_template; 112 int desc_layout_num; 113 int descriptor_sets_num; 114 int pool_size_desc_num; 115 116 /* Temporary, used to store data in between initialization stages */ 117 VkDescriptorUpdateTemplateCreateInfo *desc_template_info; 118 VkDescriptorPoolSize *pool_size_desc; 119 } VulkanPipeline; 120 121 typedef struct FFVkQueueCtx { 122 VkFence fence; 123 VkQueue queue; 124 125 /* Buffer dependencies */ 126 AVBufferRef **buf_deps; 127 int nb_buf_deps; 128 int buf_deps_alloc_size; 129 130 /* Frame dependencies */ 131 AVFrame **frame_deps; 132 int nb_frame_deps; 133 int frame_deps_alloc_size; 134 } FFVkQueueCtx; 135 136 typedef struct FFVkExecContext { 137 VkCommandPool pool; 138 VkCommandBuffer *bufs; 139 FFVkQueueCtx *queues; 140 141 AVBufferRef ***deps; 142 int *nb_deps; 143 int *dep_alloc_size; 144 145 VulkanPipeline *bound_pl; 146 147 VkSemaphore *sem_wait; 148 int sem_wait_alloc; /* Allocated sem_wait */ 149 int sem_wait_cnt; 150 151 VkPipelineStageFlagBits *sem_wait_dst; 152 int sem_wait_dst_alloc; /* Allocated sem_wait_dst */ 153 154 VkSemaphore *sem_sig; 155 int sem_sig_alloc; /* Allocated sem_sig */ 156 int sem_sig_cnt; 157 } FFVkExecContext; 158 159 typedef struct VulkanFilterContext { 160 const AVClass *class; 161 162 AVBufferRef *device_ref; 163 AVBufferRef *frames_ref; /* For in-place filtering */ 164 AVHWDeviceContext *device; 165 AVVulkanDeviceContext *hwctx; 166 167 /* State - mirrored with the exec ctx */ 168 int cur_queue_idx; 169 int queue_family_idx; 170 int queue_count; 171 172 /* Properties */ 173 int output_width; 174 int output_height; 175 enum AVPixelFormat output_format; 176 enum AVPixelFormat input_format; 177 178 /* Samplers */ 179 VkSampler **samplers; 180 int samplers_num; 181 182 /* Exec contexts */ 183 FFVkExecContext **exec_ctx; 184 int exec_ctx_num; 185 186 /* Pipelines (each can have 1 shader of each type) */ 187 VulkanPipeline **pipelines; 188 int pipelines_num; 189 190 void *scratch; /* Scratch memory used only in functions */ 191 unsigned int scratch_size; 192 } VulkanFilterContext; 193 194 /* Identity mapping - r = r, b = b, g = g, a = a */ 195 extern const VkComponentMapping ff_comp_identity_map; 196 197 /** 198 * General lavfi IO functions 199 */ 200 int ff_vk_filter_query_formats (AVFilterContext *avctx); 201 int ff_vk_filter_init (AVFilterContext *avctx); 202 int ff_vk_filter_config_input (AVFilterLink *inlink); 203 int ff_vk_filter_config_output (AVFilterLink *outlink); 204 int ff_vk_filter_config_output_inplace(AVFilterLink *outlink); 205 void ff_vk_filter_uninit (AVFilterContext *avctx); 206 207 /** 208 * Converts Vulkan return values to strings 209 */ 210 const char *ff_vk_ret2str(VkResult res); 211 212 /** 213 * Returns 1 if the image is any sort of supported RGB 214 */ 215 int ff_vk_mt_is_np_rgb(enum AVPixelFormat pix_fmt); 216 217 /** 218 * Gets the glsl format string for a pixel format 219 */ 220 const char *ff_vk_shader_rep_fmt(enum AVPixelFormat pixfmt); 221 222 /** 223 * Create a Vulkan sampler, will be auto-freed in ff_vk_filter_uninit() 224 */ 225 VkSampler *ff_vk_init_sampler(AVFilterContext *avctx, int unnorm_coords, 226 VkFilter filt); 227 228 /** 229 * Create an imageview. 230 * Guaranteed to remain alive until the queue submission has finished executing, 231 * and will be destroyed after that. 232 */ 233 int ff_vk_create_imageview(AVFilterContext *avctx, FFVkExecContext *e, 234 VkImageView *v, VkImage img, VkFormat fmt, 235 const VkComponentMapping map); 236 237 /** 238 * Define a push constant for a given stage into a pipeline. 239 * Must be called before the pipeline layout has been initialized. 240 */ 241 int ff_vk_add_push_constant(AVFilterContext *avctx, VulkanPipeline *pl, 242 int offset, int size, VkShaderStageFlagBits stage); 243 244 /** 245 * Inits a pipeline. Everything in it will be auto-freed when calling 246 * ff_vk_filter_uninit(). 247 */ 248 VulkanPipeline *ff_vk_create_pipeline(AVFilterContext *avctx); 249 250 /** 251 * Inits a shader for a specific pipeline. Will be auto-freed on uninit. 252 */ 253 SPIRVShader *ff_vk_init_shader(AVFilterContext *avctx, VulkanPipeline *pl, 254 const char *name, VkShaderStageFlags stage); 255 256 /** 257 * Writes the workgroup size for a shader. 258 */ 259 void ff_vk_set_compute_shader_sizes(AVFilterContext *avctx, SPIRVShader *shd, 260 int local_size[3]); 261 262 /** 263 * Adds a descriptor set to the shader and registers them in the pipeline. 264 */ 265 int ff_vk_add_descriptor_set(AVFilterContext *avctx, VulkanPipeline *pl, 266 SPIRVShader *shd, VulkanDescriptorSetBinding *desc, 267 int num, int only_print_to_shader); 268 269 /** 270 * Compiles the shader, entrypoint must be set to "main". 271 */ 272 int ff_vk_compile_shader(AVFilterContext *avctx, SPIRVShader *shd, 273 const char *entrypoint); 274 275 /** 276 * Initializes the pipeline layout after all shaders and descriptor sets have 277 * been finished. 278 */ 279 int ff_vk_init_pipeline_layout(AVFilterContext *avctx, VulkanPipeline *pl); 280 281 /** 282 * Initializes a compute pipeline. Will pick the first shader with the 283 * COMPUTE flag set. 284 */ 285 int ff_vk_init_compute_pipeline(AVFilterContext *avctx, VulkanPipeline *pl); 286 287 /** 288 * Updates a descriptor set via the updaters defined. 289 * Can be called immediately after pipeline creation, but must be called 290 * at least once before queue submission. 291 */ 292 void ff_vk_update_descriptor_set(AVFilterContext *avctx, VulkanPipeline *pl, 293 int set_id); 294 295 /** 296 * Init an execution context for command recording and queue submission. 297 * WIll be auto-freed on uninit. 298 */ 299 int ff_vk_create_exec_ctx(AVFilterContext *avctx, FFVkExecContext **ctx); 300 301 /** 302 * Begin recording to the command buffer. Previous execution must have been 303 * completed, which ff_vk_submit_exec_queue() will ensure. 304 */ 305 int ff_vk_start_exec_recording(AVFilterContext *avctx, FFVkExecContext *e); 306 307 /** 308 * Add a command to bind the completed pipeline and its descriptor sets. 309 * Must be called after ff_vk_start_exec_recording() and before submission. 310 */ 311 void ff_vk_bind_pipeline_exec(AVFilterContext *avctx, FFVkExecContext *e, 312 VulkanPipeline *pl); 313 314 /** 315 * Updates push constants. 316 * Must be called after binding a pipeline if any push constants were defined. 317 */ 318 void ff_vk_update_push_exec(AVFilterContext *avctx, FFVkExecContext *e, 319 VkShaderStageFlagBits stage, int offset, 320 size_t size, void *src); 321 322 /** 323 * Gets the command buffer to use for this submission from the exe context. 324 */ 325 VkCommandBuffer ff_vk_get_exec_buf(AVFilterContext *avctx, FFVkExecContext *e); 326 327 /** 328 * Adds a generic AVBufferRef as a queue depenency. 329 */ 330 int ff_vk_add_dep_exec_ctx(AVFilterContext *avctx, FFVkExecContext *e, 331 AVBufferRef **deps, int nb_deps); 332 333 /** 334 * Discards all queue dependencies 335 */ 336 void ff_vk_discard_exec_deps(AVFilterContext *avctx, FFVkExecContext *e); 337 338 /** 339 * Adds a frame as a queue dependency. This also manages semaphore signalling. 340 * Must be called before submission. 341 */ 342 int ff_vk_add_exec_dep(AVFilterContext *avctx, FFVkExecContext *e, 343 AVFrame *frame, VkPipelineStageFlagBits in_wait_dst_flag); 344 345 /** 346 * Submits a command buffer to the queue for execution. 347 * Will block until execution has finished in order to simplify resource 348 * management. 349 */ 350 int ff_vk_submit_exec_queue(AVFilterContext *avctx, FFVkExecContext *e); 351 352 /** 353 * Create a VkBuffer with the specified parameters. 354 */ 355 int ff_vk_create_buf(AVFilterContext *avctx, FFVkBuffer *buf, size_t size, 356 VkBufferUsageFlags usage, VkMemoryPropertyFlagBits flags); 357 358 /** 359 * Maps the buffer to userspace. Set invalidate to 1 if reading the contents 360 * is necessary. 361 */ 362 int ff_vk_map_buffers(AVFilterContext *avctx, FFVkBuffer *buf, uint8_t *mem[], 363 int nb_buffers, int invalidate); 364 365 /** 366 * Unmaps the buffer from userspace. Set flush to 1 to write and sync. 367 */ 368 int ff_vk_unmap_buffers(AVFilterContext *avctx, FFVkBuffer *buf, int nb_buffers, 369 int flush); 370 371 /** 372 * Frees a buffer. 373 */ 374 void ff_vk_free_buf(AVFilterContext *avctx, FFVkBuffer *buf); 375 376 #endif /* AVFILTER_VULKAN_H */ 377