1 /*
2 * Copyright (c) 2020
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 /**
22 * @file
23 * DNN OpenVINO backend implementation.
24 */
25
26 #include "dnn_backend_openvino.h"
27 #include "dnn_io_proc.h"
28 #include "libavformat/avio.h"
29 #include "libavutil/avassert.h"
30 #include "libavutil/opt.h"
31 #include "libavutil/avstring.h"
32 #include "../internal.h"
33 #include "queue.h"
34 #include "safe_queue.h"
35 #include <c_api/ie_c_api.h>
36
37 typedef struct OVOptions{
38 char *device_type;
39 int nireq;
40 int batch_size;
41 int input_resizable;
42 } OVOptions;
43
44 typedef struct OVContext {
45 const AVClass *class;
46 OVOptions options;
47 } OVContext;
48
49 typedef struct OVModel{
50 OVContext ctx;
51 DNNModel *model;
52 ie_core_t *core;
53 ie_network_t *network;
54 ie_executable_network_t *exe_network;
55 ie_infer_request_t *infer_request;
56
57 /* for async execution */
58 SafeQueue *request_queue; // holds RequestItem
59 Queue *task_queue; // holds TaskItem
60 } OVModel;
61
62 typedef struct TaskItem {
63 OVModel *ov_model;
64 const char *input_name;
65 AVFrame *in_frame;
66 const char *output_name;
67 AVFrame *out_frame;
68 int do_ioproc;
69 int async;
70 int done;
71 } TaskItem;
72
73 typedef struct RequestItem {
74 ie_infer_request_t *infer_request;
75 TaskItem **tasks;
76 int task_count;
77 ie_complete_call_back_t callback;
78 } RequestItem;
79
80 #define APPEND_STRING(generated_string, iterate_string) \
81 generated_string = generated_string ? av_asprintf("%s %s", generated_string, iterate_string) : \
82 av_asprintf("%s", iterate_string);
83
84 #define OFFSET(x) offsetof(OVContext, x)
85 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
86 static const AVOption dnn_openvino_options[] = {
87 { "device", "device to run model", OFFSET(options.device_type), AV_OPT_TYPE_STRING, { .str = "CPU" }, 0, 0, FLAGS },
88 { "nireq", "number of request", OFFSET(options.nireq), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS },
89 { "batch_size", "batch size per request", OFFSET(options.batch_size), AV_OPT_TYPE_INT, { .i64 = 1 }, 1, 1000, FLAGS},
90 { "input_resizable", "can input be resizable or not", OFFSET(options.input_resizable), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
91 { NULL }
92 };
93
94 AVFILTER_DEFINE_CLASS(dnn_openvino);
95
precision_to_datatype(precision_e precision)96 static DNNDataType precision_to_datatype(precision_e precision)
97 {
98 switch (precision)
99 {
100 case FP32:
101 return DNN_FLOAT;
102 case U8:
103 return DNN_UINT8;
104 default:
105 av_assert0(!"not supported yet.");
106 return DNN_FLOAT;
107 }
108 }
109
get_datatype_size(DNNDataType dt)110 static int get_datatype_size(DNNDataType dt)
111 {
112 switch (dt)
113 {
114 case DNN_FLOAT:
115 return sizeof(float);
116 case DNN_UINT8:
117 return sizeof(uint8_t);
118 default:
119 av_assert0(!"not supported yet.");
120 return 1;
121 }
122 }
123
fill_model_input_ov(OVModel * ov_model,RequestItem * request)124 static DNNReturnType fill_model_input_ov(OVModel *ov_model, RequestItem *request)
125 {
126 dimensions_t dims;
127 precision_e precision;
128 ie_blob_buffer_t blob_buffer;
129 OVContext *ctx = &ov_model->ctx;
130 IEStatusCode status;
131 DNNData input;
132 ie_blob_t *input_blob = NULL;
133 TaskItem *task = request->tasks[0];
134
135 status = ie_infer_request_get_blob(request->infer_request, task->input_name, &input_blob);
136 if (status != OK) {
137 av_log(ctx, AV_LOG_ERROR, "Failed to get input blob with name %s\n", task->input_name);
138 return DNN_ERROR;
139 }
140
141 status |= ie_blob_get_dims(input_blob, &dims);
142 status |= ie_blob_get_precision(input_blob, &precision);
143 if (status != OK) {
144 ie_blob_free(&input_blob);
145 av_log(ctx, AV_LOG_ERROR, "Failed to get input blob dims/precision\n");
146 return DNN_ERROR;
147 }
148
149 status = ie_blob_get_buffer(input_blob, &blob_buffer);
150 if (status != OK) {
151 ie_blob_free(&input_blob);
152 av_log(ctx, AV_LOG_ERROR, "Failed to get input blob buffer\n");
153 return DNN_ERROR;
154 }
155
156 input.height = dims.dims[2];
157 input.width = dims.dims[3];
158 input.channels = dims.dims[1];
159 input.data = blob_buffer.buffer;
160 input.dt = precision_to_datatype(precision);
161 // all models in openvino open model zoo use BGR as input,
162 // change to be an option when necessary.
163 input.order = DCO_BGR;
164
165 av_assert0(request->task_count <= dims.dims[0]);
166 for (int i = 0; i < request->task_count; ++i) {
167 task = request->tasks[i];
168 if (task->do_ioproc) {
169 if (ov_model->model->pre_proc != NULL) {
170 ov_model->model->pre_proc(task->in_frame, &input, ov_model->model->filter_ctx);
171 } else {
172 ff_proc_from_frame_to_dnn(task->in_frame, &input, ov_model->model->func_type, ctx);
173 }
174 }
175 input.data = (uint8_t *)input.data
176 + input.width * input.height * input.channels * get_datatype_size(input.dt);
177 }
178 ie_blob_free(&input_blob);
179
180 return DNN_SUCCESS;
181 }
182
infer_completion_callback(void * args)183 static void infer_completion_callback(void *args)
184 {
185 dimensions_t dims;
186 precision_e precision;
187 IEStatusCode status;
188 RequestItem *request = args;
189 TaskItem *task = request->tasks[0];
190 SafeQueue *requestq = task->ov_model->request_queue;
191 ie_blob_t *output_blob = NULL;
192 ie_blob_buffer_t blob_buffer;
193 DNNData output;
194 OVContext *ctx = &task->ov_model->ctx;
195
196 status = ie_infer_request_get_blob(request->infer_request, task->output_name, &output_blob);
197 if (status != OK) {
198 //incorrect output name
199 char *model_output_name = NULL;
200 char *all_output_names = NULL;
201 size_t model_output_count = 0;
202 av_log(ctx, AV_LOG_ERROR, "Failed to get model output data\n");
203 status = ie_network_get_outputs_number(task->ov_model->network, &model_output_count);
204 for (size_t i = 0; i < model_output_count; i++) {
205 status = ie_network_get_output_name(task->ov_model->network, i, &model_output_name);
206 APPEND_STRING(all_output_names, model_output_name)
207 }
208 av_log(ctx, AV_LOG_ERROR,
209 "output \"%s\" may not correct, all output(s) are: \"%s\"\n",
210 task->output_name, all_output_names);
211 return;
212 }
213
214 status = ie_blob_get_buffer(output_blob, &blob_buffer);
215 if (status != OK) {
216 ie_blob_free(&output_blob);
217 av_log(ctx, AV_LOG_ERROR, "Failed to access output memory\n");
218 return;
219 }
220
221 status |= ie_blob_get_dims(output_blob, &dims);
222 status |= ie_blob_get_precision(output_blob, &precision);
223 if (status != OK) {
224 ie_blob_free(&output_blob);
225 av_log(ctx, AV_LOG_ERROR, "Failed to get dims or precision of output\n");
226 return;
227 }
228
229 output.channels = dims.dims[1];
230 output.height = dims.dims[2];
231 output.width = dims.dims[3];
232 output.dt = precision_to_datatype(precision);
233 output.data = blob_buffer.buffer;
234
235 av_assert0(request->task_count <= dims.dims[0]);
236 av_assert0(request->task_count >= 1);
237 for (int i = 0; i < request->task_count; ++i) {
238 task = request->tasks[i];
239 if (task->do_ioproc) {
240 if (task->ov_model->model->post_proc != NULL) {
241 task->ov_model->model->post_proc(task->out_frame, &output, task->ov_model->model->filter_ctx);
242 } else {
243 ff_proc_from_dnn_to_frame(task->out_frame, &output, ctx);
244 }
245 } else {
246 task->out_frame->width = output.width;
247 task->out_frame->height = output.height;
248 }
249 task->done = 1;
250 output.data = (uint8_t *)output.data
251 + output.width * output.height * output.channels * get_datatype_size(output.dt);
252 }
253 ie_blob_free(&output_blob);
254
255 request->task_count = 0;
256
257 if (task->async) {
258 if (ff_safe_queue_push_back(requestq, request) < 0) {
259 av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
260 return;
261 }
262 }
263 }
264
init_model_ov(OVModel * ov_model,const char * input_name,const char * output_name)265 static DNNReturnType init_model_ov(OVModel *ov_model, const char *input_name, const char *output_name)
266 {
267 OVContext *ctx = &ov_model->ctx;
268 IEStatusCode status;
269 ie_available_devices_t a_dev;
270 ie_config_t config = {NULL, NULL, NULL};
271 char *all_dev_names = NULL;
272
273 // batch size
274 if (ctx->options.batch_size <= 0) {
275 ctx->options.batch_size = 1;
276 }
277
278 if (ctx->options.batch_size > 1) {
279 input_shapes_t input_shapes;
280 status = ie_network_get_input_shapes(ov_model->network, &input_shapes);
281 if (status != OK)
282 goto err;
283 for (int i = 0; i < input_shapes.shape_num; i++)
284 input_shapes.shapes[i].shape.dims[0] = ctx->options.batch_size;
285 status = ie_network_reshape(ov_model->network, input_shapes);
286 ie_network_input_shapes_free(&input_shapes);
287 if (status != OK)
288 goto err;
289 }
290
291 // The order of dims in the openvino is fixed and it is always NCHW for 4-D data.
292 // while we pass NHWC data from FFmpeg to openvino
293 status = ie_network_set_input_layout(ov_model->network, input_name, NHWC);
294 if (status != OK) {
295 av_log(ctx, AV_LOG_ERROR, "Failed to set layout as NHWC for input %s\n", input_name);
296 goto err;
297 }
298 status = ie_network_set_output_layout(ov_model->network, output_name, NHWC);
299 if (status != OK) {
300 av_log(ctx, AV_LOG_ERROR, "Failed to set layout as NHWC for output %s\n", output_name);
301 goto err;
302 }
303
304 // all models in openvino open model zoo use BGR with range [0.0f, 255.0f] as input,
305 // we don't have a AVPixelFormat to descibe it, so we'll use AV_PIX_FMT_BGR24 and
306 // ask openvino to do the conversion internally.
307 // the current supported SR model (frame processing) is generated from tensorflow model,
308 // and its input is Y channel as float with range [0.0f, 1.0f], so do not set for this case.
309 // TODO: we need to get a final clear&general solution with all backends/formats considered.
310 if (ov_model->model->func_type != DFT_PROCESS_FRAME) {
311 status = ie_network_set_input_precision(ov_model->network, input_name, U8);
312 if (status != OK) {
313 av_log(ctx, AV_LOG_ERROR, "Failed to set input precision as U8 for %s\n", input_name);
314 goto err;
315 }
316 }
317
318 status = ie_core_load_network(ov_model->core, ov_model->network, ctx->options.device_type, &config, &ov_model->exe_network);
319 if (status != OK) {
320 av_log(ctx, AV_LOG_ERROR, "Failed to load OpenVINO model network\n");
321 status = ie_core_get_available_devices(ov_model->core, &a_dev);
322 if (status != OK) {
323 av_log(ctx, AV_LOG_ERROR, "Failed to get available devices\n");
324 goto err;
325 }
326 for (int i = 0; i < a_dev.num_devices; i++) {
327 APPEND_STRING(all_dev_names, a_dev.devices[i])
328 }
329 av_log(ctx, AV_LOG_ERROR,"device %s may not be supported, all available devices are: \"%s\"\n",
330 ctx->options.device_type, all_dev_names);
331 goto err;
332 }
333
334 // create infer_request for sync execution
335 status = ie_exec_network_create_infer_request(ov_model->exe_network, &ov_model->infer_request);
336 if (status != OK)
337 goto err;
338
339 // create infer_requests for async execution
340 if (ctx->options.nireq <= 0) {
341 // the default value is a rough estimation
342 ctx->options.nireq = av_cpu_count() / 2 + 1;
343 }
344
345 ov_model->request_queue = ff_safe_queue_create();
346 if (!ov_model->request_queue) {
347 goto err;
348 }
349
350 for (int i = 0; i < ctx->options.nireq; i++) {
351 RequestItem *item = av_mallocz(sizeof(*item));
352 if (!item) {
353 goto err;
354 }
355
356 item->callback.completeCallBackFunc = infer_completion_callback;
357 item->callback.args = item;
358 if (ff_safe_queue_push_back(ov_model->request_queue, item) < 0) {
359 av_freep(&item);
360 goto err;
361 }
362
363 status = ie_exec_network_create_infer_request(ov_model->exe_network, &item->infer_request);
364 if (status != OK) {
365 goto err;
366 }
367
368 item->tasks = av_malloc_array(ctx->options.batch_size, sizeof(*item->tasks));
369 if (!item->tasks) {
370 goto err;
371 }
372 item->task_count = 0;
373 }
374
375 ov_model->task_queue = ff_queue_create();
376 if (!ov_model->task_queue) {
377 goto err;
378 }
379
380 return DNN_SUCCESS;
381
382 err:
383 ff_dnn_free_model_ov(&ov_model->model);
384 return DNN_ERROR;
385 }
386
execute_model_ov(RequestItem * request)387 static DNNReturnType execute_model_ov(RequestItem *request)
388 {
389 IEStatusCode status;
390 DNNReturnType ret;
391 TaskItem *task = request->tasks[0];
392 OVContext *ctx = &task->ov_model->ctx;
393
394 if (task->async) {
395 if (request->task_count < ctx->options.batch_size) {
396 if (ff_safe_queue_push_front(task->ov_model->request_queue, request) < 0) {
397 av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
398 return DNN_ERROR;
399 }
400 return DNN_SUCCESS;
401 }
402 ret = fill_model_input_ov(task->ov_model, request);
403 if (ret != DNN_SUCCESS) {
404 return ret;
405 }
406 status = ie_infer_set_completion_callback(request->infer_request, &request->callback);
407 if (status != OK) {
408 av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n");
409 return DNN_ERROR;
410 }
411 status = ie_infer_request_infer_async(request->infer_request);
412 if (status != OK) {
413 av_log(ctx, AV_LOG_ERROR, "Failed to start async inference\n");
414 return DNN_ERROR;
415 }
416 return DNN_SUCCESS;
417 } else {
418 ret = fill_model_input_ov(task->ov_model, request);
419 if (ret != DNN_SUCCESS) {
420 return ret;
421 }
422 status = ie_infer_request_infer(request->infer_request);
423 if (status != OK) {
424 av_log(ctx, AV_LOG_ERROR, "Failed to start synchronous model inference\n");
425 return DNN_ERROR;
426 }
427 infer_completion_callback(request);
428 return task->done ? DNN_SUCCESS : DNN_ERROR;
429 }
430 }
431
get_input_ov(void * model,DNNData * input,const char * input_name)432 static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input_name)
433 {
434 OVModel *ov_model = model;
435 OVContext *ctx = &ov_model->ctx;
436 char *model_input_name = NULL;
437 char *all_input_names = NULL;
438 IEStatusCode status;
439 size_t model_input_count = 0;
440 dimensions_t dims;
441 precision_e precision;
442 int input_resizable = ctx->options.input_resizable;
443
444 status = ie_network_get_inputs_number(ov_model->network, &model_input_count);
445 if (status != OK) {
446 av_log(ctx, AV_LOG_ERROR, "Failed to get input count\n");
447 return DNN_ERROR;
448 }
449
450 for (size_t i = 0; i < model_input_count; i++) {
451 status = ie_network_get_input_name(ov_model->network, i, &model_input_name);
452 if (status != OK) {
453 av_log(ctx, AV_LOG_ERROR, "Failed to get No.%d input's name\n", (int)i);
454 return DNN_ERROR;
455 }
456 if (strcmp(model_input_name, input_name) == 0) {
457 ie_network_name_free(&model_input_name);
458 status |= ie_network_get_input_dims(ov_model->network, input_name, &dims);
459 status |= ie_network_get_input_precision(ov_model->network, input_name, &precision);
460 if (status != OK) {
461 av_log(ctx, AV_LOG_ERROR, "Failed to get No.%d input's dims or precision\n", (int)i);
462 return DNN_ERROR;
463 }
464
465 input->channels = dims.dims[1];
466 input->height = input_resizable ? -1 : dims.dims[2];
467 input->width = input_resizable ? -1 : dims.dims[3];
468 input->dt = precision_to_datatype(precision);
469 return DNN_SUCCESS;
470 } else {
471 //incorrect input name
472 APPEND_STRING(all_input_names, model_input_name)
473 }
474
475 ie_network_name_free(&model_input_name);
476 }
477
478 av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model, all input(s) are: \"%s\"\n", input_name, all_input_names);
479 return DNN_ERROR;
480 }
481
get_output_ov(void * model,const char * input_name,int input_width,int input_height,const char * output_name,int * output_width,int * output_height)482 static DNNReturnType get_output_ov(void *model, const char *input_name, int input_width, int input_height,
483 const char *output_name, int *output_width, int *output_height)
484 {
485 DNNReturnType ret;
486 OVModel *ov_model = model;
487 OVContext *ctx = &ov_model->ctx;
488 TaskItem task;
489 RequestItem request;
490 AVFrame *in_frame = NULL;
491 AVFrame *out_frame = NULL;
492 TaskItem *ptask = &task;
493 IEStatusCode status;
494 input_shapes_t input_shapes;
495
496 if (ctx->options.input_resizable) {
497 status = ie_network_get_input_shapes(ov_model->network, &input_shapes);
498 input_shapes.shapes->shape.dims[2] = input_height;
499 input_shapes.shapes->shape.dims[3] = input_width;
500 status |= ie_network_reshape(ov_model->network, input_shapes);
501 ie_network_input_shapes_free(&input_shapes);
502 if (status != OK) {
503 av_log(ctx, AV_LOG_ERROR, "Failed to reshape input size for %s\n", input_name);
504 return DNN_ERROR;
505 }
506 }
507
508 if (!ov_model->exe_network) {
509 if (init_model_ov(ov_model, input_name, output_name) != DNN_SUCCESS) {
510 av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
511 return DNN_ERROR;
512 }
513 }
514
515 in_frame = av_frame_alloc();
516 if (!in_frame) {
517 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n");
518 return DNN_ERROR;
519 }
520 in_frame->width = input_width;
521 in_frame->height = input_height;
522
523 out_frame = av_frame_alloc();
524 if (!out_frame) {
525 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output frame\n");
526 av_frame_free(&in_frame);
527 return DNN_ERROR;
528 }
529
530 task.done = 0;
531 task.do_ioproc = 0;
532 task.async = 0;
533 task.input_name = input_name;
534 task.in_frame = in_frame;
535 task.output_name = output_name;
536 task.out_frame = out_frame;
537 task.ov_model = ov_model;
538
539 request.infer_request = ov_model->infer_request;
540 request.task_count = 1;
541 request.tasks = &ptask;
542
543 ret = execute_model_ov(&request);
544 *output_width = out_frame->width;
545 *output_height = out_frame->height;
546
547 av_frame_free(&out_frame);
548 av_frame_free(&in_frame);
549 return ret;
550 }
551
ff_dnn_load_model_ov(const char * model_filename,DNNFunctionType func_type,const char * options,AVFilterContext * filter_ctx)552 DNNModel *ff_dnn_load_model_ov(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
553 {
554 DNNModel *model = NULL;
555 OVModel *ov_model = NULL;
556 OVContext *ctx = NULL;
557 IEStatusCode status;
558
559 model = av_mallocz(sizeof(DNNModel));
560 if (!model){
561 return NULL;
562 }
563
564 ov_model = av_mallocz(sizeof(OVModel));
565 if (!ov_model) {
566 av_freep(&model);
567 return NULL;
568 }
569 model->model = ov_model;
570 ov_model->model = model;
571 ov_model->ctx.class = &dnn_openvino_class;
572 ctx = &ov_model->ctx;
573
574 //parse options
575 av_opt_set_defaults(ctx);
576 if (av_opt_set_from_string(ctx, options, NULL, "=", "&") < 0) {
577 av_log(ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options);
578 goto err;
579 }
580
581 status = ie_core_create("", &ov_model->core);
582 if (status != OK)
583 goto err;
584
585 status = ie_core_read_network(ov_model->core, model_filename, NULL, &ov_model->network);
586 if (status != OK) {
587 ie_version_t ver;
588 ver = ie_c_api_version();
589 av_log(ctx, AV_LOG_ERROR, "Failed to read the network from model file %s,\n"
590 "Please check if the model version matches the runtime OpenVINO %s\n",
591 model_filename, ver.api_version);
592 ie_version_free(&ver);
593 goto err;
594 }
595
596 model->get_input = &get_input_ov;
597 model->get_output = &get_output_ov;
598 model->options = options;
599 model->filter_ctx = filter_ctx;
600 model->func_type = func_type;
601
602 return model;
603
604 err:
605 ff_dnn_free_model_ov(&model);
606 return NULL;
607 }
608
ff_dnn_execute_model_ov(const DNNModel * model,const char * input_name,AVFrame * in_frame,const char ** output_names,uint32_t nb_output,AVFrame * out_frame)609 DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame,
610 const char **output_names, uint32_t nb_output, AVFrame *out_frame)
611 {
612 OVModel *ov_model = model->model;
613 OVContext *ctx = &ov_model->ctx;
614 TaskItem task;
615 RequestItem request;
616 TaskItem *ptask = &task;
617
618 if (!in_frame) {
619 av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
620 return DNN_ERROR;
621 }
622
623 if (!out_frame && model->func_type == DFT_PROCESS_FRAME) {
624 av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
625 return DNN_ERROR;
626 }
627
628 if (nb_output != 1) {
629 // currently, the filter does not need multiple outputs,
630 // so we just pending the support until we really need it.
631 avpriv_report_missing_feature(ctx, "multiple outputs");
632 return DNN_ERROR;
633 }
634
635 if (ctx->options.batch_size > 1) {
636 avpriv_report_missing_feature(ctx, "batch mode for sync execution");
637 return DNN_ERROR;
638 }
639
640 if (!ov_model->exe_network) {
641 if (init_model_ov(ov_model, input_name, output_names[0]) != DNN_SUCCESS) {
642 av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
643 return DNN_ERROR;
644 }
645 }
646
647 task.done = 0;
648 task.do_ioproc = 1;
649 task.async = 0;
650 task.input_name = input_name;
651 task.in_frame = in_frame;
652 task.output_name = output_names[0];
653 task.out_frame = out_frame;
654 task.ov_model = ov_model;
655
656 request.infer_request = ov_model->infer_request;
657 request.task_count = 1;
658 request.tasks = &ptask;
659
660 return execute_model_ov(&request);
661 }
662
ff_dnn_execute_model_async_ov(const DNNModel * model,const char * input_name,AVFrame * in_frame,const char ** output_names,uint32_t nb_output,AVFrame * out_frame)663 DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame,
664 const char **output_names, uint32_t nb_output, AVFrame *out_frame)
665 {
666 OVModel *ov_model = model->model;
667 OVContext *ctx = &ov_model->ctx;
668 RequestItem *request;
669 TaskItem *task;
670
671 if (!in_frame) {
672 av_log(ctx, AV_LOG_ERROR, "in frame is NULL when async execute model.\n");
673 return DNN_ERROR;
674 }
675
676 if (!out_frame && model->func_type == DFT_PROCESS_FRAME) {
677 av_log(ctx, AV_LOG_ERROR, "out frame is NULL when async execute model.\n");
678 return DNN_ERROR;
679 }
680
681 if (!ov_model->exe_network) {
682 if (init_model_ov(ov_model, input_name, output_names[0]) != DNN_SUCCESS) {
683 av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
684 return DNN_ERROR;
685 }
686 }
687
688 task = av_malloc(sizeof(*task));
689 if (!task) {
690 av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
691 return DNN_ERROR;
692 }
693
694 task->done = 0;
695 task->do_ioproc = 1;
696 task->async = 1;
697 task->input_name = input_name;
698 task->in_frame = in_frame;
699 task->output_name = output_names[0];
700 task->out_frame = out_frame;
701 task->ov_model = ov_model;
702 if (ff_queue_push_back(ov_model->task_queue, task) < 0) {
703 av_freep(&task);
704 av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
705 return DNN_ERROR;
706 }
707
708 request = ff_safe_queue_pop_front(ov_model->request_queue);
709 if (!request) {
710 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
711 return DNN_ERROR;
712 }
713
714 request->tasks[request->task_count++] = task;
715 return execute_model_ov(request);
716 }
717
ff_dnn_get_async_result_ov(const DNNModel * model,AVFrame ** in,AVFrame ** out)718 DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out)
719 {
720 OVModel *ov_model = model->model;
721 TaskItem *task = ff_queue_peek_front(ov_model->task_queue);
722
723 if (!task) {
724 return DAST_EMPTY_QUEUE;
725 }
726
727 if (!task->done) {
728 return DAST_NOT_READY;
729 }
730
731 *in = task->in_frame;
732 *out = task->out_frame;
733 ff_queue_pop_front(ov_model->task_queue);
734 av_freep(&task);
735
736 return DAST_SUCCESS;
737 }
738
ff_dnn_flush_ov(const DNNModel * model)739 DNNReturnType ff_dnn_flush_ov(const DNNModel *model)
740 {
741 OVModel *ov_model = model->model;
742 OVContext *ctx = &ov_model->ctx;
743 RequestItem *request;
744 IEStatusCode status;
745 DNNReturnType ret;
746
747 request = ff_safe_queue_pop_front(ov_model->request_queue);
748 if (!request) {
749 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
750 return DNN_ERROR;
751 }
752
753 if (request->task_count == 0) {
754 // no pending task need to flush
755 if (ff_safe_queue_push_back(ov_model->request_queue, request) < 0) {
756 av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
757 return DNN_ERROR;
758 }
759 return DNN_SUCCESS;
760 }
761
762 ret = fill_model_input_ov(ov_model, request);
763 if (ret != DNN_SUCCESS) {
764 av_log(ctx, AV_LOG_ERROR, "Failed to fill model input.\n");
765 return ret;
766 }
767 status = ie_infer_set_completion_callback(request->infer_request, &request->callback);
768 if (status != OK) {
769 av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n");
770 return DNN_ERROR;
771 }
772 status = ie_infer_request_infer_async(request->infer_request);
773 if (status != OK) {
774 av_log(ctx, AV_LOG_ERROR, "Failed to start async inference\n");
775 return DNN_ERROR;
776 }
777
778 return DNN_SUCCESS;
779 }
780
ff_dnn_free_model_ov(DNNModel ** model)781 void ff_dnn_free_model_ov(DNNModel **model)
782 {
783 if (*model){
784 OVModel *ov_model = (*model)->model;
785 while (ff_safe_queue_size(ov_model->request_queue) != 0) {
786 RequestItem *item = ff_safe_queue_pop_front(ov_model->request_queue);
787 if (item && item->infer_request) {
788 ie_infer_request_free(&item->infer_request);
789 }
790 av_freep(&item->tasks);
791 av_freep(&item);
792 }
793 ff_safe_queue_destroy(ov_model->request_queue);
794
795 while (ff_queue_size(ov_model->task_queue) != 0) {
796 TaskItem *item = ff_queue_pop_front(ov_model->task_queue);
797 av_frame_free(&item->in_frame);
798 av_frame_free(&item->out_frame);
799 av_freep(&item);
800 }
801 ff_queue_destroy(ov_model->task_queue);
802
803 if (ov_model->infer_request)
804 ie_infer_request_free(&ov_model->infer_request);
805 if (ov_model->exe_network)
806 ie_exec_network_free(&ov_model->exe_network);
807 if (ov_model->network)
808 ie_network_free(&ov_model->network);
809 if (ov_model->core)
810 ie_core_free(&ov_model->core);
811 av_freep(&ov_model);
812 av_freep(model);
813 }
814 }
815