1 /*
2 * Copyright (c) 2018 Sergey Lavrushkin
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 /**
22 * @file
23 * DNN native backend implementation.
24 */
25
26 #include "dnn_backend_native.h"
27 #include "libavutil/avassert.h"
28 #include "dnn_backend_native_layer_conv2d.h"
29 #include "dnn_backend_native_layers.h"
30
get_input_native(void * model,DNNData * input,const char * input_name)31 static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
32 {
33 ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
34
35 for (int i = 0; i < network->operands_num; ++i) {
36 DnnOperand *oprd = &network->operands[i];
37 if (strcmp(oprd->name, input_name) == 0) {
38 if (oprd->type != DOT_INPUT)
39 return DNN_ERROR;
40 input->dt = oprd->data_type;
41 av_assert0(oprd->dims[0] == 1);
42 input->height = oprd->dims[1];
43 input->width = oprd->dims[2];
44 input->channels = oprd->dims[3];
45 return DNN_SUCCESS;
46 }
47 }
48
49 // do not find the input operand
50 return DNN_ERROR;
51 }
52
set_input_output_native(void * model,DNNData * input,const char * input_name,const char ** output_names,uint32_t nb_output)53 static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
54 {
55 ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
56 DnnOperand *oprd = NULL;
57
58 if (network->layers_num <= 0 || network->operands_num <= 0)
59 return DNN_ERROR;
60
61 /* inputs */
62 for (int i = 0; i < network->operands_num; ++i) {
63 oprd = &network->operands[i];
64 if (strcmp(oprd->name, input_name) == 0) {
65 if (oprd->type != DOT_INPUT)
66 return DNN_ERROR;
67 break;
68 }
69 oprd = NULL;
70 }
71
72 if (!oprd)
73 return DNN_ERROR;
74
75 oprd->dims[0] = 1;
76 oprd->dims[1] = input->height;
77 oprd->dims[2] = input->width;
78 oprd->dims[3] = input->channels;
79
80 av_freep(&oprd->data);
81 oprd->length = calculate_operand_data_length(oprd);
82 if (oprd->length <= 0)
83 return DNN_ERROR;
84 oprd->data = av_malloc(oprd->length);
85 if (!oprd->data)
86 return DNN_ERROR;
87
88 input->data = oprd->data;
89
90 /* outputs */
91 network->nb_output = 0;
92 av_freep(&network->output_indexes);
93 network->output_indexes = av_mallocz_array(nb_output, sizeof(*network->output_indexes));
94 if (!network->output_indexes)
95 return DNN_ERROR;
96
97 for (uint32_t i = 0; i < nb_output; ++i) {
98 const char *output_name = output_names[i];
99 for (int j = 0; j < network->operands_num; ++j) {
100 oprd = &network->operands[j];
101 if (strcmp(oprd->name, output_name) == 0) {
102 network->output_indexes[network->nb_output++] = j;
103 break;
104 }
105 }
106 }
107
108 if (network->nb_output != nb_output)
109 return DNN_ERROR;
110
111 return DNN_SUCCESS;
112 }
113
114 // Loads model and its parameters that are stored in a binary file with following structure:
115 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
116 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
117 // For DEPTH_TO_SPACE layer: block_size
ff_dnn_load_model_native(const char * model_filename)118 DNNModel *ff_dnn_load_model_native(const char *model_filename)
119 {
120 DNNModel *model = NULL;
121 char header_expected[] = "FFMPEGDNNNATIVE";
122 char *buf;
123 size_t size;
124 int version, header_size, major_version_expected = 1;
125 ConvolutionalNetwork *network = NULL;
126 AVIOContext *model_file_context;
127 int file_size, dnn_size, parsed_size;
128 int32_t layer;
129 DNNLayerType layer_type;
130
131 if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
132 return NULL;
133 }
134 file_size = avio_size(model_file_context);
135
136 model = av_mallocz(sizeof(DNNModel));
137 if (!model){
138 goto fail;
139 }
140
141 /**
142 * check file header with string and version
143 */
144 size = sizeof(header_expected);
145 buf = av_malloc(size);
146 if (!buf) {
147 goto fail;
148 }
149
150 // size - 1 to skip the ending '\0' which is not saved in file
151 avio_get_str(model_file_context, size - 1, buf, size);
152 dnn_size = size - 1;
153 if (strncmp(buf, header_expected, size) != 0) {
154 av_freep(&buf);
155 goto fail;
156 }
157 av_freep(&buf);
158
159 version = (int32_t)avio_rl32(model_file_context);
160 dnn_size += 4;
161 if (version != major_version_expected) {
162 goto fail;
163 }
164
165 // currently no need to check minor version
166 version = (int32_t)avio_rl32(model_file_context);
167 dnn_size += 4;
168 header_size = dnn_size;
169
170 network = av_mallocz(sizeof(ConvolutionalNetwork));
171 if (!network){
172 goto fail;
173 }
174 model->model = (void *)network;
175
176 avio_seek(model_file_context, file_size - 8, SEEK_SET);
177 network->layers_num = (int32_t)avio_rl32(model_file_context);
178 network->operands_num = (int32_t)avio_rl32(model_file_context);
179 dnn_size += 8;
180 avio_seek(model_file_context, header_size, SEEK_SET);
181
182 network->layers = av_mallocz(network->layers_num * sizeof(Layer));
183 if (!network->layers){
184 goto fail;
185 }
186
187 network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand));
188 if (!network->operands){
189 goto fail;
190 }
191
192 for (layer = 0; layer < network->layers_num; ++layer){
193 layer_type = (int32_t)avio_rl32(model_file_context);
194 dnn_size += 4;
195
196 if (layer_type >= DLT_COUNT) {
197 goto fail;
198 }
199
200 network->layers[layer].type = layer_type;
201 parsed_size = layer_funcs[layer_type].pf_load(&network->layers[layer], model_file_context, file_size, network->operands_num);
202 if (!parsed_size) {
203 goto fail;
204 }
205 dnn_size += parsed_size;
206 }
207
208 for (int32_t i = 0; i < network->operands_num; ++i){
209 DnnOperand *oprd;
210 int32_t name_len;
211 int32_t operand_index = (int32_t)avio_rl32(model_file_context);
212 dnn_size += 4;
213
214 if (operand_index >= network->operands_num) {
215 goto fail;
216 }
217
218 oprd = &network->operands[operand_index];
219 name_len = (int32_t)avio_rl32(model_file_context);
220 dnn_size += 4;
221
222 avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
223 dnn_size += name_len;
224
225 oprd->type = (int32_t)avio_rl32(model_file_context);
226 dnn_size += 4;
227
228 oprd->data_type = (int32_t)avio_rl32(model_file_context);
229 dnn_size += 4;
230
231 for (int32_t dim = 0; dim < 4; ++dim) {
232 oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
233 dnn_size += 4;
234 }
235
236 oprd->isNHWC = 1;
237 }
238
239 avio_closep(&model_file_context);
240
241 if (dnn_size != file_size){
242 ff_dnn_free_model_native(&model);
243 return NULL;
244 }
245
246 model->set_input_output = &set_input_output_native;
247 model->get_input = &get_input_native;
248
249 return model;
250
251 fail:
252 ff_dnn_free_model_native(&model);
253 avio_closep(&model_file_context);
254 return NULL;
255 }
256
ff_dnn_execute_model_native(const DNNModel * model,DNNData * outputs,uint32_t nb_output)257 DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
258 {
259 ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
260 int32_t layer;
261 uint32_t nb = FFMIN(nb_output, network->nb_output);
262
263 if (network->layers_num <= 0 || network->operands_num <= 0)
264 return DNN_ERROR;
265 if (!network->operands[0].data)
266 return DNN_ERROR;
267
268 for (layer = 0; layer < network->layers_num; ++layer){
269 DNNLayerType layer_type = network->layers[layer].type;
270 layer_funcs[layer_type].pf_exec(network->operands,
271 network->layers[layer].input_operand_indexes,
272 network->layers[layer].output_operand_index,
273 network->layers[layer].params);
274 }
275
276 for (uint32_t i = 0; i < nb; ++i) {
277 DnnOperand *oprd = &network->operands[network->output_indexes[i]];
278 outputs[i].data = oprd->data;
279 outputs[i].height = oprd->dims[1];
280 outputs[i].width = oprd->dims[2];
281 outputs[i].channels = oprd->dims[3];
282 outputs[i].dt = oprd->data_type;
283 }
284
285 return DNN_SUCCESS;
286 }
287
calculate_operand_dims_count(const DnnOperand * oprd)288 int32_t calculate_operand_dims_count(const DnnOperand *oprd)
289 {
290 int32_t result = 1;
291 for (int i = 0; i < 4; ++i)
292 result *= oprd->dims[i];
293
294 return result;
295 }
296
calculate_operand_data_length(const DnnOperand * oprd)297 int32_t calculate_operand_data_length(const DnnOperand* oprd)
298 {
299 // currently, we just support DNN_FLOAT
300 uint64_t len = sizeof(float);
301 for (int i = 0; i < 4; i++) {
302 len *= oprd->dims[i];
303 if (len > INT32_MAX)
304 return 0;
305 }
306 return len;
307 }
308
ff_dnn_free_model_native(DNNModel ** model)309 void ff_dnn_free_model_native(DNNModel **model)
310 {
311 ConvolutionalNetwork *network;
312 ConvolutionalParams *conv_params;
313 int32_t layer;
314
315 if (*model)
316 {
317 if ((*model)->model) {
318 network = (ConvolutionalNetwork *)(*model)->model;
319 if (network->layers) {
320 for (layer = 0; layer < network->layers_num; ++layer){
321 if (network->layers[layer].type == DLT_CONV2D){
322 conv_params = (ConvolutionalParams *)network->layers[layer].params;
323 av_freep(&conv_params->kernel);
324 av_freep(&conv_params->biases);
325 }
326 av_freep(&network->layers[layer].params);
327 }
328 av_freep(&network->layers);
329 }
330
331 if (network->operands) {
332 for (uint32_t operand = 0; operand < network->operands_num; ++operand)
333 av_freep(&network->operands[operand].data);
334 av_freep(&network->operands);
335 }
336
337 av_freep(&network->output_indexes);
338 av_freep(&network);
339 }
340 av_freep(model);
341 }
342 }
343