1 /*
2 * Copyright (c) 2018 Sergey Lavrushkin
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #include "libavutil/avassert.h"
22 #include "dnn_backend_native_layer_conv2d.h"
23
24 #define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x)))
25
dnn_load_layer_conv2d(Layer * layer,AVIOContext * model_file_context,int file_size,int operands_num)26 int dnn_load_layer_conv2d(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
27 {
28 ConvolutionalParams *conv_params;
29 int kernel_size;
30 int dnn_size = 0;
31 conv_params = av_malloc(sizeof(*conv_params));
32 if (!conv_params)
33 return 0;
34
35 conv_params->dilation = (int32_t)avio_rl32(model_file_context);
36 conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
37 conv_params->activation = (int32_t)avio_rl32(model_file_context);
38 conv_params->input_num = (int32_t)avio_rl32(model_file_context);
39 conv_params->output_num = (int32_t)avio_rl32(model_file_context);
40 conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
41 conv_params->has_bias = (int32_t)avio_rl32(model_file_context);
42 dnn_size += 28;
43
44 kernel_size = conv_params->input_num * conv_params->output_num *
45 conv_params->kernel_size * conv_params->kernel_size;
46 dnn_size += kernel_size * 4;
47 if (conv_params->has_bias)
48 dnn_size += conv_params->output_num * 4;
49
50 if (dnn_size > file_size || conv_params->input_num <= 0 ||
51 conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
52 av_freep(&conv_params);
53 return 0;
54 }
55
56 conv_params->kernel = av_malloc(kernel_size * sizeof(float));
57 if (!conv_params->kernel) {
58 av_freep(&conv_params);
59 return 0;
60 }
61 for (int i = 0; i < kernel_size; ++i) {
62 conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context));
63 }
64
65 conv_params->biases = NULL;
66 if (conv_params->has_bias) {
67 conv_params->biases = av_malloc(conv_params->output_num * sizeof(float));
68 if (!conv_params->biases){
69 av_freep(&conv_params->kernel);
70 av_freep(&conv_params);
71 return 0;
72 }
73 for (int i = 0; i < conv_params->output_num; ++i){
74 conv_params->biases[i] = av_int2float(avio_rl32(model_file_context));
75 }
76 }
77
78 layer->params = conv_params;
79
80 layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
81 layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
82 dnn_size += 8;
83
84 if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
85 return 0;
86 }
87
88 return dnn_size;
89 }
90
dnn_execute_layer_conv2d(DnnOperand * operands,const int32_t * input_operand_indexes,int32_t output_operand_index,const void * parameters)91 int dnn_execute_layer_conv2d(DnnOperand *operands, const int32_t *input_operand_indexes,
92 int32_t output_operand_index, const void *parameters)
93 {
94 float *output;
95 int32_t input_operand_index = input_operand_indexes[0];
96 int number = operands[input_operand_index].dims[0];
97 int height = operands[input_operand_index].dims[1];
98 int width = operands[input_operand_index].dims[2];
99 int channel = operands[input_operand_index].dims[3];
100 const float *input = operands[input_operand_index].data;
101 const ConvolutionalParams *conv_params = (const ConvolutionalParams *)parameters;
102
103 int radius = conv_params->kernel_size >> 1;
104 int src_linesize = width * conv_params->input_num;
105 int filter_linesize = conv_params->kernel_size * conv_params->input_num;
106 int filter_size = conv_params->kernel_size * filter_linesize;
107 int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0;
108
109 DnnOperand *output_operand = &operands[output_operand_index];
110 output_operand->dims[0] = number;
111 output_operand->dims[1] = height - pad_size * 2;
112 output_operand->dims[2] = width - pad_size * 2;
113 output_operand->dims[3] = conv_params->output_num;
114 output_operand->data_type = operands[input_operand_index].data_type;
115 output_operand->length = calculate_operand_data_length(output_operand);
116 if (output_operand->length <= 0)
117 return -1;
118 output_operand->data = av_realloc(output_operand->data, output_operand->length);
119 if (!output_operand->data)
120 return -1;
121 output = output_operand->data;
122
123 av_assert0(channel == conv_params->input_num);
124
125 for (int y = pad_size; y < height - pad_size; ++y) {
126 for (int x = pad_size; x < width - pad_size; ++x) {
127 for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) {
128 if (conv_params->has_bias)
129 output[n_filter] = conv_params->biases[n_filter];
130 else
131 output[n_filter] = 0.f;
132
133 for (int ch = 0; ch < conv_params->input_num; ++ch) {
134 for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) {
135 for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) {
136 float input_pel;
137 if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) {
138 int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height);
139 int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width);
140 input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
141 } else {
142 int y_pos = y + (kernel_y - radius) * conv_params->dilation;
143 int x_pos = x + (kernel_x - radius) * conv_params->dilation;
144 input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 :
145 input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
146 }
147
148
149 output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
150 kernel_x * conv_params->input_num + ch];
151 }
152 }
153 }
154 switch (conv_params->activation){
155 case RELU:
156 output[n_filter] = FFMAX(output[n_filter], 0.0);
157 break;
158 case TANH:
159 output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f;
160 break;
161 case SIGMOID:
162 output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter]));
163 break;
164 case NONE:
165 break;
166 case LEAKY_RELU:
167 output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0);
168 }
169 }
170 output += conv_params->output_num;
171 }
172 }
173 return 0;
174 }
175