1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "minddata/dataset/kernels/image/lite_cv/image_process.h"
18
19 #include <cfloat>
20 #include <climits>
21 #include <cmath>
22 #include <cstring>
23 #include <limits>
24 #include <random>
25 #include <utility>
26 #include <vector>
27
28 #ifdef ENABLE_NEON
29 #include <arm_neon.h>
30 #endif
31
32 namespace mindspore {
33 namespace dataset {
34 constexpr uint32_t kR2Gray = 9798;
35 constexpr uint32_t kG2Gray = 19235;
36 constexpr uint32_t kB2Gray = 3735;
37 constexpr int32_t kGrayShift = 15;
38 constexpr int32_t kGrayShiftDelta = 1 << (kGrayShift - 1);
39 constexpr int32_t kYScale = 0x0101;
40 constexpr int32_t kU2B = -128;
41 constexpr int32_t kU2G = 25;
42 constexpr int32_t kV2R = -102;
43 constexpr int32_t kV2G = 52;
44 constexpr int32_t kY2G = 18997;
45 constexpr int32_t kY2GB = -1160;
46 constexpr int32_t kB2B = kU2B * 128 + kY2GB;
47 constexpr int32_t kB2G = kU2G * 128 + kV2G * 128 + kY2GB;
48 constexpr int32_t kB2R = kV2R * 128 + kY2GB;
49
Equal(const float & a,const float & b)50 static bool Equal(const float &a, const float &b) { return std::fabs(a - b) < 1e-6; }
51
InitBilinearWeight(int * data_ptr,int16_t * weight_ptr,double scale,int dst_length,int src_length,int a)52 static inline void InitBilinearWeight(int *data_ptr, int16_t *weight_ptr, double scale, int dst_length, int src_length,
53 int a) {
54 const int RESIZE_SCALE = 1 << 11;
55 if (data_ptr == nullptr || weight_ptr == nullptr) {
56 return;
57 }
58
59 int *data_start_ptr = data_ptr;
60 int16_t *weight_start_ptr = weight_ptr;
61
62 for (unsigned int i = 0; i < dst_length; i++) {
63 float src_f_x = static_cast<float>((i + 0.5) * scale - 0.5);
64 int src_u_x = static_cast<int>(floor(src_f_x));
65 src_f_x -= src_u_x;
66 if (src_u_x < 0) {
67 src_u_x = 0;
68 src_f_x = 0.0f;
69 }
70 if (src_u_x >= src_length - 1) {
71 src_u_x = src_length - 2;
72 src_f_x = 1.0f;
73 }
74 data_start_ptr[i] = src_u_x * a;
75 int16_t t0 = INT16_CAST((1.0f - src_f_x) * RESIZE_SCALE);
76 int16_t t1 = INT16_CAST(src_f_x * RESIZE_SCALE);
77
78 weight_start_ptr[i * 2] = t0;
79 weight_start_ptr[i * 2 + 1] = t1;
80 }
81 }
82
ResizeBilinear3C(const unsigned char * src,int src_width,int src_height,unsigned char * dst,int dst_width,int dst_height)83 static void ResizeBilinear3C(const unsigned char *src, int src_width, int src_height, unsigned char *dst, int dst_width,
84 int dst_height) {
85 double scale_width = static_cast<double>(src_width) / dst_width;
86 double scale_height = static_cast<double>(src_height) / dst_height;
87
88 if (dst_height >= (INT_MAX / 2 - dst_width)) {
89 return;
90 }
91 int *data_buf = new int[2 * dst_width + 2 * dst_height];
92
93 int *x_offset = data_buf;
94 int *y_offset = data_buf + dst_width;
95
96 int16_t *x_weight = reinterpret_cast<int16_t *>(data_buf + dst_width + dst_height);
97 int16_t *y_weight = reinterpret_cast<int16_t *>(x_weight + dst_width);
98
99 InitBilinearWeight(x_offset, x_weight, scale_width, dst_width, src_width, 3);
100 InitBilinearWeight(y_offset, y_weight, scale_height, dst_height, src_height, 1);
101
102 LiteMat x_tmp_buf0(dst_width * 3 + 1, LDataType::UINT16);
103 LiteMat x_tmp_buf1(dst_width * 3 + 1, LDataType::UINT16);
104 int16_t *row0_ptr = reinterpret_cast<int16_t *>(x_tmp_buf0.data_ptr_);
105 int16_t *row1_ptr = reinterpret_cast<int16_t *>(x_tmp_buf1.data_ptr_);
106
107 int prev_height = -2;
108
109 for (int y = 0; y < dst_height; y++) {
110 int y_span = y_offset[y];
111
112 if (y_span == prev_height) {
113 } else if (y_span == prev_height + 1) {
114 int16_t *tmp = row0_ptr;
115 row0_ptr = row1_ptr;
116 row1_ptr = tmp;
117 const unsigned char *src_start = src + 3 * src_width * (y_span + 1);
118 const int16_t *x_weight_p = x_weight;
119 int16_t *row1_ptr1 = row1_ptr;
120 for (int x = 0; x < dst_width; x++) {
121 const unsigned char *src_start_p = src_start + x_offset[x];
122 row1_ptr1[0] = (src_start_p[0] * x_weight_p[0] + src_start_p[3] * x_weight_p[1]) >> 4;
123 row1_ptr1[1] = (src_start_p[1] * x_weight_p[0] + src_start_p[4] * x_weight_p[1]) >> 4;
124 row1_ptr1[2] = (src_start_p[2] * x_weight_p[0] + src_start_p[5] * x_weight_p[1]) >> 4;
125 x_weight_p += 2;
126 row1_ptr1 += 3;
127 }
128 } else {
129 const unsigned char *src0 = src + 3 * src_width * (y_span);
130 const unsigned char *src1 = src + 3 * src_width * (y_span + 1);
131
132 const int16_t *x_weight_ptr = x_weight;
133 int16_t *row0_ptr0 = row0_ptr;
134 int16_t *row1_ptr1 = row1_ptr;
135 for (int x = 0; x < dst_width; x++) {
136 const unsigned char *src0_ptr = src0 + x_offset[x];
137 const unsigned char *src1_ptr = src1 + x_offset[x];
138
139 for (int c = 0; c < 3; c++) {
140 row0_ptr0[c] = (src0_ptr[c] * x_weight_ptr[0] + src0_ptr[c + 3] * x_weight_ptr[1]) >> 4;
141 row1_ptr1[c] = (src1_ptr[c] * x_weight_ptr[0] + src1_ptr[c + 3] * x_weight_ptr[1]) >> 4;
142 }
143
144 x_weight_ptr += 2;
145 row0_ptr0 += 3;
146 row1_ptr1 += 3;
147 }
148 }
149 prev_height = y_span;
150
151 int16_t *row0_ptr0 = row0_ptr;
152 int16_t *row1_ptr1 = row1_ptr;
153 unsigned char *dst_ptr = dst + dst_width * 3 * (y);
154
155 for (int k = 0; k < dst_width * 3; k++) {
156 int16_t t0 = (int16_t)((y_weight[0] * (int16_t)(*row0_ptr0++)) >> 16);
157 int16_t t1 = (int16_t)((y_weight[1] * (int16_t)(*row1_ptr1++)) >> 16);
158 *dst_ptr++ = static_cast<unsigned char>((t0 + t1 + 2) >> 2);
159 }
160 y_weight += 2;
161 }
162 delete[] data_buf;
163 }
164
ResizeBilinear1C(const unsigned char * src,int src_width,int src_height,unsigned char * dst,int dst_width,int dst_height)165 static void ResizeBilinear1C(const unsigned char *src, int src_width, int src_height, unsigned char *dst, int dst_width,
166 int dst_height) {
167 double scale_width = static_cast<double>(src_width) / dst_width;
168 double scale_height = static_cast<double>(src_height) / dst_height;
169
170 if (dst_height >= (INT_MAX / 2 - dst_width)) {
171 return;
172 }
173 int *data_buf = new int[2 * dst_width + 2 * dst_height];
174
175 int *x_offset = data_buf;
176 int *y_offset = data_buf + dst_width;
177
178 int16_t *x_weight = reinterpret_cast<int16_t *>(data_buf + dst_width + dst_height);
179 int16_t *y_weight = reinterpret_cast<int16_t *>(x_weight + dst_width);
180
181 InitBilinearWeight(x_offset, x_weight, scale_width, dst_width, src_width, 1);
182 InitBilinearWeight(y_offset, y_weight, scale_height, dst_height, src_height, 1);
183
184 LiteMat x_tmp_buf0(dst_width, LDataType::UINT16);
185 LiteMat x_tmp_buf1(dst_width, LDataType::UINT16);
186 int16_t *row0_ptr = reinterpret_cast<int16_t *>(x_tmp_buf0.data_ptr_);
187 int16_t *row1_ptr = reinterpret_cast<int16_t *>(x_tmp_buf1.data_ptr_);
188
189 int prev_height = -2;
190
191 for (int y = 0; y < dst_height; y++) {
192 int y_span = y_offset[y];
193
194 if (y_span == prev_height) {
195 } else if (y_span == prev_height + 1) {
196 int16_t *tmp = row0_ptr;
197 row0_ptr = row1_ptr;
198 row1_ptr = tmp;
199 const unsigned char *src_start = src + src_width * (y_span + 1);
200 const int16_t *x_weight_p = x_weight;
201 int16_t *row1_ptr1 = row1_ptr;
202 for (int x = 0; x < dst_width; x++) {
203 const unsigned char *src_start_p = src_start + x_offset[x];
204 if ((src_start_p + 3 - src) >= (src_width * src_height)) {
205 continue;
206 }
207 row1_ptr1[x] = (src_start_p[0] * x_weight_p[0] + src_start_p[3] * x_weight_p[1]) >> 4;
208 x_weight_p += 2;
209 }
210 } else {
211 const unsigned char *src0 = src + src_width * (y_span);
212 const unsigned char *src1 = src + src_width * (y_span + 1);
213
214 const int16_t *x_weight_ptr = x_weight;
215 int16_t *row0_ptr0 = row0_ptr;
216 int16_t *row1_ptr1 = row1_ptr;
217 for (int x = 0; x < dst_width; x++) {
218 const unsigned char *src0_ptr = src0 + x_offset[x];
219 const unsigned char *src1_ptr = src1 + x_offset[x];
220
221 row0_ptr0[x] = (src0_ptr[0] * x_weight_ptr[0] + src0_ptr[3] * x_weight_ptr[1]) >> 4;
222 row1_ptr1[x] = (src1_ptr[0] * x_weight_ptr[0] + src1_ptr[3] * x_weight_ptr[1]) >> 4;
223
224 x_weight_ptr += 2;
225 }
226 }
227 prev_height = y_span;
228
229 int16_t *row0_ptr0 = row0_ptr;
230 int16_t *row1_ptr1 = row1_ptr;
231 unsigned char *dst_ptr = dst + dst_width * (y);
232
233 for (int k = 0; k < dst_width; k++) {
234 int16_t t0 = (int16_t)((y_weight[0] * (int16_t)(*row0_ptr0++)) >> 16);
235 int16_t t1 = (int16_t)((y_weight[1] * (int16_t)(*row1_ptr1++)) >> 16);
236 *dst_ptr++ = static_cast<unsigned char>((t0 + t1 + 2) >> 2);
237 }
238
239 y_weight += 2;
240 }
241 delete[] data_buf;
242 }
243
clip(float value)244 static inline uint8_t clip(float value) {
245 int int_val = roundf(value);
246 return std::max<int32_t>(std::numeric_limits<uint8_t>::min(),
247 std::min<int32_t>(std::numeric_limits<uint8_t>::max(), int_val));
248 }
249
250 template <typename T1, typename T2>
Conv2DImplement(const LiteMat & src,const LiteMat & kernel,T2 * dst,LDataType dst_type,PaddBorderType pad_type)251 static bool Conv2DImplement(const LiteMat &src, const LiteMat &kernel, T2 *dst, LDataType dst_type,
252 PaddBorderType pad_type) {
253 int border_x = static_cast<int>(kernel.width_ / 2);
254 int border_y = static_cast<int>(kernel.height_ / 2);
255
256 LiteMat pad_mat;
257
258 if ((border_x > INT_MAX / 2) || (src.width_ > INT_MAX - 2 * border_x)) {
259 return false;
260 }
261 if ((border_y > INT_MAX / 2) || (src.height_ > INT_MAX - 2 * border_y)) {
262 return false;
263 }
264
265 pad_mat.Init(src.width_ + 2 * border_x, src.height_ + 2 * border_y, src.channel_, src.data_type_);
266
267 if (!Pad(src, pad_mat, border_y, border_y, border_x, border_x, pad_type)) {
268 return false;
269 }
270
271 const T1 *pad_ptr = pad_mat;
272 const float *kernel_ptr = kernel;
273
274 int pad_step = pad_mat.width_ * pad_mat.channel_;
275 int dst_step = src.width_ * src.channel_;
276
277 if (src.channel_ == 1) {
278 for (int y = border_y; y < pad_mat.height_ - border_y; y++) {
279 for (int x = border_x; x < pad_mat.width_ - border_x; x++) {
280 float conv_sum = 0;
281 for (int i = -border_y; i < -border_y + kernel.height_; i++) {
282 for (int j = -border_x; j < -border_x + kernel.width_; j++) {
283 conv_sum += pad_ptr[(y + i) * pad_step + (x + j) * pad_mat.channel_] *
284 kernel_ptr[(i + border_y) * kernel.width_ + (j + border_x)];
285 }
286 }
287 if (dst_type == LDataType::UINT8) {
288 dst[(y - border_y) * dst_step + (x - border_x) * src.channel_] = clip(conv_sum);
289 } else {
290 dst[(y - border_y) * dst_step + (x - border_x) * src.channel_] = conv_sum;
291 }
292 }
293 }
294 } else if (src.channel_ == 3) {
295 for (int y = border_y; y < pad_mat.height_ - border_y; y++) {
296 for (int x = border_x; x < pad_mat.width_ - border_x; x++) {
297 float conv_sum_b = 0;
298 float conv_sum_g = 0;
299 float conv_sum_r = 0;
300 for (int i = -border_y; i < -border_y + kernel.height_; i++) {
301 for (int j = -border_x; j < -border_x + kernel.width_; j++) {
302 conv_sum_b += pad_ptr[(y + i) * pad_step + (x + j) * pad_mat.channel_] *
303 kernel_ptr[(i + border_y) * kernel.width_ + (j + border_x)];
304 conv_sum_g += pad_ptr[(y + i) * pad_step + (x + j) * pad_mat.channel_ + 1] *
305 kernel_ptr[(i + border_y) * kernel.width_ + (j + border_x)];
306 conv_sum_r += pad_ptr[(y + i) * pad_step + (x + j) * pad_mat.channel_ + 2] *
307 kernel_ptr[(i + border_y) * kernel.width_ + (j + border_x)];
308 }
309 }
310 if (dst_type == LDataType::UINT8) {
311 dst[(y - border_y) * dst_step + (x - border_x) * src.channel_] = clip(conv_sum_b);
312 dst[(y - border_y) * dst_step + (x - border_x) * src.channel_ + 1] = clip(conv_sum_g);
313 dst[(y - border_y) * dst_step + (x - border_x) * src.channel_ + 2] = clip(conv_sum_r);
314 } else {
315 dst[(y - border_y) * dst_step + (x - border_x) * src.channel_] = conv_sum_b;
316 dst[(y - border_y) * dst_step + (x - border_x) * src.channel_ + 1] = conv_sum_g;
317 dst[(y - border_y) * dst_step + (x - border_x) * src.channel_ + 2] = conv_sum_r;
318 }
319 }
320 }
321 } else {
322 return false;
323 }
324 return true;
325 }
326
Conv2D(const LiteMat & src,const LiteMat & kernel,LiteMat & dst,LDataType dst_type,PaddBorderType pad_type)327 bool Conv2D(const LiteMat &src, const LiteMat &kernel, LiteMat &dst, LDataType dst_type, PaddBorderType pad_type) {
328 if (src.IsEmpty() || kernel.IsEmpty()) {
329 return false;
330 }
331 if ((dst_type != LDataType::UINT8 && dst_type != LDataType::FLOAT32) || kernel.data_type_ != LDataType::FLOAT32) {
332 return false;
333 }
334 if (dst.IsEmpty() || dst.width_ != src.width_ || dst.height_ != src.height_ || dst.channel_ != src.channel_ ||
335 dst.data_type_ != dst_type) {
336 dst.Init(src.width_, src.height_, src.channel_, dst_type);
337 }
338
339 if (src.data_type_ == LDataType::UINT8 && dst.data_type_ == LDataType::UINT8) {
340 return Conv2DImplement<uint8_t, uint8_t>(src, kernel, dst, dst_type, pad_type);
341 } else if (src.data_type_ == LDataType::UINT8 && dst.data_type_ == LDataType::FLOAT32) {
342 return Conv2DImplement<uint8_t, float>(src, kernel, dst, dst_type, pad_type);
343 } else if (src.data_type_ == LDataType::FLOAT32 && dst.data_type_ == LDataType::UINT8) {
344 return Conv2DImplement<float, uint8_t>(src, kernel, dst, dst_type, pad_type);
345 } else if (src.data_type_ == LDataType::FLOAT32 && dst.data_type_ == LDataType::FLOAT32) {
346 return Conv2DImplement<float, float>(src, kernel, dst, dst_type, pad_type);
347 } else {
348 return false;
349 }
350 }
351
ConvRowCol(const LiteMat & src,const LiteMat & kx,const LiteMat & ky,LiteMat & dst,LDataType dst_type,PaddBorderType pad_type)352 bool ConvRowCol(const LiteMat &src, const LiteMat &kx, const LiteMat &ky, LiteMat &dst, LDataType dst_type,
353 PaddBorderType pad_type) {
354 if (src.IsEmpty() || kx.IsEmpty() || ky.IsEmpty()) {
355 return false;
356 }
357 if (dst_type != LDataType::UINT8 && dst_type != LDataType::FLOAT32) {
358 return false;
359 }
360 if (dst.IsEmpty() || dst.width_ != src.width_ || dst.height_ != src.height_ || dst.channel_ != src.channel_ ||
361 dst.data_type_ != dst_type) {
362 dst.Init(src.width_, src.height_, src.channel_, dst_type);
363 }
364
365 LiteMat mid;
366 bool ret = Conv2D(src, kx, mid, LDataType::FLOAT32, pad_type) && Conv2D(mid, ky, dst, dst_type, pad_type);
367 return ret;
368 }
369
ResizeBilinear(const LiteMat & src,LiteMat & dst,int dst_w,int dst_h)370 bool ResizeBilinear(const LiteMat &src, LiteMat &dst, int dst_w, int dst_h) {
371 if (dst_h <= 0 || dst_w <= 0) {
372 return false;
373 }
374 if (src.data_type_ != LDataType::UINT8) {
375 return false;
376 }
377 if (src.channel_ != 3 && src.channel_ != 1) {
378 return false;
379 }
380 if (dst.IsEmpty()) {
381 (void)dst.Init(dst_w, dst_h, src.channel_, LDataType::UINT8);
382 } else if (dst.height_ != dst_h || dst.width_ != dst_w || dst.channel_ != src.channel_) {
383 return false;
384 } else if (dst.data_type_ != LDataType::UINT8) {
385 return false;
386 } else {
387 }
388
389 if (src.channel_ == 3) {
390 const unsigned char *src_start_p = src;
391 unsigned char *dst_start_p = dst;
392 (void)ResizeBilinear3C(src_start_p, src.width_, src.height_, dst_start_p, dst_w, dst_h);
393 } else { // channel == 1
394 const unsigned char *src_start_p = src;
395 unsigned char *dst_start_p = dst;
396 (void)ResizeBilinear1C(src_start_p, src.width_, src.height_, dst_start_p, dst_w, dst_h);
397 }
398 return true;
399 }
400
ConvertBGR(const unsigned char * data,LDataType data_type,int w,int h,LiteMat & mat)401 static bool ConvertBGR(const unsigned char *data, LDataType data_type, int w, int h, LiteMat &mat) {
402 if (data_type == LDataType::UINT8) {
403 mat.Init(w, h, 3, LDataType::UINT8);
404 unsigned char *dst_ptr = mat;
405 // mindspore lite version, there is no securec lib
406 (void)memcpy(dst_ptr, data, w * h * 3 * sizeof(unsigned char));
407 } else {
408 return false;
409 }
410 return true;
411 }
412
ConvertRGBAToBGR(const unsigned char * data,LDataType data_type,int w,int h,LiteMat & mat)413 static bool ConvertRGBAToBGR(const unsigned char *data, LDataType data_type, int w, int h, LiteMat &mat) {
414 if (data_type == LDataType::UINT8) {
415 mat.Init(w, h, 3, LDataType::UINT8);
416 unsigned char *ptr = mat;
417 const unsigned char *data_ptr = data;
418 for (int y = 0; y < h; y++) {
419 for (int x = 0; x < w; x++) {
420 ptr[0] = data_ptr[2];
421 ptr[1] = data_ptr[1];
422 ptr[2] = data_ptr[0];
423 ptr += 3;
424 data_ptr += 4;
425 }
426 }
427 } else {
428 return false;
429 }
430 return true;
431 }
432
ConvertRGBAToRGB(const unsigned char * data,LDataType data_type,int w,int h,LiteMat & mat)433 static bool ConvertRGBAToRGB(const unsigned char *data, LDataType data_type, int w, int h, LiteMat &mat) {
434 if (data_type == LDataType::UINT8) {
435 mat.Init(w, h, 3, LDataType::UINT8);
436 unsigned char *ptr = mat;
437 const unsigned char *data_ptr = data;
438 for (int y = 0; y < h; y++) {
439 for (int x = 0; x < w; x++) {
440 ptr[0] = data_ptr[0];
441 ptr[1] = data_ptr[1];
442 ptr[2] = data_ptr[2];
443 ptr += 3;
444 data_ptr += 4;
445 }
446 }
447 } else {
448 return false;
449 }
450 return true;
451 }
452
ConvertYUV420SPToBGR(const uint8_t * data,LDataType data_type,bool flag,int w,int h,LiteMat & mat)453 static bool ConvertYUV420SPToBGR(const uint8_t *data, LDataType data_type, bool flag, int w, int h, LiteMat &mat) {
454 if (data == nullptr || w <= 0 || h <= 0) {
455 return false;
456 }
457 if (data_type == LDataType::UINT8) {
458 mat.Init(w, h, 3, LDataType::UINT8);
459 const uint8_t *y_ptr = data;
460 const uint8_t *uv_ptr = y_ptr + w * h;
461 uint8_t *bgr_ptr = mat;
462 const int bgr_stride = 3 * w;
463
464 for (uint64_t y = 0; y < h; ++y) {
465 uint8_t *bgr_buf = bgr_ptr;
466 const uint8_t *uv_buf = uv_ptr;
467 const uint8_t *y_buf = y_ptr;
468 uint8_t u;
469 uint8_t v;
470 for (int x = 0; x < w - 1; x += 2) {
471 if (flag) {
472 // NV21
473 u = uv_buf[1];
474 v = uv_buf[0];
475 } else {
476 // NV12
477 u = uv_buf[0];
478 v = uv_buf[1];
479 }
480 uint32_t tmp_y = (uint32_t)(y_buf[0] * kYScale * kY2G) >> 16;
481 // b
482 bgr_buf[0] = std::clamp((int32_t)(-(u * kU2B) + tmp_y + kB2B) >> 6, 0, 255);
483 // g
484 bgr_buf[1] = std::clamp((int32_t)(-(u * kU2G + v * kV2G) + tmp_y + kB2G) >> 6, 0, 255);
485 // r
486 bgr_buf[2] = std::clamp((int32_t)(-(v * kV2R) + tmp_y + kB2R) >> 6, 0, 255);
487
488 tmp_y = (uint32_t)(y_buf[1] * kYScale * kY2G) >> 16;
489 bgr_buf[3] = std::clamp((int32_t)(-(u * kU2B) + tmp_y + kB2B) >> 6, 0, 255);
490 bgr_buf[4] = std::clamp((int32_t)(-(u * kU2G + v * kV2G) + tmp_y + kB2G) >> 6, 0, 255);
491 bgr_buf[5] = std::clamp((int32_t)(-(v * kV2R) + tmp_y + kB2R) >> 6, 0, 255);
492
493 y_buf += 2;
494 uv_buf += 2;
495 bgr_buf += 6;
496 }
497 if (w & 1) {
498 if (flag) {
499 // NV21
500 u = uv_buf[1];
501 v = uv_buf[0];
502 } else {
503 // NV12
504 u = uv_buf[0];
505 v = uv_buf[1];
506 }
507 uint32_t tmp_y = (uint32_t)(y_buf[0] * kYScale * kY2G) >> 16;
508 bgr_buf[0] = std::clamp((int32_t)(-(u * kU2B) + tmp_y + kB2B) >> 6, 0, 255);
509 bgr_buf[1] = std::clamp((int32_t)(-(u * kU2G + v * kV2G) + tmp_y + kB2G) >> 6, 0, 255);
510 bgr_buf[2] = std::clamp((int32_t)(-(v * kV2R) + tmp_y + kB2R) >> 6, 0, 255);
511 }
512
513 bgr_ptr += bgr_stride;
514 y_ptr += w;
515 if (y & 1) {
516 uv_ptr += w;
517 }
518 }
519 }
520 return true;
521 }
522
ConvertRGBAToGRAY(const unsigned char * data,LDataType data_type,int w,int h,LiteMat & mat)523 static bool ConvertRGBAToGRAY(const unsigned char *data, LDataType data_type, int w, int h, LiteMat &mat) {
524 if (data_type == LDataType::UINT8) {
525 mat.Init(w, h, 1, LDataType::UINT8);
526 if (mat.IsEmpty()) {
527 return false;
528 }
529 unsigned char *ptr = mat;
530 const unsigned char *data_ptr = data;
531 for (int y = 0; y < h; y++) {
532 for (int x = 0; x < w; x++) {
533 *ptr = (data_ptr[2] * kB2Gray + data_ptr[1] * kG2Gray + data_ptr[0] * kR2Gray + kGrayShiftDelta) >> kGrayShift;
534 ptr++;
535 data_ptr += 4;
536 }
537 }
538 } else {
539 return false;
540 }
541 return true;
542 }
543
InitFromPixel(const unsigned char * data,LPixelType pixel_type,LDataType data_type,int w,int h,LiteMat & m)544 bool InitFromPixel(const unsigned char *data, LPixelType pixel_type, LDataType data_type, int w, int h, LiteMat &m) {
545 if (data == nullptr) {
546 return false;
547 }
548 if (w <= 0 || h <= 0) {
549 return false;
550 }
551
552 if (data_type != LDataType::UINT8) {
553 return false;
554 }
555 if (pixel_type == LPixelType::RGBA2BGR) {
556 return ConvertRGBAToBGR(data, data_type, w, h, m);
557 } else if (pixel_type == LPixelType::RGBA2GRAY) {
558 return ConvertRGBAToGRAY(data, data_type, w, h, m);
559 } else if (pixel_type == LPixelType::RGBA2RGB) {
560 return ConvertRGBAToRGB(data, data_type, w, h, m);
561 } else if (pixel_type == LPixelType::NV212BGR) {
562 return ConvertYUV420SPToBGR(data, data_type, true, w, h, m);
563 } else if (pixel_type == LPixelType::NV122BGR) {
564 return ConvertYUV420SPToBGR(data, data_type, false, w, h, m);
565 } else if (pixel_type == LPixelType::BGR) {
566 return ConvertBGR(data, data_type, w, h, m);
567 } else if (pixel_type == LPixelType::RGB) {
568 return ConvertBGR(data, data_type, w, h, m);
569 } else {
570 return false;
571 }
572 return true;
573 }
574
ConvertTo(const LiteMat & src,LiteMat & dst,double scale)575 bool ConvertTo(const LiteMat &src, LiteMat &dst, double scale) {
576 if (src.data_type_ != LDataType::UINT8) {
577 return false;
578 }
579
580 if (scale < 0.0 || scale > 100) {
581 return false;
582 }
583
584 if (dst.IsEmpty()) {
585 dst.Init(src.width_, src.height_, src.channel_, LDataType::FLOAT32);
586 } else if (src.width_ != dst.width_ || src.height_ != dst.height_ || src.channel_ != dst.channel_ ||
587 dst.data_type_ != LDataType::FLOAT32) {
588 return false;
589 }
590
591 const uint8_t *src_ptr = (const uint8_t *)src;
592 float *dst_ptr = reinterpret_cast<float *>(dst.data_ptr_);
593 int64_t total_size = src.height_ * src.width_ * src.channel_;
594 int64_t x = 0;
595 #ifdef ENABLE_NEON
596 float32x4_t v_scale = vdupq_n_f32(static_cast<float>(scale));
597 float32x4_t v_c = vdupq_n_f32(0.0f);
598 const int64_t step = 16;
599 for (; x <= total_size - step; x += step) {
600 uint8x16_t v_src = vld1q_u8(src_ptr + x);
601 uint8x16_t v_dst;
602
603 uint16x8_t v_l_16x8 = vmovl_u8(vget_low_u8(v_src));
604 uint16x8_t v_h_16x8 = vmovl_u8(vget_high_u8(v_src));
605
606 float32x4_t v_ll_f32x4 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_l_16x8)));
607 float32x4_t v_lh_f32x4 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_l_16x8)));
608 float32x4_t v_hl_f32x4 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(v_h_16x8)));
609 float32x4_t v_hh_f32x4 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(v_h_16x8)));
610
611 #if defined(__aarch64__) || defined(_M_ARM64)
612 v_ll_f32x4 = vfmaq_f32(v_c, v_ll_f32x4, v_scale);
613 v_lh_f32x4 = vfmaq_f32(v_c, v_lh_f32x4, v_scale);
614 v_hl_f32x4 = vfmaq_f32(v_c, v_hl_f32x4, v_scale);
615 v_hh_f32x4 = vfmaq_f32(v_c, v_hh_f32x4, v_scale);
616 #else
617 v_ll_f32x4 = vmlaq_f32(v_c, v_ll_f32x4, v_scale);
618 v_lh_f32x4 = vmlaq_f32(v_c, v_lh_f32x4, v_scale);
619 v_hl_f32x4 = vmlaq_f32(v_c, v_hl_f32x4, v_scale);
620 v_hh_f32x4 = vmlaq_f32(v_c, v_hh_f32x4, v_scale);
621 #endif
622
623 vst1q_f32(dst_ptr + x, v_ll_f32x4);
624 vst1q_f32(dst_ptr + x + 4, v_lh_f32x4);
625 vst1q_f32(dst_ptr + x + 8, v_hl_f32x4);
626 vst1q_f32(dst_ptr + x + 12, v_hh_f32x4);
627 }
628 #endif
629 for (; x < total_size; x++) {
630 dst_ptr[x] = static_cast<float>(src_ptr[x] * scale);
631 }
632 return true;
633 }
634
635 template <typename T>
CropInternal(const LiteMat & src,LiteMat & dst,int x,int y,int w,int h)636 static bool CropInternal(const LiteMat &src, LiteMat &dst, int x, int y, int w, int h) {
637 int dst_h = h;
638 int dst_w = w;
639 int dst_c = src.channel_;
640 if (dst.IsEmpty()) {
641 dst.Init(dst_w, dst_h, dst_c, src.data_type_);
642 } else if (dst.height_ != h || dst.width_ != w || dst.channel_ != src.channel_) {
643 return false;
644 } else if (dst.data_type_ != src.data_type_) {
645 return false;
646 }
647 const T *src_start_p = src;
648 T *dst_start_p = dst;
649 for (int i_h = 0; i_h < dst_h; i_h++) {
650 const T *src_index_p = src_start_p + (y + i_h) * src.width_ * dst_c + x * dst_c;
651 T *dst_index_p = dst_start_p + i_h * dst_w * dst_c;
652 // mindspore lite version, there is no securec lib
653 (void)memcpy(dst_index_p, src_index_p, dst_w * dst_c * sizeof(T));
654 }
655 return true;
656 }
657
Crop(const LiteMat & src,LiteMat & dst,int x,int y,int w,int h)658 bool Crop(const LiteMat &src, LiteMat &dst, int x, int y, int w, int h) {
659 if (x < 0 || y < 0 || w <= 0 || h <= 0) {
660 return false;
661 }
662 if (y > src.height_ - h || x > src.width_ - w) {
663 return false;
664 }
665
666 if (src.data_type_ == LDataType::UINT8) {
667 return CropInternal<uint8_t>(src, dst, x, y, w, h);
668 } else if (src.data_type_ == LDataType::FLOAT32) {
669 return CropInternal<float>(src, dst, x, y, w, h);
670 } else {
671 return false;
672 }
673 return true;
674 }
675
CheckZero(const std::vector<float> & vs)676 static bool CheckZero(const std::vector<float> &vs) {
677 return std::any_of(vs.begin(), vs.end(), [](const float &v) { return Equal(v, 0.0f); });
678 }
679
CheckZero(const std::vector<size_t> & vs)680 static bool CheckZero(const std::vector<size_t> &vs) {
681 return std::any_of(vs.begin(), vs.end(), [](const float &v) { return v == 0; });
682 }
683
CheckMeanAndStd(const LiteMat & src,LiteMat & dst,int channel,const std::vector<float> & mean,const std::vector<float> & std)684 static bool CheckMeanAndStd(const LiteMat &src, LiteMat &dst, int channel, const std::vector<float> &mean,
685 const std::vector<float> &std) {
686 if (mean.empty() && std.empty()) {
687 return false;
688 }
689 if (src.data_type_ != LDataType::FLOAT32) {
690 return false;
691 }
692 if (mean.size() > 0) {
693 if (mean.size() != channel) {
694 return false;
695 }
696 }
697 if (std.size() > 0) {
698 if (CheckZero(std)) {
699 return false;
700 }
701 if (std.size() != channel) {
702 return false;
703 }
704 }
705 if (dst.IsEmpty()) {
706 dst.Init(src.width_, src.height_, src.channel_, LDataType::FLOAT32);
707 } else if (dst.height_ != src.height_ || dst.width_ != src.width_ || dst.channel_ != src.channel_) {
708 return false;
709 } else if (dst.data_type_ != LDataType::FLOAT32) {
710 return false;
711 }
712 return true;
713 }
714
SubStractMeanNormalize(const LiteMat & src,LiteMat & dst,const std::vector<float> & mean,const std::vector<float> & std)715 bool SubStractMeanNormalize(const LiteMat &src, LiteMat &dst, const std::vector<float> &mean,
716 const std::vector<float> &std) {
717 if (!CheckMeanAndStd(src, dst, src.channel_, mean, std)) {
718 return false;
719 }
720
721 const float *src_start_p = src;
722 float *dst_start_p = dst;
723 if ((!mean.empty()) && std.empty()) {
724 for (int h = 0; h < src.height_; h++) {
725 for (int w = 0; w < src.width_; w++) {
726 uint32_t src_start = (h * src.width_ + w) * src.channel_;
727 for (int c = 0; c < src.channel_; c++) {
728 uint32_t index = src_start + c;
729 dst_start_p[index] = src_start_p[index] - mean[c];
730 }
731 }
732 }
733 } else if (mean.empty() && (!std.empty())) {
734 for (int h = 0; h < src.height_; h++) {
735 for (int w = 0; w < src.width_; w++) {
736 uint32_t src_start = (h * src.width_ + w) * src.channel_;
737 for (int c = 0; c < src.channel_; c++) {
738 uint32_t index = src_start + c;
739 dst_start_p[index] = src_start_p[index] / std[c];
740 }
741 }
742 }
743 } else if ((!mean.empty()) && (!std.empty())) {
744 for (int h = 0; h < src.height_; h++) {
745 for (int w = 0; w < src.width_; w++) {
746 uint32_t src_start = (h * src.width_ + w) * src.channel_;
747 for (int c = 0; c < src.channel_; c++) {
748 uint32_t index = src_start + c;
749 dst_start_p[index] = (src_start_p[index] / std[c]) - mean[c];
750 }
751 }
752 }
753 } else {
754 return false;
755 }
756 return true;
757 }
758
759 template <typename T>
PadWithConstant(const LiteMat & src,LiteMat & dst,const int top,const int bottom,const int left,const int right,const PaddBorderType pad_type,uint8_t fill_b_or_gray,uint8_t fill_g,uint8_t fill_r)760 static void PadWithConstant(const LiteMat &src, LiteMat &dst, const int top, const int bottom, const int left,
761 const int right, const PaddBorderType pad_type, uint8_t fill_b_or_gray, uint8_t fill_g,
762 uint8_t fill_r) {
763 std::vector<uint8_t> row_buffer(dst.width_ * dst.channel_ * dst.elem_size_);
764 T *const_ptr = reinterpret_cast<T *>(row_buffer.data());
765 int src_step = src.width_ * src.channel_ * src.elem_size_;
766 int dst_step = dst.width_ * dst.channel_ * dst.elem_size_;
767 if (dst.channel_ == 1) {
768 for (int i = 0; i < dst_step; i++) {
769 const_ptr[i] = fill_b_or_gray;
770 }
771 } else if (dst.channel_ == 3) {
772 for (int i = 0; i < dst.width_; i++) {
773 const_ptr[i * dst.channel_] = fill_b_or_gray;
774 const_ptr[i * dst.channel_ + 1] = fill_g;
775 const_ptr[i * dst.channel_ + 2] = fill_r;
776 }
777 }
778
779 uint8_t *dst_ptr = reinterpret_cast<uint8_t *>(dst.data_ptr_);
780 uint8_t *src_ptr = reinterpret_cast<uint8_t *>(src.data_ptr_);
781 for (int i = 0; i < top; i++) {
782 // mindspore lite version, there is no securec lib
783 memcpy(dst_ptr + i * dst_step, const_ptr, dst_step);
784 }
785
786 int left_size = left * dst.channel_ * dst.elem_size_;
787 int right_size = right * dst.channel_ * dst.elem_size_;
788 uint8_t *dst_raw_data = dst_ptr + top * dst_step + left_size;
789 for (int i = 0; i < src.height_; i++, dst_raw_data += dst_step, src_ptr += src_step) {
790 // mindspore lite version, there is no securec lib
791 memcpy(dst_raw_data, src_ptr, src_step);
792 memcpy(dst_raw_data - left_size, const_ptr, left_size);
793 memcpy(dst_raw_data + src_step, const_ptr, right_size);
794 }
795
796 for (int i = dst.height_ - bottom; i < dst.height_; i++) {
797 // mindspore lite version, there is no securec lib
798 memcpy(dst_ptr + i * dst_step, const_ptr, dst_step);
799 }
800 }
801
PadFromPos(int p,int len,PaddBorderType pad_type)802 static int PadFromPos(int p, int len, PaddBorderType pad_type) {
803 if (p >= 0 && p < len) {
804 return p;
805 }
806 if (pad_type == PaddBorderType::PADD_BORDER_REPLICATE) {
807 return p < 0 ? 0 : len - 1;
808 } else {
809 // calculate the position of pixel in reflect mode like edcb|abcdef|edcb
810 return p < 0 ? -p : 2 * len - p - 2;
811 }
812 }
813
814 template <typename T>
PadImplement(const LiteMat & src,LiteMat & dst,const int top,const int bottom,const int left,const int right,const PaddBorderType pad_type)815 static void PadImplement(const LiteMat &src, LiteMat &dst, const int top, const int bottom, const int left,
816 const int right, const PaddBorderType pad_type) {
817 int src_step = src.width_ * src.channel_;
818 int dst_step = dst.width_ * dst.channel_;
819
820 uint8_t *src_data_ptr = reinterpret_cast<uint8_t *>(src.data_ptr_);
821 uint8_t *dst_data_ptr = reinterpret_cast<uint8_t *>(dst.data_ptr_);
822 for (int i = 0; i < src.height_; i++) {
823 // mindspore lite version, there is no securec lib
824 memcpy(dst_data_ptr + (i + top) * dst.steps_[0] + left * dst.steps_[1], src_data_ptr + i * src.steps_[0],
825 src.steps_[0]);
826 }
827
828 const T *src_ptr = src;
829 T *dst_ptr = dst;
830 for (int y = 0; y < dst.height_; y++) {
831 for (int x = 0; x < dst.width_; x++) {
832 if (y < top || y >= dst.height_ - bottom || x < left || x >= dst.width_ - right) {
833 int src_y = PadFromPos(y - top, src.height_, pad_type);
834 int src_x = PadFromPos(x - left, src.width_, pad_type);
835 for (int cn = 0; cn < dst.channel_; cn++) {
836 dst_ptr[y * dst_step + x * dst.channel_ + cn] = src_ptr[src_y * src_step + src_x * src.channel_ + cn];
837 }
838 }
839 }
840 }
841 }
842
843 template <typename T>
ExtractChannelImpl(const T * src_ptr,T * dst_ptr,int height,int width,int channel,int col)844 void ExtractChannelImpl(const T *src_ptr, T *dst_ptr, int height, int width, int channel, int col) {
845 int total = height * width;
846 int i = 0;
847 int src_idx = col;
848 for (; i < total; i++, src_idx += channel) {
849 dst_ptr[i] = src_ptr[src_idx];
850 }
851 }
852
ExtractChannel(LiteMat & src,LiteMat & dst,int col)853 bool ExtractChannel(LiteMat &src, LiteMat &dst, int col) {
854 if (src.IsEmpty() || col < 0 || col > src.channel_ - 1) {
855 return false;
856 }
857
858 if (src.data_type_ == LDataType::FLOAT32 || src.data_type_ == LDataType::UINT8) {
859 if (dst.IsEmpty() || dst.width_ != src.width_ || dst.height_ != src.height_ || dst.channel_ != 1 ||
860 dst.data_type_ != src.data_type_) {
861 dst.Init(src.width_, src.height_, 1, src.data_type_);
862 }
863 }
864
865 if (src.data_type_ == LDataType::FLOAT32) {
866 ExtractChannelImpl<float>(src, dst, src.height_, src.width_, src.channel_, col);
867 } else if (src.data_type_ == LDataType::UINT8) {
868 ExtractChannelImpl<uint8_t>(src, dst, src.height_, src.width_, src.channel_, col);
869 } else {
870 return false;
871 }
872 return true;
873 }
874
Split(const LiteMat & src,std::vector<LiteMat> & mv)875 bool Split(const LiteMat &src, std::vector<LiteMat> &mv) {
876 if (src.data_type_ == LDataType::FLOAT32) {
877 const float *src_start_p = src;
878 for (int c = 0; c < src.channel_; c++) {
879 LiteMat dst;
880 (void)dst.Init(src.width_, src.height_, 1, src.data_type_);
881 float *dst_start_p = dst;
882 for (int h = 0; h < src.height_; h++) {
883 uint32_t src_start = h * src.width_ * src.channel_;
884 uint32_t dst_start = h * dst.width_;
885 for (int w = 0; w < src.width_; w++) {
886 uint32_t src_index = src_start + w * src.channel_ + c;
887 uint32_t dst_index = dst_start + w;
888 dst_start_p[dst_index] = src_start_p[src_index];
889 }
890 }
891 mv.emplace_back(dst);
892 }
893 return true;
894 } else if (src.data_type_ == LDataType::UINT8) {
895 const uint8_t *src_start_p = src;
896 for (int c = 0; c < src.channel_; c++) {
897 LiteMat dst;
898 (void)dst.Init(src.width_, src.height_, 1, src.data_type_);
899 uint8_t *dst_start_p = dst;
900 for (int h = 0; h < src.height_; h++) {
901 uint32_t src_start = h * src.width_ * src.channel_;
902 uint32_t dst_start = h * dst.width_;
903 for (int w = 0; w < src.width_; w++) {
904 uint32_t src_index = src_start + w * src.channel_ + c;
905 uint32_t dst_index = dst_start + w;
906 dst_start_p[dst_index] = src_start_p[src_index];
907 }
908 }
909 mv.emplace_back(dst);
910 }
911 return true;
912 } else {
913 return false;
914 }
915 return false;
916 }
917
918 template <typename T>
MergeImpl(const std::vector<LiteMat> & mv,T * dst_ptr,int height,int width,int channel)919 inline void MergeImpl(const std::vector<LiteMat> &mv, T *dst_ptr, int height, int width, int channel) {
920 T *mv_ptr[4];
921 int area = height * width;
922 for (int c = 0; c < channel; c++) {
923 mv_ptr[c] = reinterpret_cast<T *>(mv[c].data_ptr_);
924 }
925 for (int i = 0; i < area; i++) {
926 for (int c = 0; c < channel; c++) {
927 dst_ptr[c] = *mv_ptr[c];
928 mv_ptr[c]++;
929 }
930 dst_ptr += channel;
931 }
932 }
933
Merge(const std::vector<LiteMat> & mv,LiteMat & dst)934 bool Merge(const std::vector<LiteMat> &mv, LiteMat &dst) {
935 if (mv.size() != 1 && mv.size() != 3 && mv.size() != 4) return false;
936
937 int width = mv[0].width_;
938 int height = mv[0].height_;
939 int channel = mv.size();
940 LDataType data_type = mv[0].data_type_;
941
942 // The arrays in list must be single-channel
943 if (std::any_of(mv.begin(), mv.end(), [](const LiteMat &m) { return m.channel_ != 1; })) {
944 return false;
945 }
946
947 for (int i = 1; i < mv.size(); i++) {
948 if (width != mv[i].width_ || height != mv[i].height_ || data_type != mv[i].data_type_) {
949 return false;
950 }
951 }
952
953 if (dst.IsEmpty() || dst.width_ != width || dst.height_ != height || dst.channel_ != channel ||
954 dst.data_type_ != data_type) {
955 dst.Init(width, height, channel, data_type);
956 }
957
958 if (dst.data_type_ == LDataType::FLOAT32) {
959 MergeImpl<float>(mv, dst, height, width, channel);
960 } else if (dst.data_type_ == LDataType::UINT8) {
961 MergeImpl<uint8_t>(mv, dst, height, width, channel);
962 } else {
963 return false;
964 }
965 return true;
966 }
967
Pad(const LiteMat & src,LiteMat & dst,int top,int bottom,int left,int right,PaddBorderType pad_type,uint8_t fill_b_or_gray,uint8_t fill_g,uint8_t fill_r)968 bool Pad(const LiteMat &src, LiteMat &dst, int top, int bottom, int left, int right, PaddBorderType pad_type,
969 uint8_t fill_b_or_gray, uint8_t fill_g, uint8_t fill_r) {
970 RETURN_FALSE_IF_LITEMAT_EMPTY(src);
971 if (top < 0 || bottom < 0 || left < 0 || right < 0) {
972 return false;
973 }
974 if (src.width_ > std::numeric_limits<int>::max() - left ||
975 src.width_ + left > std::numeric_limits<int>::max() - right) {
976 return false;
977 }
978 if (src.height_ > std::numeric_limits<int>::max() - top ||
979 src.height_ + top > std::numeric_limits<int>::max() - bottom) {
980 return false;
981 }
982 int dst_width = src.width_ + left + right;
983 int dst_height = src.height_ + top + bottom;
984 if (dst.IsEmpty()) {
985 dst.Init(dst_width, dst_height, src.channel_, src.data_type_);
986 RETURN_FALSE_IF_LITEMAT_EMPTY(dst);
987 } else if (dst.width_ != dst_width || dst.height_ != dst_height || src.channel_ != dst.channel_) {
988 return false;
989 } else if (src.data_type_ != dst.data_type_) {
990 return false;
991 }
992 if (pad_type == PADD_BORDER_CONSTANT && src.data_type_ == LDataType::FLOAT32) {
993 PadWithConstant<float>(src, dst, top, bottom, left, right, pad_type, fill_b_or_gray, fill_g, fill_r);
994 } else if (pad_type == PADD_BORDER_CONSTANT && src.data_type_ == LDataType::UINT8) {
995 PadWithConstant<uint8_t>(src, dst, top, bottom, left, right, pad_type, fill_b_or_gray, fill_g, fill_r);
996 } else if (src.data_type_ == LDataType::FLOAT32) {
997 PadImplement<float>(src, dst, top, bottom, left, right, pad_type);
998 } else if (src.data_type_ == LDataType::UINT8) {
999 PadImplement<uint8_t>(src, dst, top, bottom, left, right, pad_type);
1000 } else {
1001 return false;
1002 }
1003 return true;
1004 }
1005
GetDefaultBoxes(const BoxesConfig config)1006 std::vector<std::vector<float>> GetDefaultBoxes(const BoxesConfig config) {
1007 size_t size = config.num_default.size();
1008 if (size <= 1 || config.feature_size.size() != size || config.steps.size() != size ||
1009 config.aspect_rations.size() != size) {
1010 return {};
1011 }
1012 if (config.max_scale < config.min_scale) {
1013 return {};
1014 }
1015 std::vector<float> fk;
1016 float num = static_cast<float>(config.img_shape[0]);
1017 for (int i = 0; i < config.steps.size(); i++) {
1018 if (config.steps[i] == 0) {
1019 return {};
1020 }
1021 fk.push_back(num / config.steps[i]);
1022 }
1023 float scale_rate = (config.max_scale - config.min_scale) / (config.num_default.size() - 1);
1024 std::vector<float> scales(config.num_default.size());
1025 for (int i = 0; i < scales.size(); i++) {
1026 scales[i] = config.min_scale + scale_rate * i;
1027 }
1028 scales.push_back(1.0f);
1029 std::vector<std::vector<float>> default_boxes;
1030 for (auto i = 0; i < config.feature_size.size(); i++) {
1031 float sk1 = scales[i];
1032 float sk2 = scales[i + 1];
1033 float sk3 = sqrt(sk1 * sk2);
1034 std::vector<std::vector<float>> all_sizes;
1035 float w, h;
1036 if (i == 0) {
1037 w = sk1 * sqrt(2);
1038 h = sk1 / sqrt(2);
1039 all_sizes = {{0.1, 0.1}, {w, h}, {h, w}};
1040 } else {
1041 all_sizes = {{sk1, sk1}};
1042 for (int j = 0; j < config.aspect_rations[i].size(); j++) {
1043 w = sk1 * sqrt(config.aspect_rations[i][j]);
1044 h = sk1 / sqrt(config.aspect_rations[i][j]);
1045 all_sizes.push_back({w, h});
1046 all_sizes.push_back({h, w});
1047 }
1048 all_sizes.push_back({sk3, sk3});
1049 }
1050
1051 for (int j = 0; j < config.feature_size[i]; j++) {
1052 for (int k = 0; k < config.feature_size[i]; k++) {
1053 for (int m = 0; m < all_sizes.size(); m++) {
1054 float cx = (k + 0.5) / fk[i];
1055 float cy = (j + 0.5) / fk[i];
1056 default_boxes.push_back({cy, cx, all_sizes[m][1], all_sizes[m][0]});
1057 }
1058 }
1059 }
1060 }
1061 return default_boxes;
1062 }
1063
ConvertBoxes(std::vector<std::vector<float>> & boxes,const std::vector<std::vector<float>> & default_boxes,const BoxesConfig config)1064 void ConvertBoxes(std::vector<std::vector<float>> &boxes, const std::vector<std::vector<float>> &default_boxes,
1065 const BoxesConfig config) {
1066 if (boxes.size() != default_boxes.size() || config.prior_scaling.size() != 2) {
1067 boxes = {};
1068 return;
1069 }
1070 for (int i = 0; i < default_boxes.size(); i++) {
1071 if (boxes[i].size() != 4 || default_boxes[i].size() != 4) {
1072 boxes = {};
1073 return;
1074 }
1075 boxes[i][0] = boxes[i][0] * config.prior_scaling[0] * default_boxes[i][2] + default_boxes[i][0];
1076 boxes[i][1] = boxes[i][1] * config.prior_scaling[0] * default_boxes[i][3] + default_boxes[i][1];
1077 boxes[i][2] = exp(boxes[i][2] * config.prior_scaling[1]) * default_boxes[i][2];
1078 boxes[i][3] = exp(boxes[i][3] * config.prior_scaling[1]) * default_boxes[i][3];
1079 }
1080 }
1081
ApplyNms(const std::vector<std::vector<float>> & all_boxes,std::vector<float> & all_scores,float thres,int max_boxes)1082 std::vector<int> ApplyNms(const std::vector<std::vector<float>> &all_boxes, std::vector<float> &all_scores, float thres,
1083 int max_boxes) {
1084 size_t boxes_num = all_boxes.size();
1085 std::vector<float> areas(boxes_num);
1086 std::vector<int> order(boxes_num);
1087 for (auto i = 0; i < boxes_num; i++) {
1088 if (all_boxes[i].size() < 4) {
1089 return {};
1090 }
1091 areas[i] = (all_boxes[i][3] - all_boxes[i][1] + 1) * (all_boxes[i][2] - all_boxes[i][0] + 1);
1092 order[i] = i;
1093 }
1094
1095 std::sort(order.begin(), order.end(),
1096 [&all_scores](int pos1, int pos2) { return (all_scores[pos1] > all_scores[pos2]); });
1097 std::vector<int> keep;
1098 while (order.size() > 0) {
1099 int i = order[0];
1100 keep.push_back(i);
1101 if (keep.size() >= max_boxes) {
1102 break;
1103 }
1104 int len = order.size() - 1;
1105 std::vector<float> ovr(len);
1106 for (int j = 0; j < len; j++) {
1107 float xx1 = std::max(all_boxes[i][1], all_boxes[order[j + 1]][1]);
1108 float yy1 = std::max(all_boxes[i][0], all_boxes[order[j + 1]][0]);
1109 float xx2 = std::min(all_boxes[i][3], all_boxes[order[j + 1]][3]);
1110 float yy2 = std::min(all_boxes[i][2], all_boxes[order[j + 1]][2]);
1111
1112 float w = std::max(0.0f, xx2 - xx1 + 1);
1113 float h = std::max(0.0f, yy2 - yy1 + 1);
1114 float inter = w * h;
1115 ovr[j] = inter / (areas[i] + areas[order[j + 1]] - inter);
1116 }
1117 std::vector<int> inds;
1118 for (int j = 0; j < len; j++) {
1119 if (ovr[j] <= thres) {
1120 inds.push_back(j + 1);
1121 }
1122 }
1123 std::vector<int> new_order;
1124 new_order.reserve(inds.size());
1125 for (int k = 0; k < inds.size(); k++) {
1126 new_order.push_back(order[inds[k]]);
1127 }
1128 order = new_order;
1129 }
1130 return keep;
1131 }
1132
1133 template <typename Pixel_Type>
ImplementAffine(LiteMat & src,LiteMat & out_img,const double M[6],std::vector<size_t> & dsize,Pixel_Type borderValue)1134 bool ImplementAffine(LiteMat &src, LiteMat &out_img, const double M[6], std::vector<size_t> &dsize,
1135 Pixel_Type borderValue) {
1136 if (dsize.size() != 2 || CheckZero(dsize)) {
1137 return false;
1138 }
1139
1140 double IM[6];
1141 for (int i = 0; i < 6; i++) {
1142 IM[i] = M[i];
1143 }
1144
1145 double D = IM[0] * IM[4] - IM[1] * IM[3];
1146 D = D != 0 ? 1.0f / D : 0;
1147 double A11 = IM[4] * D, A22 = IM[0] * D;
1148 IM[0] = A11;
1149 IM[1] *= -D;
1150 IM[3] *= -D;
1151 IM[4] = A22;
1152 double b1 = -IM[0] * IM[2] - IM[1] * IM[5];
1153 double b2 = -IM[3] * IM[2] - IM[4] * IM[5];
1154 IM[2] = b1;
1155 IM[5] = b2;
1156 if (out_img.IsEmpty()) {
1157 out_img.Init(dsize[0], dsize[1], sizeof(Pixel_Type), src.data_type_);
1158 } else if (out_img.height_ != dsize[1] || out_img.width_ != dsize[0] || out_img.channel_ != src.channel_) {
1159 return false;
1160 } else if (out_img.data_type_ != src.data_type_) {
1161 return false;
1162 }
1163
1164 for (int y = 0; y < out_img.height_; y++) {
1165 for (int x = 0; x < out_img.width_; x++) {
1166 int src_x = IM[0] * x + IM[1] * y + IM[2];
1167 int src_y = IM[3] * x + IM[4] * y + IM[5];
1168 if (src_x >= 0 && src_y >= 0 && src_x < src.width_ && src_y < src.height_) {
1169 Pixel_Type src_pixel = static_cast<Pixel_Type *>(src.data_ptr_)[src_y * src.width_ + src_x];
1170 static_cast<Pixel_Type *>(out_img.data_ptr_)[y * out_img.width_ + x] = src_pixel;
1171 } else {
1172 static_cast<Pixel_Type *>(out_img.data_ptr_)[y * out_img.width_ + x] = borderValue;
1173 }
1174 }
1175 }
1176
1177 return true;
1178 }
1179
Affine(LiteMat & src,LiteMat & out_img,const double M[6],std::vector<size_t> dsize,UINT8_C1 borderValue)1180 bool Affine(LiteMat &src, LiteMat &out_img, const double M[6], std::vector<size_t> dsize, UINT8_C1 borderValue) {
1181 if (src.channel_ == 1 && src.data_type_ == LDataType::UINT8) {
1182 return ImplementAffine(src, out_img, M, dsize, borderValue);
1183 } else {
1184 return false;
1185 }
1186 }
1187
Affine(LiteMat & src,LiteMat & out_img,const double M[6],std::vector<size_t> dsize,UINT8_C3 borderValue)1188 bool Affine(LiteMat &src, LiteMat &out_img, const double M[6], std::vector<size_t> dsize, UINT8_C3 borderValue) {
1189 if (src.channel_ == 3 && src.data_type_ == LDataType::UINT8) {
1190 return ImplementAffine(src, out_img, M, dsize, borderValue);
1191 } else {
1192 return false;
1193 }
1194 }
1195
Affine(LiteMat & src,LiteMat & out_img,const double M[6],std::vector<size_t> dsize,FLOAT32_C1 borderValue)1196 bool Affine(LiteMat &src, LiteMat &out_img, const double M[6], std::vector<size_t> dsize, FLOAT32_C1 borderValue) {
1197 if (src.channel_ == 1 && src.data_type_ == LDataType::FLOAT32) {
1198 return ImplementAffine(src, out_img, M, dsize, borderValue);
1199 } else {
1200 return false;
1201 }
1202 }
1203
Affine(LiteMat & src,LiteMat & out_img,const double M[6],std::vector<size_t> dsize,FLOAT32_C3 borderValue)1204 bool Affine(LiteMat &src, LiteMat &out_img, const double M[6], std::vector<size_t> dsize, FLOAT32_C3 borderValue) {
1205 if (src.channel_ == 3 && src.data_type_ == LDataType::FLOAT32) {
1206 return ImplementAffine(src, out_img, M, dsize, borderValue);
1207 } else {
1208 return false;
1209 }
1210 }
1211
RotationMatrix2DImpl(float x,float y,double angle,double scale,LiteMat & M)1212 inline void RotationMatrix2DImpl(float x, float y, double angle, double scale, LiteMat &M) {
1213 angle *= CV_PI / 180;
1214 double alpha = std::cos(angle) * scale;
1215 double beta = std::sin(angle) * scale;
1216
1217 M.ptr<double>(0)[0] = alpha;
1218 M.ptr<double>(0)[1] = beta;
1219 M.ptr<double>(0)[2] = (1 - alpha) * x - beta * y;
1220 M.ptr<double>(1)[0] = -beta;
1221 M.ptr<double>(1)[1] = alpha;
1222 M.ptr<double>(1)[2] = beta * x + (1 - alpha) * y;
1223 }
1224
GetRotationMatrix2D(float x,float y,double angle,double scale,LiteMat & M)1225 bool GetRotationMatrix2D(float x, float y, double angle, double scale, LiteMat &M) {
1226 M.Init(3, 2, LDataType(LDataType::DOUBLE));
1227 RotationMatrix2DImpl(x, y, angle, scale, M);
1228 return true;
1229 }
1230
1231 template <typename T>
TransposeImpl(const LiteMat & src,LiteMat & dst)1232 bool TransposeImpl(const LiteMat &src, LiteMat &dst) {
1233 int m = src.width_;
1234 int n = src.height_;
1235
1236 dst.Init(n, m, src.data_type_);
1237 for (int i = 0; i < m; i++) {
1238 for (int j = 0; j < n; j++) {
1239 dst.ptr<T>(i)[j] = src.ptr<T>(j)[i];
1240 }
1241 }
1242
1243 return true;
1244 }
1245
Transpose(const LiteMat & src,LiteMat & dst)1246 bool Transpose(const LiteMat &src, LiteMat &dst) {
1247 if (src.IsEmpty()) {
1248 return false;
1249 }
1250 if (src.data_type_ == LDataType::DOUBLE) {
1251 return TransposeImpl<double>(src, dst);
1252 } else if (src.data_type_ == LDataType::FLOAT32) {
1253 return TransposeImpl<float>(src, dst);
1254 } else {
1255 return false;
1256 }
1257 return true;
1258 }
1259
1260 template <typename T>
Hypot_(T a,T b)1261 static inline T Hypot_(T a, T b) {
1262 a = std::abs(a);
1263 b = std::abs(b);
1264 if (a > b) {
1265 b /= a;
1266 return a * std::sqrt(1 + b * b);
1267 }
1268
1269 if (b > 0) {
1270 a /= b;
1271 return b * std::sqrt(1 + a * a);
1272 }
1273 return 0;
1274 }
1275
1276 template <typename T>
Calculation(int n,int m,std::vector<double> & W,LiteMat & A,LiteMat & V,const T eps)1277 void Calculation(int n, int m, std::vector<double> &W, LiteMat &A, LiteMat &V, const T eps) {
1278 int max_iter = std::max(m, 30);
1279 for (int iter = 0; iter < max_iter; iter++) {
1280 bool change = false;
1281 T c;
1282 T s;
1283
1284 for (int i = 0; i < n - 1; i++) {
1285 for (int j = i + 1; j < n; j++) {
1286 T *Ai = A.ptr<T>(i);
1287 T *Aj = A.ptr<T>(j);
1288 double a = W[i];
1289 double p = 0;
1290 double b = W[j];
1291
1292 for (int k = 0; k < m; k++) {
1293 p += static_cast<double>(Ai[k] * Aj[k]);
1294 }
1295
1296 if (std::abs(p) <= eps * std::sqrt(static_cast<double>(a * b))) {
1297 continue;
1298 }
1299
1300 p *= 2;
1301 double beta = a - b;
1302 double gamma = Hypot_(static_cast<double>(p), beta);
1303
1304 if (beta < 0) {
1305 double delta = (gamma - beta) * 0.5;
1306 s = (T)std::sqrt(delta / gamma);
1307 c = (T)(p / (gamma * s * 2));
1308 } else {
1309 c = (T)std::sqrt((gamma + beta) / (gamma * 2));
1310 s = (T)(p / (gamma * c * 2));
1311 }
1312
1313 a = 0;
1314 b = 0;
1315 for (int k = 0; k < m; k++) {
1316 T t0 = c * Ai[k] + s * Aj[k];
1317 T t1 = -s * Ai[k] + c * Aj[k];
1318 Ai[k] = t0;
1319 Aj[k] = t1;
1320 a += static_cast<double>(t0 * t0);
1321 b += static_cast<double>(t1 * t1);
1322 }
1323 W[i] = a;
1324 W[j] = b;
1325 change = true;
1326 T *Vi = V.ptr<T>(i);
1327 T *Vj = V.ptr<T>(j);
1328
1329 for (int k = 0; k < n; k++) {
1330 T t0 = c * Vi[k] + s * Vj[k];
1331 T t1 = -s * Vi[k] + c * Vj[k];
1332 Vi[k] = t0;
1333 Vj[k] = t1;
1334 }
1335 }
1336 }
1337
1338 if (!change) {
1339 break;
1340 }
1341 }
1342 }
1343
1344 template <typename T>
CalculationMatrix(int n,int m,std::vector<double> & W,LiteMat & A,LiteMat & V,const T eps)1345 void CalculationMatrix(int n, int m, std::vector<double> &W, LiteMat &A, LiteMat &V, const T eps) {
1346 for (int i = 0; i < n; i++) {
1347 double sd = 0.;
1348 for (int j = 0; j < m; j++) {
1349 T t = A.ptr<T>(i)[j];
1350 sd += static_cast<double>(t * t);
1351 }
1352 W[i] = sd;
1353
1354 for (int k = 0; k < n; k++) {
1355 V.ptr<T>(i)[k] = 0;
1356 }
1357 V.ptr<T>(i)[i] = 1;
1358 }
1359
1360 Calculation<T>(n, m, W, A, V, eps);
1361 for (int i = 0; i < n; i++) {
1362 double sd = 0;
1363 for (int k = 0; k < m; k++) {
1364 T t = A.ptr<T>(i)[k];
1365 sd += static_cast<double>(t * t);
1366 }
1367 W[i] = std::sqrt(sd);
1368 }
1369
1370 for (int i = 0; i < n - 1; i++) {
1371 int mid = i;
1372 for (int j = i + 1; j < n; j++) {
1373 if (W[mid] < W[j]) {
1374 mid = j;
1375 }
1376 }
1377
1378 if (i != mid) {
1379 std::swap(W[i], W[mid]);
1380 for (int j = 0; j < m; j++) {
1381 std::swap(A.ptr<T>(i)[j], A.ptr<T>(mid)[j]);
1382 }
1383
1384 for (int j = 0; j < n; j++) {
1385 std::swap(V.ptr<T>(i)[j], V.ptr<T>(mid)[j]);
1386 }
1387 }
1388 }
1389 }
1390
1391 template <typename T>
JacobiSVD(LiteMat & A,LiteMat & _W,LiteMat & V)1392 void JacobiSVD(LiteMat &A, LiteMat &_W, LiteMat &V) {
1393 double min_val = FLT_MIN;
1394 T eps = (T)(FLT_EPSILON * 2);
1395 int m = A.width_;
1396 int n = _W.height_;
1397 int urows = m;
1398 std::vector<double> W(n, 0.);
1399
1400 CalculationMatrix<T>(n, m, W, A, V, eps);
1401 for (int i = 0; i < n; i++) {
1402 _W.ptr<T>(i)[0] = (T)W[i];
1403 }
1404
1405 std::random_device rd;
1406 std::mt19937 gen(rd());
1407 std::uniform_int_distribution<unsigned int> dis(0, 4294967294);
1408
1409 for (int i = 0; i < urows; i++) {
1410 double mid = i < n ? W[i] : 0;
1411 for (int ii = 0; ii < 100 && mid <= min_val; ii++) {
1412 const T val0 = (T)(1. / m);
1413 for (int k = 0; k < m; k++) {
1414 unsigned int rng = dis(gen);
1415 T val = (rng & 256) != 0 ? val0 : -val0;
1416 A.ptr<T>(i)[k] = val;
1417 }
1418
1419 for (int inner = 0; inner < 2; inner++) {
1420 for (int j = 0; j < i; j++) {
1421 mid = 0;
1422 for (int k = 0; k < m; k++) {
1423 mid += A.ptr<T>(i)[k] * A.ptr<T>(j)[k];
1424 }
1425 T asum = 0;
1426 for (int k = 0; k < m; k++) {
1427 T t = (T)(A.ptr<T>(i)[k] - mid * A.ptr<T>(j)[k]);
1428 A.ptr<T>(i)[k] = t;
1429 asum += std::abs(t);
1430 }
1431
1432 asum = asum > eps * 100 ? 1 / asum : 0;
1433 for (int k = 0; k < m; k++) {
1434 A.ptr<T>(i)[k] *= asum;
1435 }
1436 }
1437 }
1438
1439 mid = 0;
1440 for (int k = 0; k < m; k++) {
1441 T t = A.ptr<T>(i)[k];
1442 mid += static_cast<double>(t * t);
1443 }
1444 mid = std::sqrt(mid);
1445 }
1446
1447 T s = (T)(mid > min_val ? 1 / mid : 0.);
1448 for (int k = 0; k < m; k++) {
1449 A.ptr<T>(i)[k] *= s;
1450 }
1451 }
1452 }
1453
1454 template <typename T>
SVBkSb(int m,int n,int nb,LiteMat w,LiteMat u,LiteMat v,const LiteMat src2,LiteMat dst)1455 void SVBkSb(int m, int n, int nb, LiteMat w, LiteMat u, LiteMat v, const LiteMat src2, LiteMat dst) {
1456 T eps = DBL_EPSILON * 2;
1457 double thresgold = 0;
1458 int nm = std::min(m, n);
1459
1460 for (int i = 0; i < n; i++) {
1461 dst.ptr<T>(i)[0] = 0;
1462 }
1463
1464 for (int i = 0; i < nm; i++) {
1465 for (int j = 0; j < w.width_; j++) {
1466 thresgold += w.ptr<T>(i)[j];
1467 }
1468 }
1469 thresgold *= eps;
1470
1471 for (int i = 0; i < nm; i++) {
1472 double wi = w.ptr<T>(i)[0];
1473 if (static_cast<double>(std::abs(wi)) < thresgold) {
1474 continue;
1475 }
1476 wi = 1 / wi;
1477 double s = 0;
1478 for (int j = 0; j < n; j++) {
1479 s += u.ptr<T>(i)[j] * src2.ptr<T>(j)[0];
1480 }
1481
1482 s *= wi;
1483 for (int j = 0; j < n; j++) {
1484 dst.ptr<T>(j)[0] = dst.ptr<T>(j)[0] + s * v.ptr<T>(i)[j];
1485 }
1486 }
1487 }
1488
GetPerspectiveTransformImpl(const LiteMat & src1,const LiteMat & src2,LiteMat dst)1489 bool GetPerspectiveTransformImpl(const LiteMat &src1, const LiteMat &src2, LiteMat dst) {
1490 LDataType type = src1.data_type_;
1491 int m = src1.height_;
1492 int m_ = m;
1493 int n = src1.width_;
1494 int nb = src2.width_;
1495
1496 if (m < n) {
1497 return false;
1498 }
1499
1500 double val_a[64] = {0};
1501 double val_v[64] = {0};
1502 double val_w[8] = {0};
1503 LiteMat a(m_, n, val_a, type);
1504 Transpose(src1, a);
1505 LiteMat w(1, n, val_w, type);
1506 LiteMat v(n, n, val_v, type);
1507 LiteMat u;
1508
1509 JacobiSVD<double>(a, w, v);
1510 u = a;
1511
1512 SVBkSb<double>(m_, n, nb, w, u, v, src2, dst);
1513 return true;
1514 }
1515
GetPerspectiveTransform(std::vector<Point> src_point,std::vector<Point> dst_point,LiteMat & M)1516 bool GetPerspectiveTransform(std::vector<Point> src_point, std::vector<Point> dst_point, LiteMat &M) {
1517 if (src_point.size() != 4 || dst_point.size() != 4) {
1518 return false;
1519 }
1520 double m[8][8];
1521 double n[8];
1522 LiteMat src1(8, 8, m, LDataType(LDataType::DOUBLE));
1523 LiteMat src2(1, 8, n, LDataType(LDataType::DOUBLE));
1524
1525 for (int i = 0; i < 4; ++i) {
1526 m[i][0] = m[i + 4][3] = src_point[i].x;
1527 m[i][1] = m[i + 4][4] = src_point[i].y;
1528 m[i][2] = m[i + 4][5] = 1;
1529 m[i][3] = m[i][4] = m[i][5] = m[i + 4][0] = m[i + 4][1] = m[i + 4][2] = 0;
1530 m[i][6] = -src_point[i].x * dst_point[i].x;
1531 m[i][7] = -src_point[i].y * dst_point[i].x;
1532 m[i + 4][6] = -src_point[i].x * dst_point[i].y;
1533 m[i + 4][7] = -src_point[i].y * dst_point[i].y;
1534 n[i] = dst_point[i].x;
1535 n[i + 4] = dst_point[i].y;
1536 }
1537
1538 M.Init(3, 3, LDataType(LDataType::DOUBLE));
1539 LiteMat dst(1, 8, M.data_ptr_, LDataType(LDataType::DOUBLE));
1540
1541 GetPerspectiveTransformImpl(src1, src2, dst);
1542 M.ptr<double>(2)[2] = 1;
1543 return true;
1544 }
1545
GetAffineTransformImpl(LiteMat & src,LiteMat & dst)1546 bool GetAffineTransformImpl(LiteMat &src, LiteMat &dst) {
1547 int m = src.height_;
1548 int n = dst.width_;
1549 for (int i = 0; i < m; i++) {
1550 int k = i;
1551 for (int j = i + 1; j < m; j++) {
1552 if (std::abs(src.ptr<double>(j)[i]) > std::abs(src.ptr<double>(k)[i])) {
1553 k = j;
1554 }
1555 }
1556
1557 if (std::abs(src.ptr<double>(k)[i]) < DBL_EPSILON * 100) {
1558 dst.Init(1, 6, LDataType(LDataType::DOUBLE));
1559 (void)memset(dst.data_ptr_, 0, 6 * sizeof(double));
1560 RETURN_FALSE_IF_LITEMAT_EMPTY(dst);
1561 return false;
1562 }
1563 if (k != i) {
1564 for (int j = i; j < m; j++) {
1565 std::swap(src.ptr<double>(i)[j], src.ptr<double>(k)[j]);
1566 }
1567
1568 if (dst.data_ptr_) {
1569 for (int j = 0; j < n; j++) {
1570 std::swap(dst.ptr<double>(i)[j], dst.ptr<double>(k)[j]);
1571 }
1572 }
1573 }
1574
1575 const double d = -1 / src.ptr<double>(i)[i];
1576 for (int j = i + 1; j < m; j++) {
1577 double alpha = src.ptr<double>(j)[i] * d;
1578 for (k = i + 1; k < m; k++) {
1579 src.ptr<double>(j)[k] += alpha * src.ptr<double>(i)[k];
1580 }
1581
1582 if (dst.data_ptr_) {
1583 for (k = 0; k < n; k++) {
1584 dst.ptr<double>(j)[k] += alpha * dst.ptr<double>(i)[k];
1585 }
1586 }
1587 }
1588 }
1589
1590 if (dst.data_ptr_) {
1591 for (int i = m - 1; i >= 0; i--) {
1592 for (int j = 0; j < n; j++) {
1593 double s = dst.ptr<double>(i)[j];
1594 for (int k = i + 1; k < m; k++) {
1595 s -= src.ptr<double>(i)[k] * dst.ptr<double>(k)[j];
1596 }
1597 dst.ptr<double>(i)[j] = s / src.ptr<double>(i)[i];
1598 }
1599 }
1600 }
1601
1602 return true;
1603 }
1604
GetAffineTransform(std::vector<Point> src_point,std::vector<Point> dst_point,LiteMat & M)1605 bool GetAffineTransform(std::vector<Point> src_point, std::vector<Point> dst_point, LiteMat &M) {
1606 if (src_point.size() != 3 || dst_point.size() != 3) {
1607 return false;
1608 }
1609 double m[6 * 6];
1610 double n[6];
1611 LiteMat src1(6, 6, m, LDataType(LDataType::DOUBLE));
1612 LiteMat src2(1, 6, n, LDataType(LDataType::DOUBLE));
1613
1614 for (int i = 0; i < 3; i++) {
1615 int j = i * 12;
1616 int k = i * 12 + 6;
1617 m[j] = m[k + 3] = src_point[i].x;
1618 m[j + 1] = m[k + 4] = src_point[i].y;
1619 m[j + 2] = m[k + 5] = 1;
1620 m[j + 3] = m[j + 4] = m[j + 5] = 0;
1621 m[k] = m[k + 1] = m[k + 2] = 0;
1622 n[i * 2] = dst_point[i].x;
1623 n[i * 2 + 1] = dst_point[i].y;
1624 }
1625
1626 GetAffineTransformImpl(src1, src2);
1627 M.Init(3, 2, 1, LDataType(LDataType::DOUBLE));
1628 for (int i = 0; i < M.height_; i++) {
1629 for (int j = 0; j < M.width_; j++) {
1630 M.ptr<double>(i)[j] = src2.ptr<double>(i * M.width_ + j)[0];
1631 }
1632 }
1633 return true;
1634 }
1635
ConvertRgbToBgr(const LiteMat & src,const LDataType & data_type,int w,int h,LiteMat & mat)1636 bool ConvertRgbToBgr(const LiteMat &src, const LDataType &data_type, int w, int h, LiteMat &mat) {
1637 if (data_type == LDataType::UINT8) {
1638 if (src.IsEmpty()) {
1639 return false;
1640 }
1641 if (mat.IsEmpty()) {
1642 mat.Init(w, h, 3, LDataType::UINT8);
1643 }
1644 if (mat.channel_ != 3) {
1645 return false;
1646 }
1647 if ((src.width_ != w) || (src.height_ != h)) {
1648 return false;
1649 }
1650 unsigned char *ptr = mat;
1651 const unsigned char *data_ptr = src;
1652 for (int y = 0; y < h; y++) {
1653 for (int x = 0; x < w; x++) {
1654 ptr[0] = data_ptr[2];
1655 ptr[1] = data_ptr[1];
1656 ptr[2] = data_ptr[0];
1657
1658 ptr += 3;
1659 data_ptr += 3;
1660 }
1661 }
1662 } else {
1663 return false;
1664 }
1665 return true;
1666 }
1667
ConvertRgbToGray(const LiteMat & src,LDataType data_type,int w,int h,LiteMat & mat)1668 bool ConvertRgbToGray(const LiteMat &src, LDataType data_type, int w, int h, LiteMat &mat) {
1669 if (data_type == LDataType::UINT8) {
1670 if (src.IsEmpty()) {
1671 return false;
1672 }
1673 if (mat.IsEmpty()) {
1674 mat.Init(w, h, 1, LDataType::UINT8);
1675 }
1676 if (mat.channel_ != 1) {
1677 return false;
1678 }
1679 if ((src.width_ != w) || (src.height_ != h)) {
1680 return false;
1681 }
1682 unsigned char *ptr = mat;
1683 const unsigned char *data_ptr = src;
1684 for (int y = 0; y < h; y++) {
1685 for (int x = 0; x < w; x++) {
1686 *ptr = (data_ptr[2] * kB2Gray + data_ptr[1] * kG2Gray + data_ptr[0] * kR2Gray + kGrayShiftDelta) >> kGrayShift;
1687 ptr++;
1688 data_ptr += 3;
1689 }
1690 }
1691 } else {
1692 return false;
1693 }
1694 return true;
1695 }
1696
UpdateOrientationAfineMat(const LiteMat & src,int * rotationDstWidth,int * rotationDstHeight,float (* varM)[2][3],int img_orientation)1697 void UpdateOrientationAfineMat(const LiteMat &src, int *rotationDstWidth, int *rotationDstHeight, float (*varM)[2][3],
1698 int img_orientation) {
1699 int srcOrientation = img_orientation;
1700 if (IM_TOOL_EXIF_ORIENTATION_0_DEG_MIRROR == srcOrientation) {
1701 (*varM)[0][0] *= -1;
1702 (*varM)[0][2] += *rotationDstWidth - 1;
1703 } else if ((IM_TOOL_EXIF_ORIENTATION_180_DEG == srcOrientation) ||
1704 (IM_TOOL_EXIF_ORIENTATION_180_DEG_MIRROR == srcOrientation)) {
1705 // 0, 1, 2 is the matrix index of varM
1706 (*varM)[0][0] = -1;
1707 (*varM)[0][1] = 0;
1708 (*varM)[0][2] = *rotationDstWidth - 1;
1709 (*varM)[1][0] = 0;
1710 (*varM)[1][1] = -1;
1711 (*varM)[1][2] = *rotationDstHeight - 1;
1712 if (IM_TOOL_EXIF_ORIENTATION_180_DEG_MIRROR == srcOrientation) {
1713 /* with (*varM)irror */
1714 (*varM)[0][0] *= -1;
1715 (*varM)[0][2] -= *rotationDstWidth - 1;
1716 }
1717 } else if ((IM_TOOL_EXIF_ORIENTATION_90_DEG_MIRROR == srcOrientation) ||
1718 (IM_TOOL_EXIF_ORIENTATION_90_DEG == srcOrientation)) {
1719 /* 90 Deg rotation */
1720 *rotationDstWidth = src.height_;
1721 *rotationDstHeight = src.width_;
1722 (*varM)[0][0] = 0;
1723 (*varM)[0][1] = -1;
1724 (*varM)[0][2] = *rotationDstWidth - 1;
1725 (*varM)[1][0] = 1;
1726 (*varM)[1][1] = 0;
1727 (*varM)[1][2] = 0;
1728 if (IM_TOOL_EXIF_ORIENTATION_90_DEG_MIRROR == srcOrientation) {
1729 /* with Mirror */
1730 (*varM)[0][1] *= -1;
1731 (*varM)[0][2] -= *rotationDstWidth - 1;
1732 }
1733 } else if ((IM_TOOL_EXIF_ORIENTATION_270_DEG_MIRROR == srcOrientation) ||
1734 (IM_TOOL_EXIF_ORIENTATION_270_DEG == srcOrientation)) {
1735 /* 270 Deg rotation */
1736 *rotationDstWidth = src.height_;
1737 *rotationDstHeight = src.width_;
1738 (*varM)[0][0] = 0;
1739 (*varM)[0][1] = 1;
1740 (*varM)[0][2] = 0;
1741 (*varM)[1][0] = -1;
1742 (*varM)[1][1] = 0;
1743 (*varM)[1][2] = *rotationDstHeight - 1;
1744 if (IM_TOOL_EXIF_ORIENTATION_270_DEG_MIRROR == srcOrientation) {
1745 /* with Mirror */
1746 (*varM)[0][1] *= -1;
1747 (*varM)[0][2] += *rotationDstWidth - 1;
1748 }
1749 }
1750 }
1751
ImageToolsConvertImage(const LiteMat & src,const LiteMat & dst,imageToolsImage_t * imageIn,imageToolsImage_t * imageOut)1752 void ImageToolsConvertImage(const LiteMat &src, const LiteMat &dst, imageToolsImage_t *imageIn,
1753 imageToolsImage_t *imageOut) {
1754 imageIn->image_buff = src.data_ptr_;
1755 imageIn->h = src.height_;
1756 imageIn->w = src.width_;
1757 imageIn->stride = src.width_;
1758 imageIn->dataType = IM_TOOL_DATA_TYPE_UINT8;
1759
1760 imageOut->image_buff = dst.data_ptr_;
1761 imageOut->h = dst.height_;
1762 imageOut->w = dst.width_;
1763 imageOut->stride = dst.width_;
1764 imageOut->dataType = IM_TOOL_DATA_TYPE_FLOAT;
1765 }
1766
InvAffine2x3(float M[2][3],float invM[][3])1767 int InvAffine2x3(float M[2][3], float invM[][3]) {
1768 float inv_det = M[0][0] * M[1][1] - M[1][0] * M[0][1];
1769 if (inv_det == 0.0) {
1770 return IM_TOOL_RETURN_STATUS_FAILED;
1771 }
1772 invM[1][1] = M[0][0] / inv_det;
1773 invM[0][1] = -M[0][1] / inv_det;
1774 invM[1][0] = -M[1][0] / inv_det;
1775 invM[0][0] = M[1][1] / inv_det;
1776 invM[0][2] = (M[0][1] * M[1][2] - M[1][1] * M[0][2]) / inv_det;
1777 invM[1][2] = -(M[0][0] * M[1][2] - M[1][0] * M[0][2]) / inv_det;
1778 return IM_TOOL_RETURN_STATUS_SUCCESS;
1779 }
1780
CalDst(float * dst,float v1,float v2,float v3)1781 static float *CalDst(float *dst, float v1, float v2, float v3) {
1782 *dst++ = v1;
1783 *dst++ = v2;
1784 *dst++ = v3;
1785 return dst;
1786 }
1787
ImageWarpAffineHWCFloat(imageToolsImage_t image,imageToolsImage_t warped_image,float invM[2][3])1788 static void ImageWarpAffineHWCFloat(imageToolsImage_t image, imageToolsImage_t warped_image, float invM[2][3]) {
1789 // 3 is r, g, b
1790 warped_image.stride *= 3;
1791 image.stride *= 3;
1792
1793 float *warped_image_buff = reinterpret_cast<float *>(warped_image.image_buff);
1794
1795 float *image_buff = reinterpret_cast<float *>(image.image_buff);
1796 for (int y0 = 0; y0 < warped_image.h; y0++) {
1797 // Init pointers to start of rows
1798 float *dst = warped_image_buff + y0 * warped_image.stride;
1799
1800 for (int x0 = 0; x0 < warped_image.w; x0++) {
1801 // number 0, 1, 2 is the index of MATRIX 'invM'
1802 float fPosx = (static_cast<float>(x0) * invM[0][0]) + (static_cast<float>(y0) * invM[0][1]) + invM[0][2];
1803 float fPosy = (static_cast<float>(x0) * invM[1][0]) + (static_cast<float>(y0) * invM[1][1]) + invM[1][2];
1804 int iPosy = static_cast<int>(fPosy + 2) - 2; // for floor like result until -2.
1805 int iPosx = static_cast<int>(fPosx + 2) - 2; // for floor like result until -2.
1806 if ((iPosx < -1) || (iPosx >= image.w) || (iPosy < -1) || (iPosy >= image.h)) {
1807 dst = CalDst(dst, 0.0f, 0.0f, 0.0f);
1808 continue;
1809 }
1810 float fRsiduy = fPosy - iPosy;
1811 float fRsidux = fPosx - iPosx;
1812 float fOut0 = 0;
1813 float fOut1 = 0;
1814 float fOut2 = 0;
1815 float *fTopeLeft = image_buff + iPosy * image.stride + iPosx * 3;
1816 float fCoeff = 1 - fRsidux - fRsiduy + fRsidux * fRsiduy;
1817 if ((iPosx >= 0) && (iPosy >= 0)) {
1818 // number 0, 1, 2 is the index of MATRIX 'fTopeLeft'
1819 fOut0 += fCoeff * fTopeLeft[0];
1820 fOut1 += fCoeff * fTopeLeft[1];
1821 fOut2 += fCoeff * fTopeLeft[2];
1822 }
1823 float fSum = fCoeff;
1824 fCoeff = fRsiduy - fRsidux * fRsiduy;
1825 if ((iPosx >= 0) && (iPosy < image.h - 1)) {
1826 // Image channel G and B could be accessed by adding number of 1, 2
1827 fOut0 += fCoeff * fTopeLeft[image.stride];
1828 fOut1 += fCoeff * fTopeLeft[image.stride + 1];
1829 fOut2 += fCoeff * fTopeLeft[image.stride + 2];
1830 }
1831 fSum += fCoeff;
1832 fCoeff = fRsidux - fRsidux * fRsiduy;
1833 if ((iPosx < image.w - 1) && (iPosy >= 0)) {
1834 // Image channel G and B could be accessed by adding number of 1, 2
1835 fOut0 += fCoeff * fTopeLeft[3];
1836 fOut1 += fCoeff * fTopeLeft[3 + 1];
1837 fOut2 += fCoeff * fTopeLeft[3 + 2];
1838 }
1839 fSum += fCoeff;
1840 if ((iPosx < image.w - 1) && (iPosy < image.h - 1)) {
1841 // Image channel G and B could be accessed by adding number of 1, 2
1842 fOut0 += (1 - fSum) * fTopeLeft[image.stride + 3];
1843 fOut1 += (1 - fSum) * fTopeLeft[image.stride + 3 + 1];
1844 fOut2 += (1 - fSum) * fTopeLeft[image.stride + 3 + 2];
1845 }
1846 dst = CalDst(dst, fOut0, fOut1, fOut2);
1847 }
1848 }
1849 }
1850
ImageWarpAffineHWCUint8(imageToolsImage_t image,imageToolsImage_t warped_image,float invM[2][3])1851 static void ImageWarpAffineHWCUint8(imageToolsImage_t image, imageToolsImage_t warped_image, float invM[2][3]) {
1852 // 3 is r, g, b
1853 warped_image.stride *= 3;
1854 image.stride *= 3;
1855 float *warped_image_buff = reinterpret_cast<float *>(warped_image.image_buff);
1856
1857 uint8_t *image_buff = reinterpret_cast<uint8_t *>(image.image_buff);
1858 for (int y0 = 0; y0 < warped_image.h; y0++) {
1859 // Init pointers to start of rows
1860 float *dst = warped_image_buff + y0 * warped_image.stride;
1861
1862 for (int x0 = 0; x0 < warped_image.w; x0++) {
1863 float fPosx = (static_cast<float>(x0) * invM[0][0]) + (static_cast<float>(y0) * invM[0][1]) + invM[0][2];
1864 float fPosy = (static_cast<float>(x0) * invM[1][0]) + (static_cast<float>(y0) * invM[1][1]) + invM[1][2];
1865
1866 int iPosy = static_cast<int>(fPosy + 2) - 2; // for floor like result until -2.
1867 int iPosx = static_cast<int>(fPosx + 2) - 2; // for floor like result until -2.
1868 if ((iPosx < -1) || (iPosx >= image.w) || (iPosy < -1) || (iPosy >= image.h)) {
1869 dst = CalDst(dst, 0.0f, 0.0f, 0.0f);
1870 continue;
1871 }
1872 float fRsiduy = fPosy - iPosy;
1873 float fRsidux = fPosx - iPosx;
1874 float fOut0 = 0;
1875 float fOut1 = 0;
1876 float fOut2 = 0;
1877 uint8_t *uiTopeLeft = image_buff + iPosy * image.stride + iPosx * 3;
1878 float fCoeff = 1 - fRsidux - fRsiduy + fRsidux * fRsiduy;
1879 if ((iPosx >= 0) && (iPosy >= 0)) {
1880 // number 0, 1, 2 is the index of MATRIX round.
1881 fOut0 += fCoeff * static_cast<float>(uiTopeLeft[0]);
1882 fOut1 += fCoeff * static_cast<float>(uiTopeLeft[1]);
1883 fOut2 += fCoeff * static_cast<float>(uiTopeLeft[2]);
1884 }
1885 float fSum = fCoeff;
1886 fCoeff = fRsiduy - fRsidux * fRsiduy;
1887 if ((iPosx >= 0) && (iPosy < image.h - 1)) {
1888 fOut0 += fCoeff * static_cast<float>(uiTopeLeft[image.stride]);
1889 fOut1 += fCoeff * static_cast<float>(uiTopeLeft[image.stride + 1]);
1890 fOut2 += fCoeff * static_cast<float>(uiTopeLeft[image.stride + 2]);
1891 }
1892 fSum += fCoeff;
1893 fCoeff = fRsidux - fRsidux * fRsiduy;
1894 if ((iPosx < image.w - 1) && (iPosy >= 0)) {
1895 fOut0 += fCoeff * static_cast<float>(uiTopeLeft[3]);
1896 fOut1 += fCoeff * static_cast<float>(uiTopeLeft[3 + 1]);
1897 fOut2 += fCoeff * static_cast<float>(uiTopeLeft[3 + 2]);
1898 }
1899 fSum += fCoeff;
1900 if ((iPosx < image.w - 1) && (iPosy < image.h - 1)) {
1901 fOut0 += (1 - fSum) * static_cast<float>(uiTopeLeft[image.stride + 3]);
1902 fOut1 += (1 - fSum) * static_cast<float>(uiTopeLeft[image.stride + 3 + 1]);
1903 fOut2 += (1 - fSum) * static_cast<float>(uiTopeLeft[image.stride + 3 + 2]);
1904 }
1905 dst = CalDst(dst, fOut0, fOut1, fOut2);
1906 }
1907 }
1908 }
1909
ImageWarpAffineHWC(imageToolsImage_t image,imageToolsImage_t warped_image,float M[2][3],bool bIsMInv)1910 int ImageWarpAffineHWC(imageToolsImage_t image, imageToolsImage_t warped_image, float M[2][3], bool bIsMInv) {
1911 if ((IM_TOOL_DATA_TYPE_FLOAT != warped_image.dataType) ||
1912 ((IM_TOOL_DATA_TYPE_FLOAT != image.dataType) && (IM_TOOL_DATA_TYPE_UINT8 != image.dataType))) {
1913 return IM_TOOL_RETURN_STATUS_INVALID_INPUT;
1914 }
1915 float invM[2][3];
1916 if (bIsMInv) {
1917 for (int iy = 0; iy < 2; iy++) {
1918 for (int ix = 0; ix < 3; ix++) {
1919 invM[iy][ix] = M[iy][ix];
1920 }
1921 }
1922 } else {
1923 if (InvAffine2x3(M, invM) != IM_TOOL_RETURN_STATUS_SUCCESS) {
1924 return IM_TOOL_RETURN_STATUS_FAILED;
1925 }
1926 }
1927
1928 if (IM_TOOL_DATA_TYPE_FLOAT == image.dataType) {
1929 ImageWarpAffineHWCFloat(image, warped_image, invM);
1930 } else {
1931 ImageWarpAffineHWCUint8(image, warped_image, invM);
1932 }
1933 return IM_TOOL_RETURN_STATUS_SUCCESS;
1934 }
1935
ResizePreserveARWithFiller(LiteMat & src,LiteMat & dst,int h,int w,float (* ratioShiftWShiftH)[3],float (* invM)[2][3],int img_orientation)1936 bool ResizePreserveARWithFiller(LiteMat &src, LiteMat &dst, int h, int w, float (*ratioShiftWShiftH)[3],
1937 float (*invM)[2][3], int img_orientation) {
1938 if (src.IsEmpty() || src.channel_ != 3 || h <= 0 || w <= 0 || h > 10000 || w > 10000) {
1939 return false;
1940 }
1941 if (ratioShiftWShiftH == nullptr || invM == nullptr) {
1942 return false;
1943 }
1944 if (dst.IsEmpty()) {
1945 dst.Init(w, h, src.channel_, LDataType::FLOAT32);
1946 }
1947
1948 float varM[2][3] = {{1.0, 0, 0}, {0, 1.0, 0}};
1949 const float divisor = 2.0;
1950 int rotationDstWidth = src.width_;
1951 int rotationDstHeight = src.height_;
1952 if (rotationDstWidth == 0 || rotationDstHeight == 0) {
1953 return false;
1954 }
1955
1956 if (dst.height_ == 0) {
1957 return false;
1958 }
1959
1960 if (img_orientation > IM_TOOL_EXIF_ORIENTATION_0_DEG) {
1961 UpdateOrientationAfineMat(src, &rotationDstWidth, &rotationDstHeight, &varM, img_orientation);
1962 }
1963
1964 /* Resize after orientation fix */
1965 float srcAR = static_cast<float>(rotationDstWidth) / static_cast<float>(rotationDstHeight);
1966 float dstAR = static_cast<float>(dst.width_) / static_cast<float>(dst.height_);
1967 auto dstActiveWidth = static_cast<float>(dst.width_);
1968 auto dstActiveHeight = static_cast<float>(dst.height_);
1969 float ratio, shiftW, shiftH;
1970 if (srcAR < dstAR) {
1971 ratio = static_cast<float>(dst.height_) / static_cast<float>(rotationDstHeight);
1972 dstActiveWidth = static_cast<float>(rotationDstWidth) * ratio;
1973 } else {
1974 ratio = static_cast<float>(dst.width_) / static_cast<float>(rotationDstWidth);
1975 dstActiveHeight = static_cast<float>(rotationDstHeight) * ratio;
1976 }
1977 shiftW = (static_cast<float>(dst.width_) - dstActiveWidth) / divisor;
1978 shiftH = (static_cast<float>(dst.height_) - dstActiveHeight) / divisor;
1979 for (auto &iy : varM) {
1980 for (float &ix : iy) {
1981 // cppcheck-suppress useStlAlgorithm
1982 ix *= ratio;
1983 }
1984 }
1985
1986 varM[0][2] += shiftW;
1987 varM[1][2] += shiftH;
1988 /* Resize and shift by affine transform */
1989 imageToolsImage_t imageIn, imageOut;
1990 ImageToolsConvertImage(src, dst, &imageIn, &imageOut);
1991 if (InvAffine2x3(varM, *invM) != IM_TOOL_RETURN_STATUS_SUCCESS) {
1992 return false;
1993 }
1994 int retVal = ImageWarpAffineHWC(imageIn, imageOut, *invM, true);
1995 if (retVal != 0) {
1996 return false;
1997 }
1998
1999 // 0, 1, 2 is the index of corresponding elem in ratioShiftWShiftH
2000 (*ratioShiftWShiftH)[0] = ratio;
2001 (*ratioShiftWShiftH)[1] = shiftW;
2002 (*ratioShiftWShiftH)[2] = shiftH;
2003
2004 return true;
2005 }
2006
2007 } // namespace dataset
2008 } // namespace mindspore
2009