1 /*
2 * Copyright (c) 2015 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <limits.h>
12 #include <math.h>
13
14 #include "vp9/common/vp9_blockd.h"
15 #include "vp9/encoder/vp9_encoder.h"
16 #include "vp9/encoder/vp9_skin_detection.h"
17
18 #define MODEL_MODE 1
19
20 // Fixed-point skin color model parameters.
21 static const int skin_mean[5][2] = { { 7463, 9614 },
22 { 6400, 10240 },
23 { 7040, 10240 },
24 { 8320, 9280 },
25 { 6800, 9614 } };
26 static const int skin_inv_cov[4] = { 4107, 1663, 1663, 2157 }; // q16
27 static const int skin_threshold[6] = { 1570636, 1400000, 800000,
28 800000, 800000, 800000 }; // q18
29
30 // Thresholds on luminance.
31 static const int y_low = 40;
32 static const int y_high = 220;
33
34 // Evaluates the Mahalanobis distance measure for the input CbCr values.
evaluate_skin_color_difference(int cb,int cr,int idx)35 static int evaluate_skin_color_difference(int cb, int cr, int idx) {
36 const int cb_q6 = cb << 6;
37 const int cr_q6 = cr << 6;
38 const int cb_diff_q12 =
39 (cb_q6 - skin_mean[idx][0]) * (cb_q6 - skin_mean[idx][0]);
40 const int cbcr_diff_q12 =
41 (cb_q6 - skin_mean[idx][0]) * (cr_q6 - skin_mean[idx][1]);
42 const int cr_diff_q12 =
43 (cr_q6 - skin_mean[idx][1]) * (cr_q6 - skin_mean[idx][1]);
44 const int cb_diff_q2 = (cb_diff_q12 + (1 << 9)) >> 10;
45 const int cbcr_diff_q2 = (cbcr_diff_q12 + (1 << 9)) >> 10;
46 const int cr_diff_q2 = (cr_diff_q12 + (1 << 9)) >> 10;
47 const int skin_diff =
48 skin_inv_cov[0] * cb_diff_q2 + skin_inv_cov[1] * cbcr_diff_q2 +
49 skin_inv_cov[2] * cbcr_diff_q2 + skin_inv_cov[3] * cr_diff_q2;
50 return skin_diff;
51 }
52
vp9_skin_pixel(const uint8_t y,const uint8_t cb,const uint8_t cr,int motion)53 int vp9_skin_pixel(const uint8_t y, const uint8_t cb, const uint8_t cr,
54 int motion) {
55 if (y < y_low || y > y_high) {
56 return 0;
57 } else {
58 if (MODEL_MODE == 0) {
59 return (evaluate_skin_color_difference(cb, cr, 0) < skin_threshold[0]);
60 } else {
61 int i = 0;
62 // Exit on grey.
63 if (cb == 128 && cr == 128) return 0;
64 // Exit on very strong cb.
65 if (cb > 150 && cr < 110) return 0;
66 for (; i < 5; i++) {
67 int skin_color_diff = evaluate_skin_color_difference(cb, cr, i);
68 if (skin_color_diff < skin_threshold[i + 1]) {
69 if (y < 60 && skin_color_diff > 3 * (skin_threshold[i + 1] >> 2))
70 return 0;
71 else if (motion == 0 &&
72 skin_color_diff > (skin_threshold[i + 1] >> 1))
73 return 0;
74 else
75 return 1;
76 }
77 // Exit if difference is much large than the threshold.
78 if (skin_color_diff > (skin_threshold[i + 1] << 3)) {
79 return 0;
80 }
81 }
82 return 0;
83 }
84 }
85 }
86
vp9_compute_skin_block(const uint8_t * y,const uint8_t * u,const uint8_t * v,int stride,int strideuv,int bsize,int consec_zeromv,int curr_motion_magn)87 int vp9_compute_skin_block(const uint8_t *y, const uint8_t *u, const uint8_t *v,
88 int stride, int strideuv, int bsize,
89 int consec_zeromv, int curr_motion_magn) {
90 // No skin if block has been zero/small motion for long consecutive time.
91 if (consec_zeromv > 60 && curr_motion_magn == 0) {
92 return 0;
93 } else {
94 int motion = 1;
95 // Take center pixel in block to determine is_skin.
96 const int y_width_shift = (4 << b_width_log2_lookup[bsize]) >> 1;
97 const int y_height_shift = (4 << b_height_log2_lookup[bsize]) >> 1;
98 const int uv_width_shift = y_width_shift >> 1;
99 const int uv_height_shift = y_height_shift >> 1;
100 const uint8_t ysource = y[y_height_shift * stride + y_width_shift];
101 const uint8_t usource = u[uv_height_shift * strideuv + uv_width_shift];
102 const uint8_t vsource = v[uv_height_shift * strideuv + uv_width_shift];
103 if (consec_zeromv > 25 && curr_motion_magn == 0) motion = 0;
104 return vp9_skin_pixel(ysource, usource, vsource, motion);
105 }
106 }
107
108 #ifdef OUTPUT_YUV_SKINMAP
109 // For viewing skin map on input source.
vp9_compute_skin_map(VP9_COMP * const cpi,FILE * yuv_skinmap_file)110 void vp9_compute_skin_map(VP9_COMP *const cpi, FILE *yuv_skinmap_file) {
111 int i, j, mi_row, mi_col, num_bl;
112 VP9_COMMON *const cm = &cpi->common;
113 uint8_t *y;
114 const uint8_t *src_y = cpi->Source->y_buffer;
115 const uint8_t *src_u = cpi->Source->u_buffer;
116 const uint8_t *src_v = cpi->Source->v_buffer;
117 const int src_ystride = cpi->Source->y_stride;
118 const int src_uvstride = cpi->Source->uv_stride;
119 int y_bsize = 16; // Use 8x8 or 16x16.
120 int uv_bsize = y_bsize >> 1;
121 int ypos = y_bsize >> 1;
122 int uvpos = uv_bsize >> 1;
123 int shy = (y_bsize == 8) ? 3 : 4;
124 int shuv = shy - 1;
125 int fac = y_bsize / 8;
126 // Use center pixel or average of center 2x2 pixels.
127 int mode_filter = 0;
128 YV12_BUFFER_CONFIG skinmap;
129 memset(&skinmap, 0, sizeof(YV12_BUFFER_CONFIG));
130 if (vpx_alloc_frame_buffer(&skinmap, cm->width, cm->height, cm->subsampling_x,
131 cm->subsampling_y, VP9_ENC_BORDER_IN_PIXELS,
132 cm->byte_alignment)) {
133 vpx_free_frame_buffer(&skinmap);
134 return;
135 }
136 memset(skinmap.buffer_alloc, 128, skinmap.frame_size);
137 y = skinmap.y_buffer;
138 // Loop through blocks and set skin map based on center pixel of block.
139 // Set y to white for skin block, otherwise set to source with gray scale.
140 // Ignore rightmost/bottom boundary blocks.
141 for (mi_row = 0; mi_row < cm->mi_rows - 1; mi_row += fac) {
142 num_bl = 0;
143 for (mi_col = 0; mi_col < cm->mi_cols - 1; mi_col += fac) {
144 int is_skin = 0;
145 if (mode_filter == 1) {
146 // Use 2x2 average at center.
147 uint8_t ysource = src_y[ypos * src_ystride + ypos];
148 uint8_t usource = src_u[uvpos * src_uvstride + uvpos];
149 uint8_t vsource = src_v[uvpos * src_uvstride + uvpos];
150 uint8_t ysource2 = src_y[(ypos + 1) * src_ystride + ypos];
151 uint8_t usource2 = src_u[(uvpos + 1) * src_uvstride + uvpos];
152 uint8_t vsource2 = src_v[(uvpos + 1) * src_uvstride + uvpos];
153 uint8_t ysource3 = src_y[ypos * src_ystride + (ypos + 1)];
154 uint8_t usource3 = src_u[uvpos * src_uvstride + (uvpos + 1)];
155 uint8_t vsource3 = src_v[uvpos * src_uvstride + (uvpos + 1)];
156 uint8_t ysource4 = src_y[(ypos + 1) * src_ystride + (ypos + 1)];
157 uint8_t usource4 = src_u[(uvpos + 1) * src_uvstride + (uvpos + 1)];
158 uint8_t vsource4 = src_v[(uvpos + 1) * src_uvstride + (uvpos + 1)];
159 ysource = (ysource + ysource2 + ysource3 + ysource4) >> 2;
160 usource = (usource + usource2 + usource3 + usource4) >> 2;
161 vsource = (vsource + vsource2 + vsource3 + vsource4) >> 2;
162 is_skin = vp9_skin_pixel(ysource, usource, vsource, 1);
163 } else {
164 int block_size = BLOCK_8X8;
165 int consec_zeromv = 0;
166 int bl_index = mi_row * cm->mi_cols + mi_col;
167 int bl_index1 = bl_index + 1;
168 int bl_index2 = bl_index + cm->mi_cols;
169 int bl_index3 = bl_index2 + 1;
170 if (y_bsize == 8)
171 consec_zeromv = cpi->consec_zero_mv[bl_index];
172 else
173 consec_zeromv =
174 VPXMIN(cpi->consec_zero_mv[bl_index],
175 VPXMIN(cpi->consec_zero_mv[bl_index1],
176 VPXMIN(cpi->consec_zero_mv[bl_index2],
177 cpi->consec_zero_mv[bl_index3])));
178 if (y_bsize == 16) block_size = BLOCK_16X16;
179 is_skin =
180 vp9_compute_skin_block(src_y, src_u, src_v, src_ystride,
181 src_uvstride, block_size, consec_zeromv, 0);
182 }
183 for (i = 0; i < y_bsize; i++) {
184 for (j = 0; j < y_bsize; j++) {
185 if (is_skin)
186 y[i * src_ystride + j] = 255;
187 else
188 y[i * src_ystride + j] = src_y[i * src_ystride + j];
189 }
190 }
191 num_bl++;
192 y += y_bsize;
193 src_y += y_bsize;
194 src_u += uv_bsize;
195 src_v += uv_bsize;
196 }
197 y += (src_ystride << shy) - (num_bl << shy);
198 src_y += (src_ystride << shy) - (num_bl << shy);
199 src_u += (src_uvstride << shuv) - (num_bl << shuv);
200 src_v += (src_uvstride << shuv) - (num_bl << shuv);
201 }
202 vp9_write_yuv_frame_420(&skinmap, yuv_skinmap_file);
203 vpx_free_frame_buffer(&skinmap);
204 }
205 #endif
206