1 /*
2 * Copyright (c) 2012 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <assert.h>
12 #include <limits.h>
13 #include "./vpx_dsp_rtcd.h"
14 #include "vpx_scale/yv12config.h"
15 #include "vpx/vpx_integer.h"
16 #include "vp9/common/vp9_reconinter.h"
17 #include "vp9/encoder/vp9_context_tree.h"
18 #include "vp9/encoder/vp9_denoiser.h"
19
20 /* The VP9 denoiser is a work-in-progress. It currently is only designed to work
21 * with speed 6, though it (inexplicably) seems to also work with speed 5 (one
22 * would need to modify the source code in vp9_pickmode.c and vp9_encoder.c to
23 * make the calls to the vp9_denoiser_* functions when in speed 5).
24 *
25 * The implementation is very similar to that of the VP8 denoiser. While
26 * choosing the motion vectors / reference frames, the denoiser is run, and if
27 * it did not modify the signal to much, the denoised block is copied to the
28 * signal.
29 */
30
31 #ifdef OUTPUT_YUV_DENOISED
32 static void make_grayscale(YV12_BUFFER_CONFIG *yuv);
33 #endif
34
absdiff_thresh(BLOCK_SIZE bs,int increase_denoising)35 static int absdiff_thresh(BLOCK_SIZE bs, int increase_denoising) {
36 (void)bs;
37 return 3 + (increase_denoising ? 1 : 0);
38 }
39
delta_thresh(BLOCK_SIZE bs,int increase_denoising)40 static int delta_thresh(BLOCK_SIZE bs, int increase_denoising) {
41 (void)bs;
42 (void)increase_denoising;
43 return 4;
44 }
45
noise_motion_thresh(BLOCK_SIZE bs,int increase_denoising)46 static int noise_motion_thresh(BLOCK_SIZE bs, int increase_denoising) {
47 (void)bs;
48 (void)increase_denoising;
49 return 625;
50 }
51
sse_thresh(BLOCK_SIZE bs,int increase_denoising)52 static unsigned int sse_thresh(BLOCK_SIZE bs, int increase_denoising) {
53 return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 60 : 40);
54 }
55
sse_diff_thresh(BLOCK_SIZE bs,int increase_denoising,int motion_magnitude)56 static int sse_diff_thresh(BLOCK_SIZE bs, int increase_denoising,
57 int motion_magnitude) {
58 if (motion_magnitude >
59 noise_motion_thresh(bs, increase_denoising)) {
60 return 0;
61 } else {
62 return (1 << num_pels_log2_lookup[bs]) * 20;
63 }
64 }
65
total_adj_weak_thresh(BLOCK_SIZE bs,int increase_denoising)66 static int total_adj_weak_thresh(BLOCK_SIZE bs, int increase_denoising) {
67 return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 3 : 2);
68 }
69
70 // TODO(jackychen): If increase_denoising is enabled in the future,
71 // we might need to update the code for calculating 'total_adj' in
72 // case the C code is not bit-exact with corresponding sse2 code.
vp9_denoiser_filter_c(const uint8_t * sig,int sig_stride,const uint8_t * mc_avg,int mc_avg_stride,uint8_t * avg,int avg_stride,int increase_denoising,BLOCK_SIZE bs,int motion_magnitude)73 int vp9_denoiser_filter_c(const uint8_t *sig, int sig_stride,
74 const uint8_t *mc_avg,
75 int mc_avg_stride,
76 uint8_t *avg, int avg_stride,
77 int increase_denoising,
78 BLOCK_SIZE bs,
79 int motion_magnitude) {
80 int r, c;
81 const uint8_t *sig_start = sig;
82 const uint8_t *mc_avg_start = mc_avg;
83 uint8_t *avg_start = avg;
84 int diff, adj, absdiff, delta;
85 int adj_val[] = {3, 4, 6};
86 int total_adj = 0;
87 int shift_inc = 1;
88
89 // If motion_magnitude is small, making the denoiser more aggressive by
90 // increasing the adjustment for each level. Add another increment for
91 // blocks that are labeled for increase denoising.
92 if (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) {
93 if (increase_denoising) {
94 shift_inc = 2;
95 }
96 adj_val[0] += shift_inc;
97 adj_val[1] += shift_inc;
98 adj_val[2] += shift_inc;
99 }
100
101 // First attempt to apply a strong temporal denoising filter.
102 for (r = 0; r < (4 << b_height_log2_lookup[bs]); ++r) {
103 for (c = 0; c < (4 << b_width_log2_lookup[bs]); ++c) {
104 diff = mc_avg[c] - sig[c];
105 absdiff = abs(diff);
106
107 if (absdiff <= absdiff_thresh(bs, increase_denoising)) {
108 avg[c] = mc_avg[c];
109 total_adj += diff;
110 } else {
111 switch (absdiff) {
112 case 4: case 5: case 6: case 7:
113 adj = adj_val[0];
114 break;
115 case 8: case 9: case 10: case 11:
116 case 12: case 13: case 14: case 15:
117 adj = adj_val[1];
118 break;
119 default:
120 adj = adj_val[2];
121 }
122 if (diff > 0) {
123 avg[c] = MIN(UINT8_MAX, sig[c] + adj);
124 total_adj += adj;
125 } else {
126 avg[c] = MAX(0, sig[c] - adj);
127 total_adj -= adj;
128 }
129 }
130 }
131 sig += sig_stride;
132 avg += avg_stride;
133 mc_avg += mc_avg_stride;
134 }
135
136 // If the strong filter did not modify the signal too much, we're all set.
137 if (abs(total_adj) <= total_adj_strong_thresh(bs, increase_denoising)) {
138 return FILTER_BLOCK;
139 }
140
141 // Otherwise, we try to dampen the filter if the delta is not too high.
142 delta = ((abs(total_adj) - total_adj_strong_thresh(bs, increase_denoising))
143 >> num_pels_log2_lookup[bs]) + 1;
144
145 if (delta >= delta_thresh(bs, increase_denoising)) {
146 return COPY_BLOCK;
147 }
148
149 mc_avg = mc_avg_start;
150 avg = avg_start;
151 sig = sig_start;
152 for (r = 0; r < (4 << b_height_log2_lookup[bs]); ++r) {
153 for (c = 0; c < (4 << b_width_log2_lookup[bs]); ++c) {
154 diff = mc_avg[c] - sig[c];
155 adj = abs(diff);
156 if (adj > delta) {
157 adj = delta;
158 }
159 if (diff > 0) {
160 // Diff positive means we made positive adjustment above
161 // (in first try/attempt), so now make negative adjustment to bring
162 // denoised signal down.
163 avg[c] = MAX(0, avg[c] - adj);
164 total_adj -= adj;
165 } else {
166 // Diff negative means we made negative adjustment above
167 // (in first try/attempt), so now make positive adjustment to bring
168 // denoised signal up.
169 avg[c] = MIN(UINT8_MAX, avg[c] + adj);
170 total_adj += adj;
171 }
172 }
173 sig += sig_stride;
174 avg += avg_stride;
175 mc_avg += mc_avg_stride;
176 }
177
178 // We can use the filter if it has been sufficiently dampened
179 if (abs(total_adj) <= total_adj_weak_thresh(bs, increase_denoising)) {
180 return FILTER_BLOCK;
181 }
182 return COPY_BLOCK;
183 }
184
block_start(uint8_t * framebuf,int stride,int mi_row,int mi_col)185 static uint8_t *block_start(uint8_t *framebuf, int stride,
186 int mi_row, int mi_col) {
187 return framebuf + (stride * mi_row * 8) + (mi_col * 8);
188 }
189
perform_motion_compensation(VP9_DENOISER * denoiser,MACROBLOCK * mb,BLOCK_SIZE bs,int increase_denoising,int mi_row,int mi_col,PICK_MODE_CONTEXT * ctx,int * motion_magnitude)190 static VP9_DENOISER_DECISION perform_motion_compensation(VP9_DENOISER *denoiser,
191 MACROBLOCK *mb,
192 BLOCK_SIZE bs,
193 int increase_denoising,
194 int mi_row,
195 int mi_col,
196 PICK_MODE_CONTEXT *ctx,
197 int *motion_magnitude
198 ) {
199 int mv_col, mv_row;
200 int sse_diff = ctx->zeromv_sse - ctx->newmv_sse;
201 MV_REFERENCE_FRAME frame;
202 MACROBLOCKD *filter_mbd = &mb->e_mbd;
203 MB_MODE_INFO *mbmi = &filter_mbd->mi[0]->mbmi;
204 MB_MODE_INFO saved_mbmi;
205 int i, j;
206 struct buf_2d saved_dst[MAX_MB_PLANE];
207 struct buf_2d saved_pre[MAX_MB_PLANE][2]; // 2 pre buffers
208
209 mv_col = ctx->best_sse_mv.as_mv.col;
210 mv_row = ctx->best_sse_mv.as_mv.row;
211 *motion_magnitude = mv_row * mv_row + mv_col * mv_col;
212 frame = ctx->best_reference_frame;
213
214 saved_mbmi = *mbmi;
215
216 // If the best reference frame uses inter-prediction and there is enough of a
217 // difference in sum-squared-error, use it.
218 if (frame != INTRA_FRAME &&
219 sse_diff > sse_diff_thresh(bs, increase_denoising, *motion_magnitude)) {
220 mbmi->ref_frame[0] = ctx->best_reference_frame;
221 mbmi->mode = ctx->best_sse_inter_mode;
222 mbmi->mv[0] = ctx->best_sse_mv;
223 } else {
224 // Otherwise, use the zero reference frame.
225 frame = ctx->best_zeromv_reference_frame;
226
227 mbmi->ref_frame[0] = ctx->best_zeromv_reference_frame;
228 mbmi->mode = ZEROMV;
229 mbmi->mv[0].as_int = 0;
230
231 ctx->best_sse_inter_mode = ZEROMV;
232 ctx->best_sse_mv.as_int = 0;
233 ctx->newmv_sse = ctx->zeromv_sse;
234 }
235
236 if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
237 // Restore everything to its original state
238 *mbmi = saved_mbmi;
239 return COPY_BLOCK;
240 }
241 if (*motion_magnitude >
242 (noise_motion_thresh(bs, increase_denoising) << 3)) {
243 // Restore everything to its original state
244 *mbmi = saved_mbmi;
245 return COPY_BLOCK;
246 }
247
248 // We will restore these after motion compensation.
249 for (i = 0; i < MAX_MB_PLANE; ++i) {
250 for (j = 0; j < 2; ++j) {
251 saved_pre[i][j] = filter_mbd->plane[i].pre[j];
252 }
253 saved_dst[i] = filter_mbd->plane[i].dst;
254 }
255
256 // Set the pointers in the MACROBLOCKD to point to the buffers in the denoiser
257 // struct.
258 for (j = 0; j < 2; ++j) {
259 filter_mbd->plane[0].pre[j].buf =
260 block_start(denoiser->running_avg_y[frame].y_buffer,
261 denoiser->running_avg_y[frame].y_stride,
262 mi_row, mi_col);
263 filter_mbd->plane[0].pre[j].stride =
264 denoiser->running_avg_y[frame].y_stride;
265 filter_mbd->plane[1].pre[j].buf =
266 block_start(denoiser->running_avg_y[frame].u_buffer,
267 denoiser->running_avg_y[frame].uv_stride,
268 mi_row, mi_col);
269 filter_mbd->plane[1].pre[j].stride =
270 denoiser->running_avg_y[frame].uv_stride;
271 filter_mbd->plane[2].pre[j].buf =
272 block_start(denoiser->running_avg_y[frame].v_buffer,
273 denoiser->running_avg_y[frame].uv_stride,
274 mi_row, mi_col);
275 filter_mbd->plane[2].pre[j].stride =
276 denoiser->running_avg_y[frame].uv_stride;
277 }
278 filter_mbd->plane[0].dst.buf =
279 block_start(denoiser->mc_running_avg_y.y_buffer,
280 denoiser->mc_running_avg_y.y_stride,
281 mi_row, mi_col);
282 filter_mbd->plane[0].dst.stride = denoiser->mc_running_avg_y.y_stride;
283 filter_mbd->plane[1].dst.buf =
284 block_start(denoiser->mc_running_avg_y.u_buffer,
285 denoiser->mc_running_avg_y.uv_stride,
286 mi_row, mi_col);
287 filter_mbd->plane[1].dst.stride = denoiser->mc_running_avg_y.uv_stride;
288 filter_mbd->plane[2].dst.buf =
289 block_start(denoiser->mc_running_avg_y.v_buffer,
290 denoiser->mc_running_avg_y.uv_stride,
291 mi_row, mi_col);
292 filter_mbd->plane[2].dst.stride = denoiser->mc_running_avg_y.uv_stride;
293
294 vp9_build_inter_predictors_sby(filter_mbd, mv_row, mv_col, bs);
295
296 // Restore everything to its original state
297 *mbmi = saved_mbmi;
298 for (i = 0; i < MAX_MB_PLANE; ++i) {
299 for (j = 0; j < 2; ++j) {
300 filter_mbd->plane[i].pre[j] = saved_pre[i][j];
301 }
302 filter_mbd->plane[i].dst = saved_dst[i];
303 }
304
305 mv_row = ctx->best_sse_mv.as_mv.row;
306 mv_col = ctx->best_sse_mv.as_mv.col;
307
308 return FILTER_BLOCK;
309 }
310
vp9_denoiser_denoise(VP9_DENOISER * denoiser,MACROBLOCK * mb,int mi_row,int mi_col,BLOCK_SIZE bs,PICK_MODE_CONTEXT * ctx)311 void vp9_denoiser_denoise(VP9_DENOISER *denoiser, MACROBLOCK *mb,
312 int mi_row, int mi_col, BLOCK_SIZE bs,
313 PICK_MODE_CONTEXT *ctx) {
314 int motion_magnitude = 0;
315 VP9_DENOISER_DECISION decision = FILTER_BLOCK;
316 YV12_BUFFER_CONFIG avg = denoiser->running_avg_y[INTRA_FRAME];
317 YV12_BUFFER_CONFIG mc_avg = denoiser->mc_running_avg_y;
318 uint8_t *avg_start = block_start(avg.y_buffer, avg.y_stride, mi_row, mi_col);
319 uint8_t *mc_avg_start = block_start(mc_avg.y_buffer, mc_avg.y_stride,
320 mi_row, mi_col);
321 struct buf_2d src = mb->plane[0].src;
322
323 decision = perform_motion_compensation(denoiser, mb, bs,
324 denoiser->increase_denoising,
325 mi_row, mi_col, ctx,
326 &motion_magnitude);
327
328 if (decision == FILTER_BLOCK) {
329 decision = vp9_denoiser_filter(src.buf, src.stride,
330 mc_avg_start, mc_avg.y_stride,
331 avg_start, avg.y_stride,
332 0, bs, motion_magnitude);
333 }
334
335 if (decision == FILTER_BLOCK) {
336 vpx_convolve_copy(avg_start, avg.y_stride, src.buf, src.stride,
337 NULL, 0, NULL, 0,
338 num_4x4_blocks_wide_lookup[bs] << 2,
339 num_4x4_blocks_high_lookup[bs] << 2);
340 } else { // COPY_BLOCK
341 vpx_convolve_copy(src.buf, src.stride, avg_start, avg.y_stride,
342 NULL, 0, NULL, 0,
343 num_4x4_blocks_wide_lookup[bs] << 2,
344 num_4x4_blocks_high_lookup[bs] << 2);
345 }
346 }
347
copy_frame(YV12_BUFFER_CONFIG dest,const YV12_BUFFER_CONFIG src)348 static void copy_frame(YV12_BUFFER_CONFIG dest, const YV12_BUFFER_CONFIG src) {
349 int r;
350 const uint8_t *srcbuf = src.y_buffer;
351 uint8_t *destbuf = dest.y_buffer;
352
353 assert(dest.y_width == src.y_width);
354 assert(dest.y_height == src.y_height);
355
356 for (r = 0; r < dest.y_height; ++r) {
357 memcpy(destbuf, srcbuf, dest.y_width);
358 destbuf += dest.y_stride;
359 srcbuf += src.y_stride;
360 }
361 }
362
swap_frame_buffer(YV12_BUFFER_CONFIG * dest,YV12_BUFFER_CONFIG * src)363 static void swap_frame_buffer(YV12_BUFFER_CONFIG *dest,
364 YV12_BUFFER_CONFIG *src) {
365 uint8_t *tmp_buf = dest->y_buffer;
366 assert(dest->y_width == src->y_width);
367 assert(dest->y_height == src->y_height);
368 dest->y_buffer = src->y_buffer;
369 src->y_buffer = tmp_buf;
370 }
371
vp9_denoiser_update_frame_info(VP9_DENOISER * denoiser,YV12_BUFFER_CONFIG src,FRAME_TYPE frame_type,int refresh_alt_ref_frame,int refresh_golden_frame,int refresh_last_frame)372 void vp9_denoiser_update_frame_info(VP9_DENOISER *denoiser,
373 YV12_BUFFER_CONFIG src,
374 FRAME_TYPE frame_type,
375 int refresh_alt_ref_frame,
376 int refresh_golden_frame,
377 int refresh_last_frame) {
378 if (frame_type == KEY_FRAME) {
379 int i;
380 // Start at 1 so as not to overwrite the INTRA_FRAME
381 for (i = 1; i < MAX_REF_FRAMES; ++i)
382 copy_frame(denoiser->running_avg_y[i], src);
383 return;
384 }
385
386 /* For non key frames */
387 if (refresh_alt_ref_frame) {
388 swap_frame_buffer(&denoiser->running_avg_y[ALTREF_FRAME],
389 &denoiser->running_avg_y[INTRA_FRAME]);
390 }
391 if (refresh_golden_frame) {
392 swap_frame_buffer(&denoiser->running_avg_y[GOLDEN_FRAME],
393 &denoiser->running_avg_y[INTRA_FRAME]);
394 }
395 if (refresh_last_frame) {
396 swap_frame_buffer(&denoiser->running_avg_y[LAST_FRAME],
397 &denoiser->running_avg_y[INTRA_FRAME]);
398 }
399 }
400
vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT * ctx)401 void vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {
402 ctx->zeromv_sse = UINT_MAX;
403 ctx->newmv_sse = UINT_MAX;
404 }
405
vp9_denoiser_update_frame_stats(MB_MODE_INFO * mbmi,unsigned int sse,PREDICTION_MODE mode,PICK_MODE_CONTEXT * ctx)406 void vp9_denoiser_update_frame_stats(MB_MODE_INFO *mbmi, unsigned int sse,
407 PREDICTION_MODE mode,
408 PICK_MODE_CONTEXT *ctx) {
409 // TODO(tkopp): Use both MVs if possible
410 if (mbmi->mv[0].as_int == 0 && sse < ctx->zeromv_sse) {
411 ctx->zeromv_sse = sse;
412 ctx->best_zeromv_reference_frame = mbmi->ref_frame[0];
413 }
414
415 if (mbmi->mv[0].as_int != 0 && sse < ctx->newmv_sse) {
416 ctx->newmv_sse = sse;
417 ctx->best_sse_inter_mode = mode;
418 ctx->best_sse_mv = mbmi->mv[0];
419 ctx->best_reference_frame = mbmi->ref_frame[0];
420 }
421 }
422
vp9_denoiser_alloc(VP9_DENOISER * denoiser,int width,int height,int ssx,int ssy,int use_highbitdepth,int border)423 int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height,
424 int ssx, int ssy,
425 #if CONFIG_VP9_HIGHBITDEPTH
426 int use_highbitdepth,
427 #endif
428 int border) {
429 int i, fail;
430 const int legacy_byte_alignment = 0;
431 assert(denoiser != NULL);
432
433 for (i = 0; i < MAX_REF_FRAMES; ++i) {
434 fail = vpx_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
435 ssx, ssy,
436 #if CONFIG_VP9_HIGHBITDEPTH
437 use_highbitdepth,
438 #endif
439 border, legacy_byte_alignment);
440 if (fail) {
441 vp9_denoiser_free(denoiser);
442 return 1;
443 }
444 #ifdef OUTPUT_YUV_DENOISED
445 make_grayscale(&denoiser->running_avg_y[i]);
446 #endif
447 }
448
449 fail = vpx_alloc_frame_buffer(&denoiser->mc_running_avg_y, width, height,
450 ssx, ssy,
451 #if CONFIG_VP9_HIGHBITDEPTH
452 use_highbitdepth,
453 #endif
454 border, legacy_byte_alignment);
455 if (fail) {
456 vp9_denoiser_free(denoiser);
457 return 1;
458 }
459 #ifdef OUTPUT_YUV_DENOISED
460 make_grayscale(&denoiser->running_avg_y[i]);
461 #endif
462 denoiser->increase_denoising = 0;
463 denoiser->frame_buffer_initialized = 1;
464
465 return 0;
466 }
467
vp9_denoiser_free(VP9_DENOISER * denoiser)468 void vp9_denoiser_free(VP9_DENOISER *denoiser) {
469 int i;
470 denoiser->frame_buffer_initialized = 0;
471 if (denoiser == NULL) {
472 return;
473 }
474 for (i = 0; i < MAX_REF_FRAMES; ++i) {
475 vpx_free_frame_buffer(&denoiser->running_avg_y[i]);
476 }
477 vpx_free_frame_buffer(&denoiser->mc_running_avg_y);
478 }
479
480 #ifdef OUTPUT_YUV_DENOISED
make_grayscale(YV12_BUFFER_CONFIG * yuv)481 static void make_grayscale(YV12_BUFFER_CONFIG *yuv) {
482 int r, c;
483 uint8_t *u = yuv->u_buffer;
484 uint8_t *v = yuv->v_buffer;
485
486 for (r = 0; r < yuv->uv_height; ++r) {
487 for (c = 0; c < yuv->uv_width; ++c) {
488 u[c] = UINT8_MAX / 2;
489 v[c] = UINT8_MAX / 2;
490 }
491 u += yuv->uv_stride;
492 v += yuv->uv_stride;
493 }
494 }
495 #endif
496