1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "webrtc/common_video/libyuv/include/scaler.h"
11 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
12 #include "webrtc/modules/video_processing/video_denoiser.h"
13
14 namespace webrtc {
15
VideoDenoiser(bool runtime_cpu_detection)16 VideoDenoiser::VideoDenoiser(bool runtime_cpu_detection)
17 : width_(0),
18 height_(0),
19 filter_(DenoiserFilter::Create(runtime_cpu_detection)) {}
20
TrailingReduction(int mb_rows,int mb_cols,const uint8_t * y_src,int stride_y,uint8_t * y_dst)21 void VideoDenoiser::TrailingReduction(int mb_rows,
22 int mb_cols,
23 const uint8_t* y_src,
24 int stride_y,
25 uint8_t* y_dst) {
26 for (int mb_row = 1; mb_row < mb_rows - 1; ++mb_row) {
27 for (int mb_col = 1; mb_col < mb_cols - 1; ++mb_col) {
28 int mb_index = mb_row * mb_cols + mb_col;
29 uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
30 const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
31 // If the number of denoised neighbors is less than a threshold,
32 // do NOT denoise for the block. Set different threshold for skin MB.
33 // The change of denoising status will not propagate.
34 if (metrics_[mb_index].is_skin) {
35 // The threshold is high (more strict) for non-skin MB where the
36 // trailing usually happen.
37 if (metrics_[mb_index].denoise &&
38 metrics_[mb_index + 1].denoise + metrics_[mb_index - 1].denoise +
39 metrics_[mb_index + mb_cols].denoise +
40 metrics_[mb_index - mb_cols].denoise <=
41 2) {
42 metrics_[mb_index].denoise = 0;
43 filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
44 }
45 } else if (metrics_[mb_index].denoise &&
46 metrics_[mb_index + 1].denoise +
47 metrics_[mb_index - 1].denoise +
48 metrics_[mb_index + mb_cols + 1].denoise +
49 metrics_[mb_index + mb_cols - 1].denoise +
50 metrics_[mb_index - mb_cols + 1].denoise +
51 metrics_[mb_index - mb_cols - 1].denoise +
52 metrics_[mb_index + mb_cols].denoise +
53 metrics_[mb_index - mb_cols].denoise <=
54 7) {
55 filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
56 }
57 }
58 }
59 }
60
DenoiseFrame(const VideoFrame & frame,VideoFrame * denoised_frame)61 void VideoDenoiser::DenoiseFrame(const VideoFrame& frame,
62 VideoFrame* denoised_frame) {
63 int stride_y = frame.stride(kYPlane);
64 int stride_u = frame.stride(kUPlane);
65 int stride_v = frame.stride(kVPlane);
66 // If previous width and height are different from current frame's, then no
67 // denoising for the current frame.
68 if (width_ != frame.width() || height_ != frame.height()) {
69 width_ = frame.width();
70 height_ = frame.height();
71 denoised_frame->CreateFrame(frame.buffer(kYPlane), frame.buffer(kUPlane),
72 frame.buffer(kVPlane), width_, height_,
73 stride_y, stride_u, stride_v);
74 // Setting time parameters to the output frame.
75 denoised_frame->set_timestamp(frame.timestamp());
76 denoised_frame->set_render_time_ms(frame.render_time_ms());
77 return;
78 }
79 // For 16x16 block.
80 int mb_cols = width_ >> 4;
81 int mb_rows = height_ >> 4;
82 if (metrics_.get() == nullptr)
83 metrics_.reset(new DenoiseMetrics[mb_cols * mb_rows]());
84 // Denoise on Y plane.
85 uint8_t* y_dst = denoised_frame->buffer(kYPlane);
86 uint8_t* u_dst = denoised_frame->buffer(kUPlane);
87 uint8_t* v_dst = denoised_frame->buffer(kVPlane);
88 const uint8_t* y_src = frame.buffer(kYPlane);
89 const uint8_t* u_src = frame.buffer(kUPlane);
90 const uint8_t* v_src = frame.buffer(kVPlane);
91 // Temporary buffer to store denoising result.
92 uint8_t y_tmp[16 * 16] = {0};
93 for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {
94 for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {
95 const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
96 uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
97 int mb_index = mb_row * mb_cols + mb_col;
98 // Denoise each MB at the very start and save the result to a temporary
99 // buffer.
100 if (filter_->MbDenoise(mb_dst, stride_y, y_tmp, 16, mb_src, stride_y, 0,
101 1) == FILTER_BLOCK) {
102 uint32_t thr_var = 0;
103 // Save var and sad to the buffer.
104 metrics_[mb_index].var = filter_->Variance16x8(
105 mb_dst, stride_y, y_tmp, 16, &metrics_[mb_index].sad);
106 // Get skin map.
107 metrics_[mb_index].is_skin = MbHasSkinColor(
108 y_src, u_src, v_src, stride_y, stride_u, stride_v, mb_row, mb_col);
109 // Variance threshold for skin/non-skin MB is different.
110 // Skin MB use a small threshold to reduce blockiness.
111 thr_var = metrics_[mb_index].is_skin ? 128 : 12 * 128;
112 if (metrics_[mb_index].var > thr_var) {
113 metrics_[mb_index].denoise = 0;
114 // Use the source MB.
115 filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
116 } else {
117 metrics_[mb_index].denoise = 1;
118 // Use the denoised MB.
119 filter_->CopyMem16x16(y_tmp, 16, mb_dst, stride_y);
120 }
121 } else {
122 metrics_[mb_index].denoise = 0;
123 filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
124 }
125 // Copy source U/V plane.
126 const uint8_t* mb_src_u =
127 u_src + (mb_row << 3) * stride_u + (mb_col << 3);
128 const uint8_t* mb_src_v =
129 v_src + (mb_row << 3) * stride_v + (mb_col << 3);
130 uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u + (mb_col << 3);
131 uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v + (mb_col << 3);
132 filter_->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u);
133 filter_->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v);
134 }
135 }
136 // Second round.
137 // This is to reduce the trailing artifact and blockiness by referring
138 // neighbors' denoising status.
139 TrailingReduction(mb_rows, mb_cols, y_src, stride_y, y_dst);
140
141 // Setting time parameters to the output frame.
142 denoised_frame->set_timestamp(frame.timestamp());
143 denoised_frame->set_render_time_ms(frame.render_time_ms());
144 return;
145 }
146
147 } // namespace webrtc
148