• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2015 The WebM project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include <assert.h>
12 #include <limits.h>
13 #include <math.h>
14 
15 #include "./vpx_dsp_rtcd.h"
16 #include "vpx_dsp/vpx_dsp_common.h"
17 #include "vpx_scale/yv12config.h"
18 #include "vpx/vpx_integer.h"
19 #include "vp9/common/vp9_reconinter.h"
20 #include "vp9/encoder/vp9_context_tree.h"
21 #include "vp9/encoder/vp9_noise_estimate.h"
22 #include "vp9/encoder/vp9_encoder.h"
23 
24 #if CONFIG_VP9_TEMPORAL_DENOISING
25 // For SVC: only do noise estimation on top spatial layer.
noise_est_svc(const struct VP9_COMP * const cpi)26 static INLINE int noise_est_svc(const struct VP9_COMP *const cpi) {
27   return (!cpi->use_svc ||
28           (cpi->use_svc &&
29            cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1));
30 }
31 #endif
32 
vp9_noise_estimate_init(NOISE_ESTIMATE * const ne,int width,int height)33 void vp9_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height) {
34   ne->enabled = 0;
35   ne->level = (width * height < 1280 * 720) ? kLowLow : kLow;
36   ne->value = 0;
37   ne->count = 0;
38   ne->thresh = 90;
39   ne->last_w = 0;
40   ne->last_h = 0;
41   if (width * height >= 1920 * 1080) {
42     ne->thresh = 200;
43   } else if (width * height >= 1280 * 720) {
44     ne->thresh = 140;
45   } else if (width * height >= 640 * 360) {
46     ne->thresh = 115;
47   }
48   ne->num_frames_estimate = 15;
49   ne->adapt_thresh = (3 * ne->thresh) >> 1;
50 }
51 
enable_noise_estimation(VP9_COMP * const cpi)52 static int enable_noise_estimation(VP9_COMP *const cpi) {
53 #if CONFIG_VP9_HIGHBITDEPTH
54   if (cpi->common.use_highbitdepth) return 0;
55 #endif
56 // Enable noise estimation if denoising is on.
57 #if CONFIG_VP9_TEMPORAL_DENOISING
58   if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
59       cpi->common.width >= 320 && cpi->common.height >= 180)
60     return 1;
61 #endif
62   // Only allow noise estimate under certain encoding mode.
63   // Enabled for 1 pass CBR, speed >=5, and if resolution is same as original.
64   // Not enabled for SVC mode and screen_content_mode.
65   // Not enabled for low resolutions.
66   if (cpi->oxcf.pass == 0 && cpi->oxcf.rc_mode == VPX_CBR &&
67       cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cpi->oxcf.speed >= 5 &&
68       cpi->resize_state == ORIG && cpi->resize_pending == 0 && !cpi->use_svc &&
69       cpi->oxcf.content != VP9E_CONTENT_SCREEN &&
70       cpi->common.width * cpi->common.height >= 640 * 360)
71     return 1;
72   else
73     return 0;
74 }
75 
76 #if CONFIG_VP9_TEMPORAL_DENOISING
copy_frame(YV12_BUFFER_CONFIG * const dest,const YV12_BUFFER_CONFIG * const src)77 static void copy_frame(YV12_BUFFER_CONFIG *const dest,
78                        const YV12_BUFFER_CONFIG *const src) {
79   int r;
80   const uint8_t *srcbuf = src->y_buffer;
81   uint8_t *destbuf = dest->y_buffer;
82 
83   assert(dest->y_width == src->y_width);
84   assert(dest->y_height == src->y_height);
85 
86   for (r = 0; r < dest->y_height; ++r) {
87     memcpy(destbuf, srcbuf, dest->y_width);
88     destbuf += dest->y_stride;
89     srcbuf += src->y_stride;
90   }
91 }
92 #endif  // CONFIG_VP9_TEMPORAL_DENOISING
93 
vp9_noise_estimate_extract_level(NOISE_ESTIMATE * const ne)94 NOISE_LEVEL vp9_noise_estimate_extract_level(NOISE_ESTIMATE *const ne) {
95   int noise_level = kLowLow;
96   if (ne->value > (ne->thresh << 1)) {
97     noise_level = kHigh;
98   } else {
99     if (ne->value > ne->thresh)
100       noise_level = kMedium;
101     else if (ne->value > (ne->thresh >> 1))
102       noise_level = kLow;
103     else
104       noise_level = kLowLow;
105   }
106   return noise_level;
107 }
108 
vp9_update_noise_estimate(VP9_COMP * const cpi)109 void vp9_update_noise_estimate(VP9_COMP *const cpi) {
110   const VP9_COMMON *const cm = &cpi->common;
111   NOISE_ESTIMATE *const ne = &cpi->noise_estimate;
112   const int low_res = (cm->width <= 352 && cm->height <= 288);
113   // Estimate of noise level every frame_period frames.
114   int frame_period = 8;
115   int thresh_consec_zeromv = 6;
116   int frame_counter = cm->current_video_frame;
117   // Estimate is between current source and last source.
118   YV12_BUFFER_CONFIG *last_source = cpi->Last_Source;
119 #if CONFIG_VP9_TEMPORAL_DENOISING
120   if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) {
121     last_source = &cpi->denoiser.last_source;
122     // Tune these thresholds for different resolutions when denoising is
123     // enabled.
124     if (cm->width > 640 && cm->width <= 1920) {
125       thresh_consec_zeromv = 2;
126     }
127   }
128 #endif
129   ne->enabled = enable_noise_estimation(cpi);
130   if (cpi->svc.number_spatial_layers > 1)
131     frame_counter = cpi->svc.current_superframe;
132   if (!ne->enabled || frame_counter % frame_period != 0 ||
133       last_source == NULL ||
134       (cpi->svc.number_spatial_layers == 1 &&
135        (ne->last_w != cm->width || ne->last_h != cm->height))) {
136 #if CONFIG_VP9_TEMPORAL_DENOISING
137     if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
138       copy_frame(&cpi->denoiser.last_source, cpi->Source);
139 #endif
140     if (last_source != NULL) {
141       ne->last_w = cm->width;
142       ne->last_h = cm->height;
143     }
144     return;
145   } else if (frame_counter > 60 && cpi->svc.num_encoded_top_layer > 1 &&
146              cpi->rc.frames_since_key > cpi->svc.number_spatial_layers &&
147              cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 &&
148              cpi->rc.avg_frame_low_motion < (low_res ? 60 : 40)) {
149     // Force noise estimation to 0 and denoiser off if content has high motion.
150     ne->level = kLowLow;
151     ne->count = 0;
152     ne->num_frames_estimate = 10;
153 #if CONFIG_VP9_TEMPORAL_DENOISING
154     if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
155         cpi->svc.current_superframe > 1) {
156       vp9_denoiser_set_noise_level(cpi, ne->level);
157       copy_frame(&cpi->denoiser.last_source, cpi->Source);
158     }
159 #endif
160     return;
161   } else {
162     unsigned int bin_size = 100;
163     unsigned int hist[MAX_VAR_HIST_BINS] = { 0 };
164     unsigned int hist_avg[MAX_VAR_HIST_BINS];
165     unsigned int max_bin = 0;
166     unsigned int max_bin_count = 0;
167     unsigned int bin_cnt;
168     int bsize = BLOCK_16X16;
169     // Loop over sub-sample of 16x16 blocks of frame, and for blocks that have
170     // been encoded as zero/small mv at least x consecutive frames, compute
171     // the variance to update estimate of noise in the source.
172     const uint8_t *src_y = cpi->Source->y_buffer;
173     const int src_ystride = cpi->Source->y_stride;
174     const uint8_t *last_src_y = last_source->y_buffer;
175     const int last_src_ystride = last_source->y_stride;
176     const uint8_t *src_u = cpi->Source->u_buffer;
177     const uint8_t *src_v = cpi->Source->v_buffer;
178     const int src_uvstride = cpi->Source->uv_stride;
179     int mi_row, mi_col;
180     int num_low_motion = 0;
181     int frame_low_motion = 1;
182     for (mi_row = 0; mi_row < cm->mi_rows; mi_row++) {
183       for (mi_col = 0; mi_col < cm->mi_cols; mi_col++) {
184         int bl_index = mi_row * cm->mi_cols + mi_col;
185         if (cpi->consec_zero_mv[bl_index] > thresh_consec_zeromv)
186           num_low_motion++;
187       }
188     }
189     if (num_low_motion < ((3 * cm->mi_rows * cm->mi_cols) >> 3))
190       frame_low_motion = 0;
191     for (mi_row = 0; mi_row < cm->mi_rows; mi_row++) {
192       for (mi_col = 0; mi_col < cm->mi_cols; mi_col++) {
193         // 16x16 blocks, 1/4 sample of frame.
194         if (mi_row % 4 == 0 && mi_col % 4 == 0 && mi_row < cm->mi_rows - 1 &&
195             mi_col < cm->mi_cols - 1) {
196           int bl_index = mi_row * cm->mi_cols + mi_col;
197           int bl_index1 = bl_index + 1;
198           int bl_index2 = bl_index + cm->mi_cols;
199           int bl_index3 = bl_index2 + 1;
200           int consec_zeromv =
201               VPXMIN(cpi->consec_zero_mv[bl_index],
202                      VPXMIN(cpi->consec_zero_mv[bl_index1],
203                             VPXMIN(cpi->consec_zero_mv[bl_index2],
204                                    cpi->consec_zero_mv[bl_index3])));
205           // Only consider blocks that are likely steady background. i.e, have
206           // been encoded as zero/low motion x (= thresh_consec_zeromv) frames
207           // in a row. consec_zero_mv[] defined for 8x8 blocks, so consider all
208           // 4 sub-blocks for 16x16 block. And exclude this frame if
209           // high_source_sad is true (i.e., scene/content change).
210           if (frame_low_motion && consec_zeromv > thresh_consec_zeromv &&
211               !cpi->rc.high_source_sad &&
212               !cpi->svc.high_source_sad_superframe) {
213             int is_skin = 0;
214             if (cpi->use_skin_detection) {
215               is_skin =
216                   vp9_compute_skin_block(src_y, src_u, src_v, src_ystride,
217                                          src_uvstride, bsize, consec_zeromv, 0);
218             }
219             if (!is_skin) {
220               unsigned int sse;
221               // Compute variance between co-located blocks from current and
222               // last input frames.
223               unsigned int variance = cpi->fn_ptr[bsize].vf(
224                   src_y, src_ystride, last_src_y, last_src_ystride, &sse);
225               unsigned int hist_index = variance / bin_size;
226               if (hist_index < MAX_VAR_HIST_BINS)
227                 hist[hist_index]++;
228               else if (hist_index < 3 * (MAX_VAR_HIST_BINS >> 1))
229                 hist[MAX_VAR_HIST_BINS - 1]++;  // Account for the tail
230             }
231           }
232         }
233         src_y += 8;
234         last_src_y += 8;
235         src_u += 4;
236         src_v += 4;
237       }
238       src_y += (src_ystride << 3) - (cm->mi_cols << 3);
239       last_src_y += (last_src_ystride << 3) - (cm->mi_cols << 3);
240       src_u += (src_uvstride << 2) - (cm->mi_cols << 2);
241       src_v += (src_uvstride << 2) - (cm->mi_cols << 2);
242     }
243     ne->last_w = cm->width;
244     ne->last_h = cm->height;
245     // Adjust histogram to account for effect that histogram flattens
246     // and shifts to zero as scene darkens.
247     if (hist[0] > 10 && (hist[MAX_VAR_HIST_BINS - 1] > hist[0] >> 2)) {
248       hist[0] = 0;
249       hist[1] >>= 2;
250       hist[2] >>= 2;
251       hist[3] >>= 2;
252       hist[4] >>= 1;
253       hist[5] >>= 1;
254       hist[6] = 3 * hist[6] >> 1;
255       hist[MAX_VAR_HIST_BINS - 1] >>= 1;
256     }
257 
258     // Average hist[] and find largest bin
259     for (bin_cnt = 0; bin_cnt < MAX_VAR_HIST_BINS; bin_cnt++) {
260       if (bin_cnt == 0)
261         hist_avg[bin_cnt] = (hist[0] + hist[1] + hist[2]) / 3;
262       else if (bin_cnt == MAX_VAR_HIST_BINS - 1)
263         hist_avg[bin_cnt] = hist[MAX_VAR_HIST_BINS - 1] >> 2;
264       else if (bin_cnt == MAX_VAR_HIST_BINS - 2)
265         hist_avg[bin_cnt] = (hist[bin_cnt - 1] + 2 * hist[bin_cnt] +
266                              (hist[bin_cnt + 1] >> 1) + 2) >>
267                             2;
268       else
269         hist_avg[bin_cnt] =
270             (hist[bin_cnt - 1] + 2 * hist[bin_cnt] + hist[bin_cnt + 1] + 2) >>
271             2;
272 
273       if (hist_avg[bin_cnt] > max_bin_count) {
274         max_bin_count = hist_avg[bin_cnt];
275         max_bin = bin_cnt;
276       }
277     }
278 
279     // Scale by 40 to work with existing thresholds
280     ne->value = (int)((3 * ne->value + max_bin * 40) >> 2);
281     // Quickly increase VNR strength when the noise level increases suddenly.
282     if (ne->level < kMedium && ne->value > ne->adapt_thresh) {
283       ne->count = ne->num_frames_estimate;
284     } else {
285       ne->count++;
286     }
287     if (ne->count == ne->num_frames_estimate) {
288       // Reset counter and check noise level condition.
289       ne->num_frames_estimate = 30;
290       ne->count = 0;
291       ne->level = vp9_noise_estimate_extract_level(ne);
292 #if CONFIG_VP9_TEMPORAL_DENOISING
293       if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
294         vp9_denoiser_set_noise_level(cpi, ne->level);
295 #endif
296     }
297   }
298 #if CONFIG_VP9_TEMPORAL_DENOISING
299   if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
300     copy_frame(&cpi->denoiser.last_source, cpi->Source);
301 #endif
302 }
303