• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "webrtc/modules/audio_coding/neteq/time_stretch.h"
12 
13 #include <algorithm>  // min, max
14 
15 #include "webrtc/base/safe_conversions.h"
16 #include "webrtc/base/scoped_ptr.h"
17 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
18 #include "webrtc/modules/audio_coding/neteq/background_noise.h"
19 #include "webrtc/modules/audio_coding/neteq/dsp_helper.h"
20 
21 namespace webrtc {
22 
Process(const int16_t * input,size_t input_len,bool fast_mode,AudioMultiVector * output,size_t * length_change_samples)23 TimeStretch::ReturnCodes TimeStretch::Process(const int16_t* input,
24                                               size_t input_len,
25                                               bool fast_mode,
26                                               AudioMultiVector* output,
27                                               size_t* length_change_samples) {
28   // Pre-calculate common multiplication with |fs_mult_|.
29   size_t fs_mult_120 =
30       static_cast<size_t>(fs_mult_ * 120);  // Corresponds to 15 ms.
31 
32   const int16_t* signal;
33   rtc::scoped_ptr<int16_t[]> signal_array;
34   size_t signal_len;
35   if (num_channels_ == 1) {
36     signal = input;
37     signal_len = input_len;
38   } else {
39     // We want |signal| to be only the first channel of |input|, which is
40     // interleaved. Thus, we take the first sample, skip forward |num_channels|
41     // samples, and continue like that.
42     signal_len = input_len / num_channels_;
43     signal_array.reset(new int16_t[signal_len]);
44     signal = signal_array.get();
45     size_t j = master_channel_;
46     for (size_t i = 0; i < signal_len; ++i) {
47       signal_array[i] = input[j];
48       j += num_channels_;
49     }
50   }
51 
52   // Find maximum absolute value of input signal.
53   max_input_value_ = WebRtcSpl_MaxAbsValueW16(signal, signal_len);
54 
55   // Downsample to 4 kHz sample rate and calculate auto-correlation.
56   DspHelper::DownsampleTo4kHz(signal, signal_len, kDownsampledLen,
57                               sample_rate_hz_, true /* compensate delay*/,
58                               downsampled_input_);
59   AutoCorrelation();
60 
61   // Find the strongest correlation peak.
62   static const size_t kNumPeaks = 1;
63   size_t peak_index;
64   int16_t peak_value;
65   DspHelper::PeakDetection(auto_correlation_, kCorrelationLen, kNumPeaks,
66                            fs_mult_, &peak_index, &peak_value);
67   // Assert that |peak_index| stays within boundaries.
68   assert(peak_index <= (2 * kCorrelationLen - 1) * fs_mult_);
69 
70   // Compensate peak_index for displaced starting position. The displacement
71   // happens in AutoCorrelation(). Here, |kMinLag| is in the down-sampled 4 kHz
72   // domain, while the |peak_index| is in the original sample rate; hence, the
73   // multiplication by fs_mult_ * 2.
74   peak_index += kMinLag * fs_mult_ * 2;
75   // Assert that |peak_index| stays within boundaries.
76   assert(peak_index >= static_cast<size_t>(20 * fs_mult_));
77   assert(peak_index <= 20 * fs_mult_ + (2 * kCorrelationLen - 1) * fs_mult_);
78 
79   // Calculate scaling to ensure that |peak_index| samples can be square-summed
80   // without overflowing.
81   int scaling = 31 - WebRtcSpl_NormW32(max_input_value_ * max_input_value_) -
82       WebRtcSpl_NormW32(static_cast<int32_t>(peak_index));
83   scaling = std::max(0, scaling);
84 
85   // |vec1| starts at 15 ms minus one pitch period.
86   const int16_t* vec1 = &signal[fs_mult_120 - peak_index];
87   // |vec2| start at 15 ms.
88   const int16_t* vec2 = &signal[fs_mult_120];
89   // Calculate energies for |vec1| and |vec2|, assuming they both contain
90   // |peak_index| samples.
91   int32_t vec1_energy =
92       WebRtcSpl_DotProductWithScale(vec1, vec1, peak_index, scaling);
93   int32_t vec2_energy =
94       WebRtcSpl_DotProductWithScale(vec2, vec2, peak_index, scaling);
95 
96   // Calculate cross-correlation between |vec1| and |vec2|.
97   int32_t cross_corr =
98       WebRtcSpl_DotProductWithScale(vec1, vec2, peak_index, scaling);
99 
100   // Check if the signal seems to be active speech or not (simple VAD).
101   bool active_speech = SpeechDetection(vec1_energy, vec2_energy, peak_index,
102                                        scaling);
103 
104   int16_t best_correlation;
105   if (!active_speech) {
106     SetParametersForPassiveSpeech(signal_len, &best_correlation, &peak_index);
107   } else {
108     // Calculate correlation:
109     // cross_corr / sqrt(vec1_energy * vec2_energy).
110 
111     // Start with calculating scale values.
112     int energy1_scale = std::max(0, 16 - WebRtcSpl_NormW32(vec1_energy));
113     int energy2_scale = std::max(0, 16 - WebRtcSpl_NormW32(vec2_energy));
114 
115     // Make sure total scaling is even (to simplify scale factor after sqrt).
116     if ((energy1_scale + energy2_scale) & 1) {
117       // The sum is odd.
118       energy1_scale += 1;
119     }
120 
121     // Scale energies to int16_t.
122     int16_t vec1_energy_int16 =
123         static_cast<int16_t>(vec1_energy >> energy1_scale);
124     int16_t vec2_energy_int16 =
125         static_cast<int16_t>(vec2_energy >> energy2_scale);
126 
127     // Calculate square-root of energy product.
128     int16_t sqrt_energy_prod = WebRtcSpl_SqrtFloor(vec1_energy_int16 *
129                                                    vec2_energy_int16);
130 
131     // Calculate cross_corr / sqrt(en1*en2) in Q14.
132     int temp_scale = 14 - (energy1_scale + energy2_scale) / 2;
133     cross_corr = WEBRTC_SPL_SHIFT_W32(cross_corr, temp_scale);
134     cross_corr = std::max(0, cross_corr);  // Don't use if negative.
135     best_correlation = WebRtcSpl_DivW32W16(cross_corr, sqrt_energy_prod);
136     // Make sure |best_correlation| is no larger than 1 in Q14.
137     best_correlation = std::min(static_cast<int16_t>(16384), best_correlation);
138   }
139 
140 
141   // Check accelerate criteria and stretch the signal.
142   ReturnCodes return_value =
143       CheckCriteriaAndStretch(input, input_len, peak_index, best_correlation,
144                               active_speech, fast_mode, output);
145   switch (return_value) {
146     case kSuccess:
147       *length_change_samples = peak_index;
148       break;
149     case kSuccessLowEnergy:
150       *length_change_samples = peak_index;
151       break;
152     case kNoStretch:
153     case kError:
154       *length_change_samples = 0;
155       break;
156   }
157   return return_value;
158 }
159 
AutoCorrelation()160 void TimeStretch::AutoCorrelation() {
161   // Set scaling factor for cross correlation to protect against overflow.
162   int scaling = kLogCorrelationLen - WebRtcSpl_NormW32(
163       max_input_value_ * max_input_value_);
164   scaling = std::max(0, scaling);
165 
166   // Calculate correlation from lag kMinLag to lag kMaxLag in 4 kHz domain.
167   int32_t auto_corr[kCorrelationLen];
168   WebRtcSpl_CrossCorrelation(auto_corr, &downsampled_input_[kMaxLag],
169                              &downsampled_input_[kMaxLag - kMinLag],
170                              kCorrelationLen, kMaxLag - kMinLag, scaling, -1);
171 
172   // Normalize correlation to 14 bits and write to |auto_correlation_|.
173   int32_t max_corr = WebRtcSpl_MaxAbsValueW32(auto_corr, kCorrelationLen);
174   scaling = std::max(0, 17 - WebRtcSpl_NormW32(max_corr));
175   WebRtcSpl_VectorBitShiftW32ToW16(auto_correlation_, kCorrelationLen,
176                                    auto_corr, scaling);
177 }
178 
SpeechDetection(int32_t vec1_energy,int32_t vec2_energy,size_t peak_index,int scaling) const179 bool TimeStretch::SpeechDetection(int32_t vec1_energy, int32_t vec2_energy,
180                                   size_t peak_index, int scaling) const {
181   // Check if the signal seems to be active speech or not (simple VAD).
182   // If (vec1_energy + vec2_energy) / (2 * peak_index) <=
183   // 8 * background_noise_energy, then we say that the signal contains no
184   // active speech.
185   // Rewrite the inequality as:
186   // (vec1_energy + vec2_energy) / 16 <= peak_index * background_noise_energy.
187   // The two sides of the inequality will be denoted |left_side| and
188   // |right_side|.
189   int32_t left_side = (vec1_energy + vec2_energy) / 16;
190   int32_t right_side;
191   if (background_noise_.initialized()) {
192     right_side = background_noise_.Energy(master_channel_);
193   } else {
194     // If noise parameters have not been estimated, use a fixed threshold.
195     right_side = 75000;
196   }
197   int right_scale = 16 - WebRtcSpl_NormW32(right_side);
198   right_scale = std::max(0, right_scale);
199   left_side = left_side >> right_scale;
200   right_side =
201       rtc::checked_cast<int32_t>(peak_index) * (right_side >> right_scale);
202 
203   // Scale |left_side| properly before comparing with |right_side|.
204   // (|scaling| is the scale factor before energy calculation, thus the scale
205   // factor for the energy is 2 * scaling.)
206   if (WebRtcSpl_NormW32(left_side) < 2 * scaling) {
207     // Cannot scale only |left_side|, must scale |right_side| too.
208     int temp_scale = WebRtcSpl_NormW32(left_side);
209     left_side = left_side << temp_scale;
210     right_side = right_side >> (2 * scaling - temp_scale);
211   } else {
212     left_side = left_side << 2 * scaling;
213   }
214   return left_side > right_side;
215 }
216 
217 }  // namespace webrtc
218