1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_mixer/frame_combiner.h"
12
13 #include <cstdint>
14 #include <initializer_list>
15 #include <numeric>
16 #include <string>
17 #include <type_traits>
18
19 #include "api/array_view.h"
20 #include "audio/utility/audio_frame_operations.h"
21 #include "modules/audio_mixer/gain_change_calculator.h"
22 #include "modules/audio_mixer/sine_wave_generator.h"
23 #include "rtc_base/checks.h"
24 #include "rtc_base/strings/string_builder.h"
25 #include "test/gmock.h"
26 #include "test/gtest.h"
27
28 namespace webrtc {
29
30 namespace {
31 using LimiterType = FrameCombiner::LimiterType;
32 struct FrameCombinerConfig {
33 bool use_limiter;
34 int sample_rate_hz;
35 int number_of_channels;
36 float wave_frequency;
37 };
38
ProduceDebugText(int sample_rate_hz,int number_of_channels,int number_of_sources)39 std::string ProduceDebugText(int sample_rate_hz,
40 int number_of_channels,
41 int number_of_sources) {
42 rtc::StringBuilder ss;
43 ss << "Sample rate: " << sample_rate_hz << " ,";
44 ss << "number of channels: " << number_of_channels << " ,";
45 ss << "number of sources: " << number_of_sources;
46 return ss.Release();
47 }
48
ProduceDebugText(const FrameCombinerConfig & config)49 std::string ProduceDebugText(const FrameCombinerConfig& config) {
50 rtc::StringBuilder ss;
51 ss << "Sample rate: " << config.sample_rate_hz << " ,";
52 ss << "number of channels: " << config.number_of_channels << " ,";
53 ss << "limiter active: " << (config.use_limiter ? "on" : "off") << " ,";
54 ss << "wave frequency: " << config.wave_frequency << " ,";
55 return ss.Release();
56 }
57
58 AudioFrame frame1;
59 AudioFrame frame2;
60 AudioFrame audio_frame_for_mixing;
61
SetUpFrames(int sample_rate_hz,int number_of_channels)62 void SetUpFrames(int sample_rate_hz, int number_of_channels) {
63 for (auto* frame : {&frame1, &frame2}) {
64 frame->UpdateFrame(0, nullptr, rtc::CheckedDivExact(sample_rate_hz, 100),
65 sample_rate_hz, AudioFrame::kNormalSpeech,
66 AudioFrame::kVadActive, number_of_channels);
67 }
68 }
69 } // namespace
70
71 // The limiter requires sample rate divisible by 2000.
TEST(FrameCombiner,BasicApiCallsLimiter)72 TEST(FrameCombiner, BasicApiCallsLimiter) {
73 FrameCombiner combiner(true);
74 for (const int rate : {8000, 18000, 34000, 48000}) {
75 for (const int number_of_channels : {1, 2, 4, 8}) {
76 const std::vector<AudioFrame*> all_frames = {&frame1, &frame2};
77 SetUpFrames(rate, number_of_channels);
78
79 for (const int number_of_frames : {0, 1, 2}) {
80 SCOPED_TRACE(
81 ProduceDebugText(rate, number_of_channels, number_of_frames));
82 const std::vector<AudioFrame*> frames_to_combine(
83 all_frames.begin(), all_frames.begin() + number_of_frames);
84 combiner.Combine(frames_to_combine, number_of_channels, rate,
85 frames_to_combine.size(), &audio_frame_for_mixing);
86 }
87 }
88 }
89 }
90
91 // There are DCHECKs in place to check for invalid parameters.
TEST(FrameCombinerDeathTest,DebugBuildCrashesWithManyChannels)92 TEST(FrameCombinerDeathTest, DebugBuildCrashesWithManyChannels) {
93 FrameCombiner combiner(true);
94 for (const int rate : {8000, 18000, 34000, 48000}) {
95 for (const int number_of_channels : {10, 20, 21}) {
96 if (static_cast<size_t>(rate / 100 * number_of_channels) >
97 AudioFrame::kMaxDataSizeSamples) {
98 continue;
99 }
100 const std::vector<AudioFrame*> all_frames = {&frame1, &frame2};
101 SetUpFrames(rate, number_of_channels);
102
103 const int number_of_frames = 2;
104 SCOPED_TRACE(
105 ProduceDebugText(rate, number_of_channels, number_of_frames));
106 const std::vector<AudioFrame*> frames_to_combine(
107 all_frames.begin(), all_frames.begin() + number_of_frames);
108 #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
109 EXPECT_DEATH(
110 combiner.Combine(frames_to_combine, number_of_channels, rate,
111 frames_to_combine.size(), &audio_frame_for_mixing),
112 "");
113 #elif !RTC_DCHECK_IS_ON
114 combiner.Combine(frames_to_combine, number_of_channels, rate,
115 frames_to_combine.size(), &audio_frame_for_mixing);
116 #endif
117 }
118 }
119 }
120
TEST(FrameCombinerDeathTest,DebugBuildCrashesWithHighRate)121 TEST(FrameCombinerDeathTest, DebugBuildCrashesWithHighRate) {
122 FrameCombiner combiner(true);
123 for (const int rate : {50000, 96000, 128000, 196000}) {
124 for (const int number_of_channels : {1, 2, 3}) {
125 if (static_cast<size_t>(rate / 100 * number_of_channels) >
126 AudioFrame::kMaxDataSizeSamples) {
127 continue;
128 }
129 const std::vector<AudioFrame*> all_frames = {&frame1, &frame2};
130 SetUpFrames(rate, number_of_channels);
131
132 const int number_of_frames = 2;
133 SCOPED_TRACE(
134 ProduceDebugText(rate, number_of_channels, number_of_frames));
135 const std::vector<AudioFrame*> frames_to_combine(
136 all_frames.begin(), all_frames.begin() + number_of_frames);
137 #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
138 EXPECT_DEATH(
139 combiner.Combine(frames_to_combine, number_of_channels, rate,
140 frames_to_combine.size(), &audio_frame_for_mixing),
141 "");
142 #elif !RTC_DCHECK_IS_ON
143 combiner.Combine(frames_to_combine, number_of_channels, rate,
144 frames_to_combine.size(), &audio_frame_for_mixing);
145 #endif
146 }
147 }
148 }
149
150 // With no limiter, the rate has to be divisible by 100 since we use
151 // 10 ms frames.
TEST(FrameCombiner,BasicApiCallsNoLimiter)152 TEST(FrameCombiner, BasicApiCallsNoLimiter) {
153 FrameCombiner combiner(false);
154 for (const int rate : {8000, 10000, 11000, 32000, 44100}) {
155 for (const int number_of_channels : {1, 2, 4, 8}) {
156 const std::vector<AudioFrame*> all_frames = {&frame1, &frame2};
157 SetUpFrames(rate, number_of_channels);
158
159 for (const int number_of_frames : {0, 1, 2}) {
160 SCOPED_TRACE(
161 ProduceDebugText(rate, number_of_channels, number_of_frames));
162 const std::vector<AudioFrame*> frames_to_combine(
163 all_frames.begin(), all_frames.begin() + number_of_frames);
164 combiner.Combine(frames_to_combine, number_of_channels, rate,
165 frames_to_combine.size(), &audio_frame_for_mixing);
166 }
167 }
168 }
169 }
170
TEST(FrameCombiner,CombiningZeroFramesShouldProduceSilence)171 TEST(FrameCombiner, CombiningZeroFramesShouldProduceSilence) {
172 FrameCombiner combiner(false);
173 for (const int rate : {8000, 10000, 11000, 32000, 44100}) {
174 for (const int number_of_channels : {1, 2}) {
175 SCOPED_TRACE(ProduceDebugText(rate, number_of_channels, 0));
176
177 const std::vector<AudioFrame*> frames_to_combine;
178 combiner.Combine(frames_to_combine, number_of_channels, rate,
179 frames_to_combine.size(), &audio_frame_for_mixing);
180
181 const int16_t* audio_frame_for_mixing_data =
182 audio_frame_for_mixing.data();
183 const std::vector<int16_t> mixed_data(
184 audio_frame_for_mixing_data,
185 audio_frame_for_mixing_data + number_of_channels * rate / 100);
186
187 const std::vector<int16_t> expected(number_of_channels * rate / 100, 0);
188 EXPECT_EQ(mixed_data, expected);
189 }
190 }
191 }
192
TEST(FrameCombiner,CombiningOneFrameShouldNotChangeFrame)193 TEST(FrameCombiner, CombiningOneFrameShouldNotChangeFrame) {
194 FrameCombiner combiner(false);
195 for (const int rate : {8000, 10000, 11000, 32000, 44100}) {
196 for (const int number_of_channels : {1, 2, 4, 8, 10}) {
197 SCOPED_TRACE(ProduceDebugText(rate, number_of_channels, 1));
198
199 SetUpFrames(rate, number_of_channels);
200 int16_t* frame1_data = frame1.mutable_data();
201 std::iota(frame1_data, frame1_data + number_of_channels * rate / 100, 0);
202 const std::vector<AudioFrame*> frames_to_combine = {&frame1};
203 combiner.Combine(frames_to_combine, number_of_channels, rate,
204 frames_to_combine.size(), &audio_frame_for_mixing);
205
206 const int16_t* audio_frame_for_mixing_data =
207 audio_frame_for_mixing.data();
208 const std::vector<int16_t> mixed_data(
209 audio_frame_for_mixing_data,
210 audio_frame_for_mixing_data + number_of_channels * rate / 100);
211
212 std::vector<int16_t> expected(number_of_channels * rate / 100);
213 std::iota(expected.begin(), expected.end(), 0);
214 EXPECT_EQ(mixed_data, expected);
215 }
216 }
217 }
218
219 // Send a sine wave through the FrameCombiner, and check that the
220 // difference between input and output varies smoothly. Also check
221 // that it is inside reasonable bounds. This is to catch issues like
222 // chromium:695993 and chromium:816875.
TEST(FrameCombiner,GainCurveIsSmoothForAlternatingNumberOfStreams)223 TEST(FrameCombiner, GainCurveIsSmoothForAlternatingNumberOfStreams) {
224 // Rates are divisible by 2000 when limiter is active.
225 std::vector<FrameCombinerConfig> configs = {
226 {false, 30100, 2, 50.f}, {false, 16500, 1, 3200.f},
227 {true, 8000, 1, 3200.f}, {true, 16000, 1, 50.f},
228 {true, 18000, 8, 3200.f}, {true, 10000, 2, 50.f},
229 };
230
231 for (const auto& config : configs) {
232 SCOPED_TRACE(ProduceDebugText(config));
233
234 FrameCombiner combiner(config.use_limiter);
235
236 constexpr int16_t wave_amplitude = 30000;
237 SineWaveGenerator wave_generator(config.wave_frequency, wave_amplitude);
238
239 GainChangeCalculator change_calculator;
240 float cumulative_change = 0.f;
241
242 constexpr size_t iterations = 100;
243
244 for (size_t i = 0; i < iterations; ++i) {
245 SetUpFrames(config.sample_rate_hz, config.number_of_channels);
246 wave_generator.GenerateNextFrame(&frame1);
247 AudioFrameOperations::Mute(&frame2);
248
249 std::vector<AudioFrame*> frames_to_combine = {&frame1};
250 if (i % 2 == 0) {
251 frames_to_combine.push_back(&frame2);
252 }
253 const size_t number_of_samples =
254 frame1.samples_per_channel_ * config.number_of_channels;
255
256 // Ensures limiter is on if 'use_limiter'.
257 constexpr size_t number_of_streams = 2;
258 combiner.Combine(frames_to_combine, config.number_of_channels,
259 config.sample_rate_hz, number_of_streams,
260 &audio_frame_for_mixing);
261 cumulative_change += change_calculator.CalculateGainChange(
262 rtc::ArrayView<const int16_t>(frame1.data(), number_of_samples),
263 rtc::ArrayView<const int16_t>(audio_frame_for_mixing.data(),
264 number_of_samples));
265 }
266
267 // Check that the gain doesn't vary too much.
268 EXPECT_LT(cumulative_change, 10);
269
270 // Check that the latest gain is within reasonable bounds. It
271 // should be slightly less that 1.
272 EXPECT_LT(0.9f, change_calculator.LatestGain());
273 EXPECT_LT(change_calculator.LatestGain(), 1.01f);
274 }
275 }
276 } // namespace webrtc
277