1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_coding/codecs/opus/audio_encoder_opus.h"
12
13 #include <algorithm>
14 #include <iterator>
15 #include <memory>
16 #include <string>
17 #include <utility>
18
19 #include "absl/strings/match.h"
20 #include "modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h"
21 #include "modules/audio_coding/audio_network_adaptor/controller_manager.h"
22 #include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h"
23 #include "modules/audio_coding/codecs/opus/opus_interface.h"
24 #include "rtc_base/arraysize.h"
25 #include "rtc_base/checks.h"
26 #include "rtc_base/logging.h"
27 #include "rtc_base/numerics/exp_filter.h"
28 #include "rtc_base/numerics/safe_conversions.h"
29 #include "rtc_base/numerics/safe_minmax.h"
30 #include "rtc_base/string_encode.h"
31 #include "rtc_base/string_to_number.h"
32 #include "rtc_base/time_utils.h"
33 #include "system_wrappers/include/field_trial.h"
34
35 namespace webrtc {
36
37 namespace {
38
39 // Codec parameters for Opus.
40 // draft-spittka-payload-rtp-opus-03
41
42 // Recommended bitrates:
43 // 8-12 kb/s for NB speech,
44 // 16-20 kb/s for WB speech,
45 // 28-40 kb/s for FB speech,
46 // 48-64 kb/s for FB mono music, and
47 // 64-128 kb/s for FB stereo music.
48 // The current implementation applies the following values to mono signals,
49 // and multiplies them by 2 for stereo.
50 constexpr int kOpusBitrateNbBps = 12000;
51 constexpr int kOpusBitrateWbBps = 20000;
52 constexpr int kOpusBitrateFbBps = 32000;
53
54 constexpr int kRtpTimestampRateHz = 48000;
55 constexpr int kDefaultMaxPlaybackRate = 48000;
56
57 // These two lists must be sorted from low to high
58 #if WEBRTC_OPUS_SUPPORT_120MS_PTIME
59 constexpr int kANASupportedFrameLengths[] = {20, 40, 60, 120};
60 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120};
61 #else
62 constexpr int kANASupportedFrameLengths[] = {20, 40, 60};
63 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60};
64 #endif
65
66 // PacketLossFractionSmoother uses an exponential filter with a time constant
67 // of -1.0 / ln(0.9999) = 10000 ms.
68 constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f;
69 constexpr float kMaxPacketLossFraction = 0.2f;
70
CalculateDefaultBitrate(int max_playback_rate,size_t num_channels)71 int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) {
72 const int bitrate = [&] {
73 if (max_playback_rate <= 8000) {
74 return kOpusBitrateNbBps * rtc::dchecked_cast<int>(num_channels);
75 } else if (max_playback_rate <= 16000) {
76 return kOpusBitrateWbBps * rtc::dchecked_cast<int>(num_channels);
77 } else {
78 return kOpusBitrateFbBps * rtc::dchecked_cast<int>(num_channels);
79 }
80 }();
81 RTC_DCHECK_GE(bitrate, AudioEncoderOpusConfig::kMinBitrateBps);
82 RTC_DCHECK_LE(bitrate, AudioEncoderOpusConfig::kMaxBitrateBps);
83 return bitrate;
84 }
85
86 // Get the maxaveragebitrate parameter in string-form, so we can properly figure
87 // out how invalid it is and accurately log invalid values.
CalculateBitrate(int max_playback_rate_hz,size_t num_channels,absl::optional<std::string> bitrate_param)88 int CalculateBitrate(int max_playback_rate_hz,
89 size_t num_channels,
90 absl::optional<std::string> bitrate_param) {
91 const int default_bitrate =
92 CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
93
94 if (bitrate_param) {
95 const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
96 if (bitrate) {
97 const int chosen_bitrate =
98 std::max(AudioEncoderOpusConfig::kMinBitrateBps,
99 std::min(*bitrate, AudioEncoderOpusConfig::kMaxBitrateBps));
100 if (bitrate != chosen_bitrate) {
101 RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate
102 << " clamped to " << chosen_bitrate;
103 }
104 return chosen_bitrate;
105 }
106 RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param
107 << "\" replaced by default bitrate " << default_bitrate;
108 }
109
110 return default_bitrate;
111 }
112
GetChannelCount(const SdpAudioFormat & format)113 int GetChannelCount(const SdpAudioFormat& format) {
114 const auto param = GetFormatParameter(format, "stereo");
115 if (param == "1") {
116 return 2;
117 } else {
118 return 1;
119 }
120 }
121
GetMaxPlaybackRate(const SdpAudioFormat & format)122 int GetMaxPlaybackRate(const SdpAudioFormat& format) {
123 const auto param = GetFormatParameter<int>(format, "maxplaybackrate");
124 if (param && *param >= 8000) {
125 return std::min(*param, kDefaultMaxPlaybackRate);
126 }
127 return kDefaultMaxPlaybackRate;
128 }
129
GetFrameSizeMs(const SdpAudioFormat & format)130 int GetFrameSizeMs(const SdpAudioFormat& format) {
131 const auto ptime = GetFormatParameter<int>(format, "ptime");
132 if (ptime) {
133 // Pick the next highest supported frame length from
134 // kOpusSupportedFrameLengths.
135 for (const int supported_frame_length : kOpusSupportedFrameLengths) {
136 if (supported_frame_length >= *ptime) {
137 return supported_frame_length;
138 }
139 }
140 // If none was found, return the largest supported frame length.
141 return *(std::end(kOpusSupportedFrameLengths) - 1);
142 }
143
144 return AudioEncoderOpusConfig::kDefaultFrameSizeMs;
145 }
146
FindSupportedFrameLengths(int min_frame_length_ms,int max_frame_length_ms,std::vector<int> * out)147 void FindSupportedFrameLengths(int min_frame_length_ms,
148 int max_frame_length_ms,
149 std::vector<int>* out) {
150 out->clear();
151 std::copy_if(std::begin(kANASupportedFrameLengths),
152 std::end(kANASupportedFrameLengths), std::back_inserter(*out),
153 [&](int frame_length_ms) {
154 return frame_length_ms >= min_frame_length_ms &&
155 frame_length_ms <= max_frame_length_ms;
156 });
157 RTC_DCHECK(std::is_sorted(out->begin(), out->end()));
158 }
159
GetBitrateBps(const AudioEncoderOpusConfig & config)160 int GetBitrateBps(const AudioEncoderOpusConfig& config) {
161 RTC_DCHECK(config.IsOk());
162 return *config.bitrate_bps;
163 }
164
GetBitrateMultipliers()165 std::vector<float> GetBitrateMultipliers() {
166 constexpr char kBitrateMultipliersName[] =
167 "WebRTC-Audio-OpusBitrateMultipliers";
168 const bool use_bitrate_multipliers =
169 webrtc::field_trial::IsEnabled(kBitrateMultipliersName);
170 if (use_bitrate_multipliers) {
171 const std::string field_trial_string =
172 webrtc::field_trial::FindFullName(kBitrateMultipliersName);
173 std::vector<std::string> pieces;
174 rtc::tokenize(field_trial_string, '-', &pieces);
175 if (pieces.size() < 2 || pieces[0] != "Enabled") {
176 RTC_LOG(LS_WARNING) << "Invalid parameters for "
177 << kBitrateMultipliersName
178 << ", not using custom values.";
179 return std::vector<float>();
180 }
181 std::vector<float> multipliers(pieces.size() - 1);
182 for (size_t i = 1; i < pieces.size(); i++) {
183 if (!rtc::FromString(pieces[i], &multipliers[i - 1])) {
184 RTC_LOG(LS_WARNING)
185 << "Invalid parameters for " << kBitrateMultipliersName
186 << ", not using custom values.";
187 return std::vector<float>();
188 }
189 }
190 RTC_LOG(LS_INFO) << "Using custom bitrate multipliers: "
191 << field_trial_string;
192 return multipliers;
193 }
194 return std::vector<float>();
195 }
196
GetMultipliedBitrate(int bitrate,const std::vector<float> & multipliers)197 int GetMultipliedBitrate(int bitrate, const std::vector<float>& multipliers) {
198 // The multipliers are valid from 5 kbps.
199 const size_t bitrate_kbps = static_cast<size_t>(bitrate / 1000);
200 if (bitrate_kbps < 5 || bitrate_kbps >= multipliers.size() + 5) {
201 return bitrate;
202 }
203 return static_cast<int>(multipliers[bitrate_kbps - 5] * bitrate);
204 }
205 } // namespace
206
AppendSupportedEncoders(std::vector<AudioCodecSpec> * specs)207 void AudioEncoderOpusImpl::AppendSupportedEncoders(
208 std::vector<AudioCodecSpec>* specs) {
209 const SdpAudioFormat fmt = {"opus",
210 kRtpTimestampRateHz,
211 2,
212 {{"minptime", "10"}, {"useinbandfec", "1"}}};
213 const AudioCodecInfo info = QueryAudioEncoder(*SdpToConfig(fmt));
214 specs->push_back({fmt, info});
215 }
216
QueryAudioEncoder(const AudioEncoderOpusConfig & config)217 AudioCodecInfo AudioEncoderOpusImpl::QueryAudioEncoder(
218 const AudioEncoderOpusConfig& config) {
219 RTC_DCHECK(config.IsOk());
220 AudioCodecInfo info(config.sample_rate_hz, config.num_channels,
221 *config.bitrate_bps,
222 AudioEncoderOpusConfig::kMinBitrateBps,
223 AudioEncoderOpusConfig::kMaxBitrateBps);
224 info.allow_comfort_noise = false;
225 info.supports_network_adaption = true;
226 return info;
227 }
228
MakeAudioEncoder(const AudioEncoderOpusConfig & config,int payload_type)229 std::unique_ptr<AudioEncoder> AudioEncoderOpusImpl::MakeAudioEncoder(
230 const AudioEncoderOpusConfig& config,
231 int payload_type) {
232 RTC_DCHECK(config.IsOk());
233 return std::make_unique<AudioEncoderOpusImpl>(config, payload_type);
234 }
235
SdpToConfig(const SdpAudioFormat & format)236 absl::optional<AudioEncoderOpusConfig> AudioEncoderOpusImpl::SdpToConfig(
237 const SdpAudioFormat& format) {
238 if (!absl::EqualsIgnoreCase(format.name, "opus") ||
239 format.clockrate_hz != kRtpTimestampRateHz || format.num_channels != 2) {
240 return absl::nullopt;
241 }
242
243 AudioEncoderOpusConfig config;
244 config.num_channels = GetChannelCount(format);
245 config.frame_size_ms = GetFrameSizeMs(format);
246 config.max_playback_rate_hz = GetMaxPlaybackRate(format);
247 config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1");
248 config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1");
249 config.cbr_enabled = (GetFormatParameter(format, "cbr") == "1");
250 config.bitrate_bps =
251 CalculateBitrate(config.max_playback_rate_hz, config.num_channels,
252 GetFormatParameter(format, "maxaveragebitrate"));
253 config.application = config.num_channels == 1
254 ? AudioEncoderOpusConfig::ApplicationMode::kVoip
255 : AudioEncoderOpusConfig::ApplicationMode::kAudio;
256
257 constexpr int kMinANAFrameLength = kANASupportedFrameLengths[0];
258 constexpr int kMaxANAFrameLength =
259 kANASupportedFrameLengths[arraysize(kANASupportedFrameLengths) - 1];
260
261 // For now, minptime and maxptime are only used with ANA. If ptime is outside
262 // of this range, it will get adjusted once ANA takes hold. Ideally, we'd know
263 // if ANA was to be used when setting up the config, and adjust accordingly.
264 const int min_frame_length_ms =
265 GetFormatParameter<int>(format, "minptime").value_or(kMinANAFrameLength);
266 const int max_frame_length_ms =
267 GetFormatParameter<int>(format, "maxptime").value_or(kMaxANAFrameLength);
268
269 FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms,
270 &config.supported_frame_lengths_ms);
271 RTC_DCHECK(config.IsOk());
272 return config;
273 }
274
GetNewComplexity(const AudioEncoderOpusConfig & config)275 absl::optional<int> AudioEncoderOpusImpl::GetNewComplexity(
276 const AudioEncoderOpusConfig& config) {
277 RTC_DCHECK(config.IsOk());
278 const int bitrate_bps = GetBitrateBps(config);
279 if (bitrate_bps >= config.complexity_threshold_bps -
280 config.complexity_threshold_window_bps &&
281 bitrate_bps <= config.complexity_threshold_bps +
282 config.complexity_threshold_window_bps) {
283 // Within the hysteresis window; make no change.
284 return absl::nullopt;
285 } else {
286 return bitrate_bps <= config.complexity_threshold_bps
287 ? config.low_rate_complexity
288 : config.complexity;
289 }
290 }
291
GetNewBandwidth(const AudioEncoderOpusConfig & config,OpusEncInst * inst)292 absl::optional<int> AudioEncoderOpusImpl::GetNewBandwidth(
293 const AudioEncoderOpusConfig& config,
294 OpusEncInst* inst) {
295 constexpr int kMinWidebandBitrate = 8000;
296 constexpr int kMaxNarrowbandBitrate = 9000;
297 constexpr int kAutomaticThreshold = 11000;
298 RTC_DCHECK(config.IsOk());
299 const int bitrate = GetBitrateBps(config);
300 if (bitrate > kAutomaticThreshold) {
301 return absl::optional<int>(OPUS_AUTO);
302 }
303 const int bandwidth = WebRtcOpus_GetBandwidth(inst);
304 RTC_DCHECK_GE(bandwidth, 0);
305 if (bitrate > kMaxNarrowbandBitrate && bandwidth < OPUS_BANDWIDTH_WIDEBAND) {
306 return absl::optional<int>(OPUS_BANDWIDTH_WIDEBAND);
307 } else if (bitrate < kMinWidebandBitrate &&
308 bandwidth > OPUS_BANDWIDTH_NARROWBAND) {
309 return absl::optional<int>(OPUS_BANDWIDTH_NARROWBAND);
310 }
311 return absl::optional<int>();
312 }
313
314 class AudioEncoderOpusImpl::PacketLossFractionSmoother {
315 public:
PacketLossFractionSmoother()316 explicit PacketLossFractionSmoother()
317 : last_sample_time_ms_(rtc::TimeMillis()),
318 smoother_(kAlphaForPacketLossFractionSmoother) {}
319
320 // Gets the smoothed packet loss fraction.
GetAverage() const321 float GetAverage() const {
322 float value = smoother_.filtered();
323 return (value == rtc::ExpFilter::kValueUndefined) ? 0.0f : value;
324 }
325
326 // Add new observation to the packet loss fraction smoother.
AddSample(float packet_loss_fraction)327 void AddSample(float packet_loss_fraction) {
328 int64_t now_ms = rtc::TimeMillis();
329 smoother_.Apply(static_cast<float>(now_ms - last_sample_time_ms_),
330 packet_loss_fraction);
331 last_sample_time_ms_ = now_ms;
332 }
333
334 private:
335 int64_t last_sample_time_ms_;
336
337 // An exponential filter is used to smooth the packet loss fraction.
338 rtc::ExpFilter smoother_;
339 };
340
AudioEncoderOpusImpl(const AudioEncoderOpusConfig & config,int payload_type)341 AudioEncoderOpusImpl::AudioEncoderOpusImpl(const AudioEncoderOpusConfig& config,
342 int payload_type)
343 : AudioEncoderOpusImpl(
344 config,
345 payload_type,
346 [this](const std::string& config_string, RtcEventLog* event_log) {
347 return DefaultAudioNetworkAdaptorCreator(config_string, event_log);
348 },
349 // We choose 5sec as initial time constant due to empirical data.
350 std::make_unique<SmoothingFilterImpl>(5000)) {}
351
AudioEncoderOpusImpl(const AudioEncoderOpusConfig & config,int payload_type,const AudioNetworkAdaptorCreator & audio_network_adaptor_creator,std::unique_ptr<SmoothingFilter> bitrate_smoother)352 AudioEncoderOpusImpl::AudioEncoderOpusImpl(
353 const AudioEncoderOpusConfig& config,
354 int payload_type,
355 const AudioNetworkAdaptorCreator& audio_network_adaptor_creator,
356 std::unique_ptr<SmoothingFilter> bitrate_smoother)
357 : payload_type_(payload_type),
358 send_side_bwe_with_overhead_(
359 webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")),
360 use_stable_target_for_adaptation_(!webrtc::field_trial::IsDisabled(
361 "WebRTC-Audio-StableTargetAdaptation")),
362 adjust_bandwidth_(
363 webrtc::field_trial::IsEnabled("WebRTC-AdjustOpusBandwidth")),
364 bitrate_changed_(true),
365 bitrate_multipliers_(GetBitrateMultipliers()),
366 packet_loss_rate_(0.0),
367 inst_(nullptr),
368 packet_loss_fraction_smoother_(new PacketLossFractionSmoother()),
369 audio_network_adaptor_creator_(audio_network_adaptor_creator),
370 bitrate_smoother_(std::move(bitrate_smoother)) {
371 RTC_DCHECK(0 <= payload_type && payload_type <= 127);
372
373 // Sanity check of the redundant payload type field that we want to get rid
374 // of. See https://bugs.chromium.org/p/webrtc/issues/detail?id=7847
375 RTC_CHECK(config.payload_type == -1 || config.payload_type == payload_type);
376
377 RTC_CHECK(RecreateEncoderInstance(config));
378 SetProjectedPacketLossRate(packet_loss_rate_);
379 }
380
AudioEncoderOpusImpl(int payload_type,const SdpAudioFormat & format)381 AudioEncoderOpusImpl::AudioEncoderOpusImpl(int payload_type,
382 const SdpAudioFormat& format)
383 : AudioEncoderOpusImpl(*SdpToConfig(format), payload_type) {}
384
~AudioEncoderOpusImpl()385 AudioEncoderOpusImpl::~AudioEncoderOpusImpl() {
386 RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
387 }
388
SampleRateHz() const389 int AudioEncoderOpusImpl::SampleRateHz() const {
390 return config_.sample_rate_hz;
391 }
392
NumChannels() const393 size_t AudioEncoderOpusImpl::NumChannels() const {
394 return config_.num_channels;
395 }
396
RtpTimestampRateHz() const397 int AudioEncoderOpusImpl::RtpTimestampRateHz() const {
398 return kRtpTimestampRateHz;
399 }
400
Num10MsFramesInNextPacket() const401 size_t AudioEncoderOpusImpl::Num10MsFramesInNextPacket() const {
402 return Num10msFramesPerPacket();
403 }
404
Max10MsFramesInAPacket() const405 size_t AudioEncoderOpusImpl::Max10MsFramesInAPacket() const {
406 return Num10msFramesPerPacket();
407 }
408
GetTargetBitrate() const409 int AudioEncoderOpusImpl::GetTargetBitrate() const {
410 return GetBitrateBps(config_);
411 }
412
Reset()413 void AudioEncoderOpusImpl::Reset() {
414 RTC_CHECK(RecreateEncoderInstance(config_));
415 }
416
SetFec(bool enable)417 bool AudioEncoderOpusImpl::SetFec(bool enable) {
418 if (enable) {
419 RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_));
420 } else {
421 RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_));
422 }
423 config_.fec_enabled = enable;
424 return true;
425 }
426
SetDtx(bool enable)427 bool AudioEncoderOpusImpl::SetDtx(bool enable) {
428 if (enable) {
429 RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_));
430 } else {
431 RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_));
432 }
433 config_.dtx_enabled = enable;
434 return true;
435 }
436
GetDtx() const437 bool AudioEncoderOpusImpl::GetDtx() const {
438 return config_.dtx_enabled;
439 }
440
SetApplication(Application application)441 bool AudioEncoderOpusImpl::SetApplication(Application application) {
442 auto conf = config_;
443 switch (application) {
444 case Application::kSpeech:
445 conf.application = AudioEncoderOpusConfig::ApplicationMode::kVoip;
446 break;
447 case Application::kAudio:
448 conf.application = AudioEncoderOpusConfig::ApplicationMode::kAudio;
449 break;
450 }
451 return RecreateEncoderInstance(conf);
452 }
453
SetMaxPlaybackRate(int frequency_hz)454 void AudioEncoderOpusImpl::SetMaxPlaybackRate(int frequency_hz) {
455 auto conf = config_;
456 conf.max_playback_rate_hz = frequency_hz;
457 RTC_CHECK(RecreateEncoderInstance(conf));
458 }
459
EnableAudioNetworkAdaptor(const std::string & config_string,RtcEventLog * event_log)460 bool AudioEncoderOpusImpl::EnableAudioNetworkAdaptor(
461 const std::string& config_string,
462 RtcEventLog* event_log) {
463 audio_network_adaptor_ =
464 audio_network_adaptor_creator_(config_string, event_log);
465 return audio_network_adaptor_.get() != nullptr;
466 }
467
DisableAudioNetworkAdaptor()468 void AudioEncoderOpusImpl::DisableAudioNetworkAdaptor() {
469 audio_network_adaptor_.reset(nullptr);
470 }
471
OnReceivedUplinkPacketLossFraction(float uplink_packet_loss_fraction)472 void AudioEncoderOpusImpl::OnReceivedUplinkPacketLossFraction(
473 float uplink_packet_loss_fraction) {
474 if (audio_network_adaptor_) {
475 audio_network_adaptor_->SetUplinkPacketLossFraction(
476 uplink_packet_loss_fraction);
477 ApplyAudioNetworkAdaptor();
478 }
479 packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction);
480 float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage();
481 SetProjectedPacketLossRate(average_fraction_loss);
482 }
483
OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps)484 void AudioEncoderOpusImpl::OnReceivedTargetAudioBitrate(
485 int target_audio_bitrate_bps) {
486 SetTargetBitrate(target_audio_bitrate_bps);
487 }
488
OnReceivedUplinkBandwidth(int target_audio_bitrate_bps,absl::optional<int64_t> bwe_period_ms,absl::optional<int64_t> stable_target_bitrate_bps)489 void AudioEncoderOpusImpl::OnReceivedUplinkBandwidth(
490 int target_audio_bitrate_bps,
491 absl::optional<int64_t> bwe_period_ms,
492 absl::optional<int64_t> stable_target_bitrate_bps) {
493 if (audio_network_adaptor_) {
494 audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps);
495 if (use_stable_target_for_adaptation_) {
496 if (stable_target_bitrate_bps)
497 audio_network_adaptor_->SetUplinkBandwidth(*stable_target_bitrate_bps);
498 } else {
499 // We give smoothed bitrate allocation to audio network adaptor as
500 // the uplink bandwidth.
501 // The BWE spikes should not affect the bitrate smoother more than 25%.
502 // To simplify the calculations we use a step response as input signal.
503 // The step response of an exponential filter is
504 // u(t) = 1 - e^(-t / time_constant).
505 // In order to limit the affect of a BWE spike within 25% of its value
506 // before
507 // the next BWE update, we would choose a time constant that fulfills
508 // 1 - e^(-bwe_period_ms / time_constant) < 0.25
509 // Then 4 * bwe_period_ms is a good choice.
510 if (bwe_period_ms)
511 bitrate_smoother_->SetTimeConstantMs(*bwe_period_ms * 4);
512 bitrate_smoother_->AddSample(target_audio_bitrate_bps);
513 }
514
515 ApplyAudioNetworkAdaptor();
516 } else if (send_side_bwe_with_overhead_) {
517 if (!overhead_bytes_per_packet_) {
518 RTC_LOG(LS_INFO)
519 << "AudioEncoderOpusImpl: Overhead unknown, target audio bitrate "
520 << target_audio_bitrate_bps << " bps is ignored.";
521 return;
522 }
523 const int overhead_bps = static_cast<int>(
524 *overhead_bytes_per_packet_ * 8 * 100 / Num10MsFramesInNextPacket());
525 SetTargetBitrate(
526 std::min(AudioEncoderOpusConfig::kMaxBitrateBps,
527 std::max(AudioEncoderOpusConfig::kMinBitrateBps,
528 target_audio_bitrate_bps - overhead_bps)));
529 } else {
530 SetTargetBitrate(target_audio_bitrate_bps);
531 }
532 }
OnReceivedUplinkBandwidth(int target_audio_bitrate_bps,absl::optional<int64_t> bwe_period_ms)533 void AudioEncoderOpusImpl::OnReceivedUplinkBandwidth(
534 int target_audio_bitrate_bps,
535 absl::optional<int64_t> bwe_period_ms) {
536 OnReceivedUplinkBandwidth(target_audio_bitrate_bps, bwe_period_ms,
537 absl::nullopt);
538 }
539
OnReceivedUplinkAllocation(BitrateAllocationUpdate update)540 void AudioEncoderOpusImpl::OnReceivedUplinkAllocation(
541 BitrateAllocationUpdate update) {
542 OnReceivedUplinkBandwidth(update.target_bitrate.bps(), update.bwe_period.ms(),
543 update.stable_target_bitrate.bps());
544 }
545
OnReceivedRtt(int rtt_ms)546 void AudioEncoderOpusImpl::OnReceivedRtt(int rtt_ms) {
547 if (!audio_network_adaptor_)
548 return;
549 audio_network_adaptor_->SetRtt(rtt_ms);
550 ApplyAudioNetworkAdaptor();
551 }
552
OnReceivedOverhead(size_t overhead_bytes_per_packet)553 void AudioEncoderOpusImpl::OnReceivedOverhead(
554 size_t overhead_bytes_per_packet) {
555 if (audio_network_adaptor_) {
556 audio_network_adaptor_->SetOverhead(overhead_bytes_per_packet);
557 ApplyAudioNetworkAdaptor();
558 } else {
559 overhead_bytes_per_packet_ = overhead_bytes_per_packet;
560 }
561 }
562
SetReceiverFrameLengthRange(int min_frame_length_ms,int max_frame_length_ms)563 void AudioEncoderOpusImpl::SetReceiverFrameLengthRange(
564 int min_frame_length_ms,
565 int max_frame_length_ms) {
566 // Ensure that |SetReceiverFrameLengthRange| is called before
567 // |EnableAudioNetworkAdaptor|, otherwise we need to recreate
568 // |audio_network_adaptor_|, which is not a needed use case.
569 RTC_DCHECK(!audio_network_adaptor_);
570 FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms,
571 &config_.supported_frame_lengths_ms);
572 }
573
EncodeImpl(uint32_t rtp_timestamp,rtc::ArrayView<const int16_t> audio,rtc::Buffer * encoded)574 AudioEncoder::EncodedInfo AudioEncoderOpusImpl::EncodeImpl(
575 uint32_t rtp_timestamp,
576 rtc::ArrayView<const int16_t> audio,
577 rtc::Buffer* encoded) {
578 MaybeUpdateUplinkBandwidth();
579
580 if (input_buffer_.empty())
581 first_timestamp_in_buffer_ = rtp_timestamp;
582
583 input_buffer_.insert(input_buffer_.end(), audio.cbegin(), audio.cend());
584 if (input_buffer_.size() <
585 (Num10msFramesPerPacket() * SamplesPer10msFrame())) {
586 return EncodedInfo();
587 }
588 RTC_CHECK_EQ(input_buffer_.size(),
589 Num10msFramesPerPacket() * SamplesPer10msFrame());
590
591 const size_t max_encoded_bytes = SufficientOutputBufferSize();
592 const size_t start_offset_bytes = encoded->size();
593 EncodedInfo info;
594 info.encoded_bytes = encoded->AppendData(
595 max_encoded_bytes, [&](rtc::ArrayView<uint8_t> encoded) {
596 int status = WebRtcOpus_Encode(
597 inst_, &input_buffer_[0],
598 rtc::CheckedDivExact(input_buffer_.size(), config_.num_channels),
599 rtc::saturated_cast<int16_t>(max_encoded_bytes), encoded.data());
600
601 RTC_CHECK_GE(status, 0); // Fails only if fed invalid data.
602
603 return static_cast<size_t>(status);
604 });
605 input_buffer_.clear();
606
607 // Will use new packet size for next encoding.
608 config_.frame_size_ms = next_frame_length_ms_;
609
610 if (adjust_bandwidth_ && bitrate_changed_) {
611 const auto bandwidth = GetNewBandwidth(config_, inst_);
612 if (bandwidth) {
613 RTC_CHECK_EQ(0, WebRtcOpus_SetBandwidth(inst_, *bandwidth));
614 }
615 bitrate_changed_ = false;
616 }
617
618 info.encoded_timestamp = first_timestamp_in_buffer_;
619 info.payload_type = payload_type_;
620 info.send_even_if_empty = true; // Allows Opus to send empty packets.
621 info.encoder_type = CodecType::kOpus;
622
623 // Extract the VAD result from the encoded packet.
624 int has_voice = WebRtcOpus_PacketHasVoiceActivity(
625 &encoded->data()[start_offset_bytes], info.encoded_bytes);
626 if (has_voice == -1) {
627 // CELT mode packet or there was an error. This had set the speech flag to
628 // true historically.
629 info.speech = true;
630 } else {
631 info.speech = has_voice;
632 }
633
634 return info;
635 }
636
Num10msFramesPerPacket() const637 size_t AudioEncoderOpusImpl::Num10msFramesPerPacket() const {
638 return static_cast<size_t>(rtc::CheckedDivExact(config_.frame_size_ms, 10));
639 }
640
SamplesPer10msFrame() const641 size_t AudioEncoderOpusImpl::SamplesPer10msFrame() const {
642 return rtc::CheckedDivExact(config_.sample_rate_hz, 100) *
643 config_.num_channels;
644 }
645
SufficientOutputBufferSize() const646 size_t AudioEncoderOpusImpl::SufficientOutputBufferSize() const {
647 // Calculate the number of bytes we expect the encoder to produce,
648 // then multiply by two to give a wide margin for error.
649 const size_t bytes_per_millisecond =
650 static_cast<size_t>(GetBitrateBps(config_) / (1000 * 8) + 1);
651 const size_t approx_encoded_bytes =
652 Num10msFramesPerPacket() * 10 * bytes_per_millisecond;
653 return 2 * approx_encoded_bytes;
654 }
655
656 // If the given config is OK, recreate the Opus encoder instance with those
657 // settings, save the config, and return true. Otherwise, do nothing and return
658 // false.
RecreateEncoderInstance(const AudioEncoderOpusConfig & config)659 bool AudioEncoderOpusImpl::RecreateEncoderInstance(
660 const AudioEncoderOpusConfig& config) {
661 if (!config.IsOk())
662 return false;
663 config_ = config;
664 if (inst_)
665 RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
666 input_buffer_.clear();
667 input_buffer_.reserve(Num10msFramesPerPacket() * SamplesPer10msFrame());
668 RTC_CHECK_EQ(0, WebRtcOpus_EncoderCreate(
669 &inst_, config.num_channels,
670 config.application ==
671 AudioEncoderOpusConfig::ApplicationMode::kVoip
672 ? 0
673 : 1,
674 config.sample_rate_hz));
675 const int bitrate = GetBitrateBps(config);
676 RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, bitrate));
677 RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps.";
678 if (config.fec_enabled) {
679 RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_));
680 } else {
681 RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_));
682 }
683 RTC_CHECK_EQ(
684 0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz));
685 // Use the default complexity if the start bitrate is within the hysteresis
686 // window.
687 complexity_ = GetNewComplexity(config).value_or(config.complexity);
688 RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_));
689 bitrate_changed_ = true;
690 if (config.dtx_enabled) {
691 RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_));
692 } else {
693 RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_));
694 }
695 RTC_CHECK_EQ(0,
696 WebRtcOpus_SetPacketLossRate(
697 inst_, static_cast<int32_t>(packet_loss_rate_ * 100 + .5)));
698 if (config.cbr_enabled) {
699 RTC_CHECK_EQ(0, WebRtcOpus_EnableCbr(inst_));
700 } else {
701 RTC_CHECK_EQ(0, WebRtcOpus_DisableCbr(inst_));
702 }
703 num_channels_to_encode_ = NumChannels();
704 next_frame_length_ms_ = config_.frame_size_ms;
705 return true;
706 }
707
SetFrameLength(int frame_length_ms)708 void AudioEncoderOpusImpl::SetFrameLength(int frame_length_ms) {
709 next_frame_length_ms_ = frame_length_ms;
710 }
711
SetNumChannelsToEncode(size_t num_channels_to_encode)712 void AudioEncoderOpusImpl::SetNumChannelsToEncode(
713 size_t num_channels_to_encode) {
714 RTC_DCHECK_GT(num_channels_to_encode, 0);
715 RTC_DCHECK_LE(num_channels_to_encode, config_.num_channels);
716
717 if (num_channels_to_encode_ == num_channels_to_encode)
718 return;
719
720 RTC_CHECK_EQ(0, WebRtcOpus_SetForceChannels(inst_, num_channels_to_encode));
721 num_channels_to_encode_ = num_channels_to_encode;
722 }
723
SetProjectedPacketLossRate(float fraction)724 void AudioEncoderOpusImpl::SetProjectedPacketLossRate(float fraction) {
725 fraction = std::min(std::max(fraction, 0.0f), kMaxPacketLossFraction);
726 if (packet_loss_rate_ != fraction) {
727 packet_loss_rate_ = fraction;
728 RTC_CHECK_EQ(
729 0, WebRtcOpus_SetPacketLossRate(
730 inst_, static_cast<int32_t>(packet_loss_rate_ * 100 + .5)));
731 }
732 }
733
SetTargetBitrate(int bits_per_second)734 void AudioEncoderOpusImpl::SetTargetBitrate(int bits_per_second) {
735 const int new_bitrate = rtc::SafeClamp<int>(
736 bits_per_second, AudioEncoderOpusConfig::kMinBitrateBps,
737 AudioEncoderOpusConfig::kMaxBitrateBps);
738 if (config_.bitrate_bps && *config_.bitrate_bps != new_bitrate) {
739 config_.bitrate_bps = new_bitrate;
740 RTC_DCHECK(config_.IsOk());
741 const int bitrate = GetBitrateBps(config_);
742 RTC_CHECK_EQ(
743 0, WebRtcOpus_SetBitRate(
744 inst_, GetMultipliedBitrate(bitrate, bitrate_multipliers_)));
745 RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps.";
746 bitrate_changed_ = true;
747 }
748
749 const auto new_complexity = GetNewComplexity(config_);
750 if (new_complexity && complexity_ != *new_complexity) {
751 complexity_ = *new_complexity;
752 RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_));
753 }
754 }
755
ApplyAudioNetworkAdaptor()756 void AudioEncoderOpusImpl::ApplyAudioNetworkAdaptor() {
757 auto config = audio_network_adaptor_->GetEncoderRuntimeConfig();
758
759 if (config.bitrate_bps)
760 SetTargetBitrate(*config.bitrate_bps);
761 if (config.frame_length_ms)
762 SetFrameLength(*config.frame_length_ms);
763 if (config.enable_dtx)
764 SetDtx(*config.enable_dtx);
765 if (config.num_channels)
766 SetNumChannelsToEncode(*config.num_channels);
767 }
768
769 std::unique_ptr<AudioNetworkAdaptor>
DefaultAudioNetworkAdaptorCreator(const std::string & config_string,RtcEventLog * event_log) const770 AudioEncoderOpusImpl::DefaultAudioNetworkAdaptorCreator(
771 const std::string& config_string,
772 RtcEventLog* event_log) const {
773 AudioNetworkAdaptorImpl::Config config;
774 config.event_log = event_log;
775 return std::unique_ptr<AudioNetworkAdaptor>(new AudioNetworkAdaptorImpl(
776 config, ControllerManagerImpl::Create(
777 config_string, NumChannels(), supported_frame_lengths_ms(),
778 AudioEncoderOpusConfig::kMinBitrateBps,
779 num_channels_to_encode_, next_frame_length_ms_,
780 GetTargetBitrate(), config_.fec_enabled, GetDtx())));
781 }
782
MaybeUpdateUplinkBandwidth()783 void AudioEncoderOpusImpl::MaybeUpdateUplinkBandwidth() {
784 if (audio_network_adaptor_ && !use_stable_target_for_adaptation_) {
785 int64_t now_ms = rtc::TimeMillis();
786 if (!bitrate_smoother_last_update_time_ ||
787 now_ms - *bitrate_smoother_last_update_time_ >=
788 config_.uplink_bandwidth_update_interval_ms) {
789 absl::optional<float> smoothed_bitrate = bitrate_smoother_->GetAverage();
790 if (smoothed_bitrate)
791 audio_network_adaptor_->SetUplinkBandwidth(*smoothed_bitrate);
792 bitrate_smoother_last_update_time_ = now_ms;
793 }
794 }
795 }
796
GetANAStats() const797 ANAStats AudioEncoderOpusImpl::GetANAStats() const {
798 if (audio_network_adaptor_) {
799 return audio_network_adaptor_->GetStats();
800 }
801 return ANAStats();
802 }
803
804 absl::optional<std::pair<TimeDelta, TimeDelta> >
GetFrameLengthRange() const805 AudioEncoderOpusImpl::GetFrameLengthRange() const {
806 if (config_.supported_frame_lengths_ms.empty()) {
807 return absl::nullopt;
808 } else if (audio_network_adaptor_) {
809 return {{TimeDelta::Millis(config_.supported_frame_lengths_ms.front()),
810 TimeDelta::Millis(config_.supported_frame_lengths_ms.back())}};
811 } else {
812 return {{TimeDelta::Millis(config_.frame_size_ms),
813 TimeDelta::Millis(config_.frame_size_ms)}};
814 }
815 }
816
817 } // namespace webrtc
818