1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_coding/audio_network_adaptor/controller_manager.h"
12
13 #include <cmath>
14 #include <memory>
15 #include <string>
16 #include <utility>
17
18 #include "modules/audio_coding/audio_network_adaptor/bitrate_controller.h"
19 #include "modules/audio_coding/audio_network_adaptor/channel_controller.h"
20 #include "modules/audio_coding/audio_network_adaptor/debug_dump_writer.h"
21 #include "modules/audio_coding/audio_network_adaptor/dtx_controller.h"
22 #include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h"
23 #include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h"
24 #include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h"
25 #include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h"
26 #include "rtc_base/ignore_wundef.h"
27 #include "rtc_base/logging.h"
28 #include "rtc_base/time_utils.h"
29
30 #if WEBRTC_ENABLE_PROTOBUF
31 RTC_PUSH_IGNORING_WUNDEF()
32 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
33 #include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h"
34 #else
35 #include "modules/audio_coding/audio_network_adaptor/config.pb.h"
36 #endif
37 RTC_POP_IGNORING_WUNDEF()
38 #endif
39
40 namespace webrtc {
41
42 namespace {
43
44 #if WEBRTC_ENABLE_PROTOBUF
45
CreateFecControllerPlrBased(const audio_network_adaptor::config::FecController & config,bool initial_fec_enabled)46 std::unique_ptr<FecControllerPlrBased> CreateFecControllerPlrBased(
47 const audio_network_adaptor::config::FecController& config,
48 bool initial_fec_enabled) {
49 RTC_CHECK(config.has_fec_enabling_threshold());
50 RTC_CHECK(config.has_fec_disabling_threshold());
51 RTC_CHECK(config.has_time_constant_ms());
52
53 auto& fec_enabling_threshold = config.fec_enabling_threshold();
54 RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_bps());
55 RTC_CHECK(fec_enabling_threshold.has_low_bandwidth_packet_loss());
56 RTC_CHECK(fec_enabling_threshold.has_high_bandwidth_bps());
57 RTC_CHECK(fec_enabling_threshold.has_high_bandwidth_packet_loss());
58
59 auto& fec_disabling_threshold = config.fec_disabling_threshold();
60 RTC_CHECK(fec_disabling_threshold.has_low_bandwidth_bps());
61 RTC_CHECK(fec_disabling_threshold.has_low_bandwidth_packet_loss());
62 RTC_CHECK(fec_disabling_threshold.has_high_bandwidth_bps());
63 RTC_CHECK(fec_disabling_threshold.has_high_bandwidth_packet_loss());
64
65 return std::unique_ptr<FecControllerPlrBased>(
66 new FecControllerPlrBased(FecControllerPlrBased::Config(
67 initial_fec_enabled,
68 ThresholdCurve(fec_enabling_threshold.low_bandwidth_bps(),
69 fec_enabling_threshold.low_bandwidth_packet_loss(),
70 fec_enabling_threshold.high_bandwidth_bps(),
71 fec_enabling_threshold.high_bandwidth_packet_loss()),
72 ThresholdCurve(fec_disabling_threshold.low_bandwidth_bps(),
73 fec_disabling_threshold.low_bandwidth_packet_loss(),
74 fec_disabling_threshold.high_bandwidth_bps(),
75 fec_disabling_threshold.high_bandwidth_packet_loss()),
76 config.time_constant_ms())));
77 }
78
CreateFrameLengthController(const audio_network_adaptor::config::FrameLengthController & config,rtc::ArrayView<const int> encoder_frame_lengths_ms,int initial_frame_length_ms,int min_encoder_bitrate_bps)79 std::unique_ptr<FrameLengthController> CreateFrameLengthController(
80 const audio_network_adaptor::config::FrameLengthController& config,
81 rtc::ArrayView<const int> encoder_frame_lengths_ms,
82 int initial_frame_length_ms,
83 int min_encoder_bitrate_bps) {
84 RTC_CHECK(config.has_fl_increasing_packet_loss_fraction());
85 RTC_CHECK(config.has_fl_decreasing_packet_loss_fraction());
86
87 std::map<FrameLengthController::Config::FrameLengthChange, int>
88 fl_changing_bandwidths_bps;
89
90 if (config.has_fl_20ms_to_60ms_bandwidth_bps()) {
91 fl_changing_bandwidths_bps.insert(
92 std::make_pair(FrameLengthController::Config::FrameLengthChange(20, 60),
93 config.fl_20ms_to_60ms_bandwidth_bps()));
94 }
95
96 if (config.has_fl_60ms_to_20ms_bandwidth_bps()) {
97 fl_changing_bandwidths_bps.insert(
98 std::make_pair(FrameLengthController::Config::FrameLengthChange(60, 20),
99 config.fl_60ms_to_20ms_bandwidth_bps()));
100 }
101
102 if (config.has_fl_20ms_to_40ms_bandwidth_bps()) {
103 fl_changing_bandwidths_bps.insert(
104 std::make_pair(FrameLengthController::Config::FrameLengthChange(20, 40),
105 config.fl_20ms_to_40ms_bandwidth_bps()));
106 }
107
108 if (config.has_fl_40ms_to_20ms_bandwidth_bps()) {
109 fl_changing_bandwidths_bps.insert(
110 std::make_pair(FrameLengthController::Config::FrameLengthChange(40, 20),
111 config.fl_40ms_to_20ms_bandwidth_bps()));
112 }
113
114 if (config.has_fl_40ms_to_60ms_bandwidth_bps()) {
115 fl_changing_bandwidths_bps.insert(
116 std::make_pair(FrameLengthController::Config::FrameLengthChange(40, 60),
117 config.fl_40ms_to_60ms_bandwidth_bps()));
118 }
119
120 if (config.has_fl_60ms_to_40ms_bandwidth_bps()) {
121 fl_changing_bandwidths_bps.insert(
122 std::make_pair(FrameLengthController::Config::FrameLengthChange(60, 40),
123 config.fl_60ms_to_40ms_bandwidth_bps()));
124 }
125
126 if (config.has_fl_60ms_to_120ms_bandwidth_bps()) {
127 fl_changing_bandwidths_bps.insert(std::make_pair(
128 FrameLengthController::Config::FrameLengthChange(60, 120),
129 config.fl_60ms_to_120ms_bandwidth_bps()));
130 }
131
132 if (config.has_fl_120ms_to_60ms_bandwidth_bps()) {
133 fl_changing_bandwidths_bps.insert(std::make_pair(
134 FrameLengthController::Config::FrameLengthChange(120, 60),
135 config.fl_120ms_to_60ms_bandwidth_bps()));
136 }
137
138 int fl_increase_overhead_offset = 0;
139 if (config.has_fl_increase_overhead_offset()) {
140 fl_increase_overhead_offset = config.fl_increase_overhead_offset();
141 }
142 int fl_decrease_overhead_offset = 0;
143 if (config.has_fl_decrease_overhead_offset()) {
144 fl_decrease_overhead_offset = config.fl_decrease_overhead_offset();
145 }
146
147 FrameLengthController::Config ctor_config(
148 std::set<int>(), initial_frame_length_ms, min_encoder_bitrate_bps,
149 config.fl_increasing_packet_loss_fraction(),
150 config.fl_decreasing_packet_loss_fraction(), fl_increase_overhead_offset,
151 fl_decrease_overhead_offset, std::move(fl_changing_bandwidths_bps));
152
153 for (auto frame_length : encoder_frame_lengths_ms)
154 ctor_config.encoder_frame_lengths_ms.insert(frame_length);
155
156 return std::unique_ptr<FrameLengthController>(
157 new FrameLengthController(ctor_config));
158 }
159
CreateChannelController(const audio_network_adaptor::config::ChannelController & config,size_t num_encoder_channels,size_t intial_channels_to_encode)160 std::unique_ptr<ChannelController> CreateChannelController(
161 const audio_network_adaptor::config::ChannelController& config,
162 size_t num_encoder_channels,
163 size_t intial_channels_to_encode) {
164 RTC_CHECK(config.has_channel_1_to_2_bandwidth_bps());
165 RTC_CHECK(config.has_channel_2_to_1_bandwidth_bps());
166
167 return std::unique_ptr<ChannelController>(new ChannelController(
168 ChannelController::Config(num_encoder_channels, intial_channels_to_encode,
169 config.channel_1_to_2_bandwidth_bps(),
170 config.channel_2_to_1_bandwidth_bps())));
171 }
172
CreateDtxController(const audio_network_adaptor::config::DtxController & dtx_config,bool initial_dtx_enabled)173 std::unique_ptr<DtxController> CreateDtxController(
174 const audio_network_adaptor::config::DtxController& dtx_config,
175 bool initial_dtx_enabled) {
176 RTC_CHECK(dtx_config.has_dtx_enabling_bandwidth_bps());
177 RTC_CHECK(dtx_config.has_dtx_disabling_bandwidth_bps());
178
179 return std::unique_ptr<DtxController>(new DtxController(DtxController::Config(
180 initial_dtx_enabled, dtx_config.dtx_enabling_bandwidth_bps(),
181 dtx_config.dtx_disabling_bandwidth_bps())));
182 }
183
184 using audio_network_adaptor::BitrateController;
CreateBitrateController(const audio_network_adaptor::config::BitrateController & bitrate_config,int initial_bitrate_bps,int initial_frame_length_ms)185 std::unique_ptr<BitrateController> CreateBitrateController(
186 const audio_network_adaptor::config::BitrateController& bitrate_config,
187 int initial_bitrate_bps,
188 int initial_frame_length_ms) {
189 int fl_increase_overhead_offset = 0;
190 if (bitrate_config.has_fl_increase_overhead_offset()) {
191 fl_increase_overhead_offset = bitrate_config.fl_increase_overhead_offset();
192 }
193 int fl_decrease_overhead_offset = 0;
194 if (bitrate_config.has_fl_decrease_overhead_offset()) {
195 fl_decrease_overhead_offset = bitrate_config.fl_decrease_overhead_offset();
196 }
197 return std::unique_ptr<BitrateController>(
198 new BitrateController(BitrateController::Config(
199 initial_bitrate_bps, initial_frame_length_ms,
200 fl_increase_overhead_offset, fl_decrease_overhead_offset)));
201 }
202
CreateFrameLengthControllerV2(const audio_network_adaptor::config::FrameLengthControllerV2 & config,rtc::ArrayView<const int> encoder_frame_lengths_ms)203 std::unique_ptr<FrameLengthControllerV2> CreateFrameLengthControllerV2(
204 const audio_network_adaptor::config::FrameLengthControllerV2& config,
205 rtc::ArrayView<const int> encoder_frame_lengths_ms) {
206 return std::make_unique<FrameLengthControllerV2>(
207 encoder_frame_lengths_ms, config.min_payload_bitrate_bps(),
208 config.use_slow_adaptation());
209 }
210 #endif // WEBRTC_ENABLE_PROTOBUF
211
212 } // namespace
213
Config(int min_reordering_time_ms,float min_reordering_squared_distance)214 ControllerManagerImpl::Config::Config(int min_reordering_time_ms,
215 float min_reordering_squared_distance)
216 : min_reordering_time_ms(min_reordering_time_ms),
217 min_reordering_squared_distance(min_reordering_squared_distance) {}
218
219 ControllerManagerImpl::Config::~Config() = default;
220
Create(const std::string & config_string,size_t num_encoder_channels,rtc::ArrayView<const int> encoder_frame_lengths_ms,int min_encoder_bitrate_bps,size_t intial_channels_to_encode,int initial_frame_length_ms,int initial_bitrate_bps,bool initial_fec_enabled,bool initial_dtx_enabled)221 std::unique_ptr<ControllerManager> ControllerManagerImpl::Create(
222 const std::string& config_string,
223 size_t num_encoder_channels,
224 rtc::ArrayView<const int> encoder_frame_lengths_ms,
225 int min_encoder_bitrate_bps,
226 size_t intial_channels_to_encode,
227 int initial_frame_length_ms,
228 int initial_bitrate_bps,
229 bool initial_fec_enabled,
230 bool initial_dtx_enabled) {
231 return Create(config_string, num_encoder_channels, encoder_frame_lengths_ms,
232 min_encoder_bitrate_bps, intial_channels_to_encode,
233 initial_frame_length_ms, initial_bitrate_bps,
234 initial_fec_enabled, initial_dtx_enabled, nullptr);
235 }
236
Create(const std::string & config_string,size_t num_encoder_channels,rtc::ArrayView<const int> encoder_frame_lengths_ms,int min_encoder_bitrate_bps,size_t intial_channels_to_encode,int initial_frame_length_ms,int initial_bitrate_bps,bool initial_fec_enabled,bool initial_dtx_enabled,DebugDumpWriter * debug_dump_writer)237 std::unique_ptr<ControllerManager> ControllerManagerImpl::Create(
238 const std::string& config_string,
239 size_t num_encoder_channels,
240 rtc::ArrayView<const int> encoder_frame_lengths_ms,
241 int min_encoder_bitrate_bps,
242 size_t intial_channels_to_encode,
243 int initial_frame_length_ms,
244 int initial_bitrate_bps,
245 bool initial_fec_enabled,
246 bool initial_dtx_enabled,
247 DebugDumpWriter* debug_dump_writer) {
248 #if WEBRTC_ENABLE_PROTOBUF
249 audio_network_adaptor::config::ControllerManager controller_manager_config;
250 RTC_CHECK(controller_manager_config.ParseFromString(config_string));
251 if (debug_dump_writer)
252 debug_dump_writer->DumpControllerManagerConfig(controller_manager_config,
253 rtc::TimeMillis());
254
255 std::vector<std::unique_ptr<Controller>> controllers;
256 std::map<const Controller*, std::pair<int, float>> scoring_points;
257
258 for (int i = 0; i < controller_manager_config.controllers_size(); ++i) {
259 auto& controller_config = controller_manager_config.controllers(i);
260 std::unique_ptr<Controller> controller;
261 switch (controller_config.controller_case()) {
262 case audio_network_adaptor::config::Controller::kFecController:
263 controller = CreateFecControllerPlrBased(
264 controller_config.fec_controller(), initial_fec_enabled);
265 break;
266 case audio_network_adaptor::config::Controller::kFecControllerRplrBased:
267 // FecControllerRplrBased has been removed and can't be used anymore.
268 RTC_NOTREACHED();
269 continue;
270 case audio_network_adaptor::config::Controller::kFrameLengthController:
271 controller = CreateFrameLengthController(
272 controller_config.frame_length_controller(),
273 encoder_frame_lengths_ms, initial_frame_length_ms,
274 min_encoder_bitrate_bps);
275 break;
276 case audio_network_adaptor::config::Controller::kChannelController:
277 controller = CreateChannelController(
278 controller_config.channel_controller(), num_encoder_channels,
279 intial_channels_to_encode);
280 break;
281 case audio_network_adaptor::config::Controller::kDtxController:
282 controller = CreateDtxController(controller_config.dtx_controller(),
283 initial_dtx_enabled);
284 break;
285 case audio_network_adaptor::config::Controller::kBitrateController:
286 controller = CreateBitrateController(
287 controller_config.bitrate_controller(), initial_bitrate_bps,
288 initial_frame_length_ms);
289 break;
290 case audio_network_adaptor::config::Controller::kFrameLengthControllerV2:
291 controller = CreateFrameLengthControllerV2(
292 controller_config.frame_length_controller_v2(),
293 encoder_frame_lengths_ms);
294 break;
295 default:
296 RTC_NOTREACHED();
297 }
298 if (controller_config.has_scoring_point()) {
299 auto& scoring_point = controller_config.scoring_point();
300 RTC_CHECK(scoring_point.has_uplink_bandwidth_bps());
301 RTC_CHECK(scoring_point.has_uplink_packet_loss_fraction());
302 scoring_points[controller.get()] = std::make_pair<int, float>(
303 scoring_point.uplink_bandwidth_bps(),
304 scoring_point.uplink_packet_loss_fraction());
305 }
306 controllers.push_back(std::move(controller));
307 }
308
309 if (scoring_points.size() == 0) {
310 return std::unique_ptr<ControllerManagerImpl>(
311 new ControllerManagerImpl(ControllerManagerImpl::Config(0, 0),
312 std::move(controllers), scoring_points));
313 } else {
314 RTC_CHECK(controller_manager_config.has_min_reordering_time_ms());
315 RTC_CHECK(controller_manager_config.has_min_reordering_squared_distance());
316 return std::unique_ptr<ControllerManagerImpl>(new ControllerManagerImpl(
317 ControllerManagerImpl::Config(
318 controller_manager_config.min_reordering_time_ms(),
319 controller_manager_config.min_reordering_squared_distance()),
320 std::move(controllers), scoring_points));
321 }
322
323 #else
324 RTC_NOTREACHED();
325 return nullptr;
326 #endif // WEBRTC_ENABLE_PROTOBUF
327 }
328
ControllerManagerImpl(const Config & config)329 ControllerManagerImpl::ControllerManagerImpl(const Config& config)
330 : ControllerManagerImpl(
331 config,
332 std::vector<std::unique_ptr<Controller>>(),
333 std::map<const Controller*, std::pair<int, float>>()) {}
334
ControllerManagerImpl(const Config & config,std::vector<std::unique_ptr<Controller>> controllers,const std::map<const Controller *,std::pair<int,float>> & scoring_points)335 ControllerManagerImpl::ControllerManagerImpl(
336 const Config& config,
337 std::vector<std::unique_ptr<Controller>> controllers,
338 const std::map<const Controller*, std::pair<int, float>>& scoring_points)
339 : config_(config),
340 controllers_(std::move(controllers)),
341 last_reordering_time_ms_(absl::nullopt),
342 last_scoring_point_(0, 0.0) {
343 for (auto& controller : controllers_)
344 default_sorted_controllers_.push_back(controller.get());
345 sorted_controllers_ = default_sorted_controllers_;
346 for (auto& controller_point : scoring_points) {
347 controller_scoring_points_.insert(std::make_pair(
348 controller_point.first, ScoringPoint(controller_point.second.first,
349 controller_point.second.second)));
350 }
351 }
352
353 ControllerManagerImpl::~ControllerManagerImpl() = default;
354
GetSortedControllers(const Controller::NetworkMetrics & metrics)355 std::vector<Controller*> ControllerManagerImpl::GetSortedControllers(
356 const Controller::NetworkMetrics& metrics) {
357 if (controller_scoring_points_.size() == 0)
358 return default_sorted_controllers_;
359
360 if (!metrics.uplink_bandwidth_bps || !metrics.uplink_packet_loss_fraction)
361 return sorted_controllers_;
362
363 const int64_t now_ms = rtc::TimeMillis();
364 if (last_reordering_time_ms_ &&
365 now_ms - *last_reordering_time_ms_ < config_.min_reordering_time_ms)
366 return sorted_controllers_;
367
368 ScoringPoint scoring_point(*metrics.uplink_bandwidth_bps,
369 *metrics.uplink_packet_loss_fraction);
370
371 if (last_reordering_time_ms_ &&
372 last_scoring_point_.SquaredDistanceTo(scoring_point) <
373 config_.min_reordering_squared_distance)
374 return sorted_controllers_;
375
376 // Sort controllers according to the distances of |scoring_point| to the
377 // scoring points of controllers.
378 //
379 // A controller that does not associate with any scoring point
380 // are treated as if
381 // 1) they are less important than any controller that has a scoring point,
382 // 2) they are equally important to any controller that has no scoring point,
383 // and their relative order will follow |default_sorted_controllers_|.
384 std::vector<Controller*> sorted_controllers(default_sorted_controllers_);
385 std::stable_sort(
386 sorted_controllers.begin(), sorted_controllers.end(),
387 [this, &scoring_point](const Controller* lhs, const Controller* rhs) {
388 auto lhs_scoring_point = controller_scoring_points_.find(lhs);
389 auto rhs_scoring_point = controller_scoring_points_.find(rhs);
390
391 if (lhs_scoring_point == controller_scoring_points_.end())
392 return false;
393
394 if (rhs_scoring_point == controller_scoring_points_.end())
395 return true;
396
397 return lhs_scoring_point->second.SquaredDistanceTo(scoring_point) <
398 rhs_scoring_point->second.SquaredDistanceTo(scoring_point);
399 });
400
401 if (sorted_controllers_ != sorted_controllers) {
402 sorted_controllers_ = sorted_controllers;
403 last_reordering_time_ms_ = now_ms;
404 last_scoring_point_ = scoring_point;
405 }
406 return sorted_controllers_;
407 }
408
GetControllers() const409 std::vector<Controller*> ControllerManagerImpl::GetControllers() const {
410 return default_sorted_controllers_;
411 }
412
ScoringPoint(int uplink_bandwidth_bps,float uplink_packet_loss_fraction)413 ControllerManagerImpl::ScoringPoint::ScoringPoint(
414 int uplink_bandwidth_bps,
415 float uplink_packet_loss_fraction)
416 : uplink_bandwidth_bps(uplink_bandwidth_bps),
417 uplink_packet_loss_fraction(uplink_packet_loss_fraction) {}
418
419 namespace {
420
421 constexpr int kMinUplinkBandwidthBps = 0;
422 constexpr int kMaxUplinkBandwidthBps = 120000;
423
NormalizeUplinkBandwidth(int uplink_bandwidth_bps)424 float NormalizeUplinkBandwidth(int uplink_bandwidth_bps) {
425 uplink_bandwidth_bps =
426 std::min(kMaxUplinkBandwidthBps,
427 std::max(kMinUplinkBandwidthBps, uplink_bandwidth_bps));
428 return static_cast<float>(uplink_bandwidth_bps - kMinUplinkBandwidthBps) /
429 (kMaxUplinkBandwidthBps - kMinUplinkBandwidthBps);
430 }
431
NormalizePacketLossFraction(float uplink_packet_loss_fraction)432 float NormalizePacketLossFraction(float uplink_packet_loss_fraction) {
433 // |uplink_packet_loss_fraction| is seldom larger than 0.3, so we scale it up
434 // by 3.3333f.
435 return std::min(uplink_packet_loss_fraction * 3.3333f, 1.0f);
436 }
437
438 } // namespace
439
SquaredDistanceTo(const ScoringPoint & scoring_point) const440 float ControllerManagerImpl::ScoringPoint::SquaredDistanceTo(
441 const ScoringPoint& scoring_point) const {
442 float diff_normalized_bitrate_bps =
443 NormalizeUplinkBandwidth(scoring_point.uplink_bandwidth_bps) -
444 NormalizeUplinkBandwidth(uplink_bandwidth_bps);
445 float diff_normalized_packet_loss =
446 NormalizePacketLossFraction(scoring_point.uplink_packet_loss_fraction) -
447 NormalizePacketLossFraction(uplink_packet_loss_fraction);
448 return std::pow(diff_normalized_bitrate_bps, 2) +
449 std::pow(diff_normalized_packet_loss, 2);
450 }
451
452 } // namespace webrtc
453