1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
12
13 #include <algorithm>
14 #include <cmath>
15 #include <cstddef>
16 #include <numeric>
17 #include <vector>
18
19 #include "absl/container/inlined_vector.h"
20 #include "rtc_base/checks.h"
21
22 namespace webrtc {
23 namespace {
24
25 const float kSpatialLayeringRateScalingFactor = 0.55f;
26 const float kTemporalLayeringRateScalingFactor = 0.55f;
27
28 // Returns numberOfSpatialLayers if no layers are active.
GetFirstActiveLayer(const VideoCodec & codec)29 size_t GetFirstActiveLayer(const VideoCodec& codec) {
30 RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
31 RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
32 size_t layer = 0;
33 for (; layer < codec.VP9().numberOfSpatialLayers; ++layer) {
34 if (codec.spatialLayers[layer].active) {
35 break;
36 }
37 }
38 return layer;
39 }
40
GetNumActiveSpatialLayers(const VideoCodec & codec)41 static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) {
42 RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
43 RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
44
45 const size_t first_active_layer = GetFirstActiveLayer(codec);
46 size_t last_active_layer = first_active_layer;
47 for (; last_active_layer < codec.VP9().numberOfSpatialLayers;
48 ++last_active_layer) {
49 if (!codec.spatialLayers[last_active_layer].active) {
50 break;
51 }
52 }
53 return last_active_layer - first_active_layer;
54 }
55
AdjustAndVerify(const VideoCodec & codec,size_t first_active_layer,const std::vector<DataRate> & spatial_layer_rates)56 std::vector<DataRate> AdjustAndVerify(
57 const VideoCodec& codec,
58 size_t first_active_layer,
59 const std::vector<DataRate>& spatial_layer_rates) {
60 std::vector<DataRate> adjusted_spatial_layer_rates;
61 // Keep track of rate that couldn't be applied to the previous layer due to
62 // max bitrate constraint, try to pass it forward to the next one.
63 DataRate excess_rate = DataRate::Zero();
64 for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) {
65 DataRate min_rate = DataRate::KilobitsPerSec(
66 codec.spatialLayers[first_active_layer + sl_idx].minBitrate);
67 DataRate max_rate = DataRate::KilobitsPerSec(
68 codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
69
70 DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate;
71 if (layer_rate < min_rate) {
72 // Not enough rate to reach min bitrate for desired number of layers,
73 // abort allocation.
74 if (spatial_layer_rates.size() == 1) {
75 return spatial_layer_rates;
76 }
77 return adjusted_spatial_layer_rates;
78 }
79
80 if (layer_rate <= max_rate) {
81 excess_rate = DataRate::Zero();
82 adjusted_spatial_layer_rates.push_back(layer_rate);
83 } else {
84 excess_rate = layer_rate - max_rate;
85 adjusted_spatial_layer_rates.push_back(max_rate);
86 }
87 }
88
89 return adjusted_spatial_layer_rates;
90 }
91
SplitBitrate(size_t num_layers,DataRate total_bitrate,float rate_scaling_factor)92 static std::vector<DataRate> SplitBitrate(size_t num_layers,
93 DataRate total_bitrate,
94 float rate_scaling_factor) {
95 std::vector<DataRate> bitrates;
96
97 double denominator = 0.0;
98 for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
99 denominator += std::pow(rate_scaling_factor, layer_idx);
100 }
101
102 double numerator = std::pow(rate_scaling_factor, num_layers - 1);
103 for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
104 bitrates.push_back(numerator * total_bitrate / denominator);
105 numerator /= rate_scaling_factor;
106 }
107
108 const DataRate sum =
109 std::accumulate(bitrates.begin(), bitrates.end(), DataRate::Zero());
110
111 // Keep the sum of split bitrates equal to the total bitrate by adding or
112 // subtracting bits, which were lost due to rounding, to the latest layer.
113 if (total_bitrate > sum) {
114 bitrates.back() += total_bitrate - sum;
115 } else if (total_bitrate < sum) {
116 bitrates.back() -= sum - total_bitrate;
117 }
118
119 return bitrates;
120 }
121
122 // Returns the minimum bitrate needed for |num_active_layers| spatial layers to
123 // become active using the configuration specified by |codec|.
FindLayerTogglingThreshold(const VideoCodec & codec,size_t first_active_layer,size_t num_active_layers)124 DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
125 size_t first_active_layer,
126 size_t num_active_layers) {
127 if (num_active_layers == 1) {
128 return DataRate::KilobitsPerSec(codec.spatialLayers[0].minBitrate);
129 }
130
131 if (codec.mode == VideoCodecMode::kRealtimeVideo) {
132 DataRate lower_bound = DataRate::Zero();
133 DataRate upper_bound = DataRate::Zero();
134 if (num_active_layers > 1) {
135 for (size_t i = 0; i < num_active_layers - 1; ++i) {
136 lower_bound += DataRate::KilobitsPerSec(
137 codec.spatialLayers[first_active_layer + i].minBitrate);
138 upper_bound += DataRate::KilobitsPerSec(
139 codec.spatialLayers[first_active_layer + i].maxBitrate);
140 }
141 }
142 upper_bound += DataRate::KilobitsPerSec(
143 codec.spatialLayers[first_active_layer + num_active_layers - 1]
144 .minBitrate);
145
146 // Do a binary search until upper and lower bound is the highest bitrate for
147 // |num_active_layers| - 1 layers and lowest bitrate for |num_active_layers|
148 // layers respectively.
149 while (upper_bound - lower_bound > DataRate::BitsPerSec(1)) {
150 DataRate try_rate = (lower_bound + upper_bound) / 2;
151 if (AdjustAndVerify(codec, first_active_layer,
152 SplitBitrate(num_active_layers, try_rate,
153 kSpatialLayeringRateScalingFactor))
154 .size() == num_active_layers) {
155 upper_bound = try_rate;
156 } else {
157 lower_bound = try_rate;
158 }
159 }
160 return upper_bound;
161 } else {
162 DataRate toggling_rate = DataRate::Zero();
163 for (size_t i = 0; i < num_active_layers - 1; ++i) {
164 toggling_rate += DataRate::KilobitsPerSec(
165 codec.spatialLayers[first_active_layer + i].targetBitrate);
166 }
167 toggling_rate += DataRate::KilobitsPerSec(
168 codec.spatialLayers[first_active_layer + num_active_layers - 1]
169 .minBitrate);
170 return toggling_rate;
171 }
172 }
173
174 } // namespace
175
SvcRateAllocator(const VideoCodec & codec)176 SvcRateAllocator::SvcRateAllocator(const VideoCodec& codec)
177 : codec_(codec),
178 experiment_settings_(StableTargetRateExperiment::ParseFromFieldTrials()),
179 cumulative_layer_start_bitrates_(GetLayerStartBitrates(codec)),
180 last_active_layer_count_(0) {
181 RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
182 RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
183 RTC_DCHECK_GT(codec.VP9().numberOfTemporalLayers, 0u);
184 for (size_t layer_idx = 0; layer_idx < codec.VP9().numberOfSpatialLayers;
185 ++layer_idx) {
186 // Verify min <= target <= max.
187 if (codec.spatialLayers[layer_idx].active) {
188 RTC_DCHECK_GT(codec.spatialLayers[layer_idx].maxBitrate, 0);
189 RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate,
190 codec.spatialLayers[layer_idx].minBitrate);
191 RTC_DCHECK_GE(codec.spatialLayers[layer_idx].targetBitrate,
192 codec.spatialLayers[layer_idx].minBitrate);
193 RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate,
194 codec.spatialLayers[layer_idx].targetBitrate);
195 }
196 }
197 }
198
Allocate(VideoBitrateAllocationParameters parameters)199 VideoBitrateAllocation SvcRateAllocator::Allocate(
200 VideoBitrateAllocationParameters parameters) {
201 DataRate total_bitrate = parameters.total_bitrate;
202 if (codec_.maxBitrate != 0) {
203 total_bitrate =
204 std::min(total_bitrate, DataRate::KilobitsPerSec(codec_.maxBitrate));
205 }
206
207 if (codec_.spatialLayers[0].targetBitrate == 0) {
208 // Delegate rate distribution to VP9 encoder wrapper if bitrate thresholds
209 // are not set.
210 VideoBitrateAllocation bitrate_allocation;
211 bitrate_allocation.SetBitrate(0, 0, total_bitrate.bps());
212 return bitrate_allocation;
213 }
214
215 const size_t first_active_layer = GetFirstActiveLayer(codec_);
216 const size_t num_active_layers = GetNumActiveSpatialLayers(codec_);
217 size_t num_spatial_layers = num_active_layers;
218
219 if (num_spatial_layers == 0) {
220 return VideoBitrateAllocation(); // All layers are deactivated.
221 }
222
223 // Figure out how many spatial layers should be active.
224 if (experiment_settings_.IsEnabled() &&
225 parameters.stable_bitrate > DataRate::Zero()) {
226 double hysteresis_factor;
227 if (codec_.mode == VideoCodecMode::kScreensharing) {
228 hysteresis_factor = experiment_settings_.GetScreenshareHysteresisFactor();
229 } else {
230 hysteresis_factor = experiment_settings_.GetVideoHysteresisFactor();
231 }
232
233 DataRate stable_rate =
234 std::min(parameters.total_bitrate, parameters.stable_bitrate);
235 // First check if bitrate has grown large enough to enable new layers.
236 size_t num_enabled_with_hysteresis =
237 FindNumEnabledLayers(stable_rate / hysteresis_factor);
238 if (num_enabled_with_hysteresis >= last_active_layer_count_) {
239 num_spatial_layers = num_enabled_with_hysteresis;
240 } else {
241 // We could not enable new layers, check if any should be disabled.
242 num_spatial_layers =
243 std::min(last_active_layer_count_, FindNumEnabledLayers(stable_rate));
244 }
245 } else {
246 num_spatial_layers = FindNumEnabledLayers(parameters.total_bitrate);
247 }
248 last_active_layer_count_ = num_spatial_layers;
249
250 VideoBitrateAllocation allocation;
251 if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
252 allocation = GetAllocationNormalVideo(total_bitrate, first_active_layer,
253 num_spatial_layers);
254 } else {
255 allocation = GetAllocationScreenSharing(total_bitrate, first_active_layer,
256 num_spatial_layers);
257 }
258 allocation.set_bw_limited(num_spatial_layers < num_active_layers);
259 return allocation;
260 }
261
GetAllocationNormalVideo(DataRate total_bitrate,size_t first_active_layer,size_t num_spatial_layers) const262 VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
263 DataRate total_bitrate,
264 size_t first_active_layer,
265 size_t num_spatial_layers) const {
266 std::vector<DataRate> spatial_layer_rates;
267 if (num_spatial_layers == 0) {
268 // Not enough rate for even the base layer. Force allocation at the total
269 // bitrate anyway.
270 num_spatial_layers = 1;
271 spatial_layer_rates.push_back(total_bitrate);
272 } else {
273 spatial_layer_rates =
274 AdjustAndVerify(codec_, first_active_layer,
275 SplitBitrate(num_spatial_layers, total_bitrate,
276 kSpatialLayeringRateScalingFactor));
277 RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers);
278 }
279
280 VideoBitrateAllocation bitrate_allocation;
281
282 const size_t num_temporal_layers = codec_.VP9().numberOfTemporalLayers;
283 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
284 std::vector<DataRate> temporal_layer_rates =
285 SplitBitrate(num_temporal_layers, spatial_layer_rates[sl_idx],
286 kTemporalLayeringRateScalingFactor);
287
288 // Distribute rate across temporal layers. Allocate more bits to lower
289 // layers since they are used for prediction of higher layers and their
290 // references are far apart.
291 if (num_temporal_layers == 1) {
292 bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
293 temporal_layer_rates[0].bps());
294 } else if (num_temporal_layers == 2) {
295 bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
296 temporal_layer_rates[1].bps());
297 bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
298 temporal_layer_rates[0].bps());
299 } else {
300 RTC_CHECK_EQ(num_temporal_layers, 3);
301 // In case of three temporal layers the high layer has two frames and the
302 // middle layer has one frame within GOP (in between two consecutive low
303 // layer frames). Thus high layer requires more bits (comparing pure
304 // bitrate of layer, excluding bitrate of base layers) to keep quality on
305 // par with lower layers.
306 bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
307 temporal_layer_rates[2].bps());
308 bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
309 temporal_layer_rates[0].bps());
310 bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2,
311 temporal_layer_rates[1].bps());
312 }
313 }
314
315 return bitrate_allocation;
316 }
317
318 // Bit-rate is allocated in such a way, that the highest enabled layer will have
319 // between min and max bitrate, and all others will have exactly target
320 // bit-rate allocated.
GetAllocationScreenSharing(DataRate total_bitrate,size_t first_active_layer,size_t num_spatial_layers) const321 VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing(
322 DataRate total_bitrate,
323 size_t first_active_layer,
324 size_t num_spatial_layers) const {
325 VideoBitrateAllocation bitrate_allocation;
326
327 if (num_spatial_layers == 0 ||
328 total_bitrate <
329 DataRate::KilobitsPerSec(
330 codec_.spatialLayers[first_active_layer].minBitrate)) {
331 // Always enable at least one layer.
332 bitrate_allocation.SetBitrate(first_active_layer, 0, total_bitrate.bps());
333 return bitrate_allocation;
334 }
335
336 DataRate allocated_rate = DataRate::Zero();
337 DataRate top_layer_rate = DataRate::Zero();
338 size_t sl_idx;
339 for (sl_idx = first_active_layer;
340 sl_idx < first_active_layer + num_spatial_layers; ++sl_idx) {
341 const DataRate min_rate =
342 DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx].minBitrate);
343 const DataRate target_rate =
344 DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx].targetBitrate);
345
346 if (allocated_rate + min_rate > total_bitrate) {
347 // Use stable rate to determine if layer should be enabled.
348 break;
349 }
350
351 top_layer_rate = std::min(target_rate, total_bitrate - allocated_rate);
352 bitrate_allocation.SetBitrate(sl_idx, 0, top_layer_rate.bps());
353 allocated_rate += top_layer_rate;
354 }
355
356 if (sl_idx > 0 && total_bitrate - allocated_rate > DataRate::Zero()) {
357 // Add leftover to the last allocated layer.
358 top_layer_rate = std::min(
359 top_layer_rate + (total_bitrate - allocated_rate),
360 DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx - 1].maxBitrate));
361 bitrate_allocation.SetBitrate(sl_idx - 1, 0, top_layer_rate.bps());
362 }
363
364 return bitrate_allocation;
365 }
366
FindNumEnabledLayers(DataRate target_rate) const367 size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const {
368 if (cumulative_layer_start_bitrates_.empty()) {
369 return 0;
370 }
371
372 size_t num_enabled_layers = 0;
373 for (DataRate start_rate : cumulative_layer_start_bitrates_) {
374 // First layer is always enabled.
375 if (num_enabled_layers == 0 || start_rate <= target_rate) {
376 ++num_enabled_layers;
377 } else {
378 break;
379 }
380 }
381
382 return num_enabled_layers;
383 }
384
GetMaxBitrate(const VideoCodec & codec)385 DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) {
386 const size_t first_active_layer = GetFirstActiveLayer(codec);
387 const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec);
388
389 DataRate max_bitrate = DataRate::Zero();
390 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
391 max_bitrate += DataRate::KilobitsPerSec(
392 codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
393 }
394
395 if (codec.maxBitrate != 0) {
396 max_bitrate =
397 std::min(max_bitrate, DataRate::KilobitsPerSec(codec.maxBitrate));
398 }
399
400 return max_bitrate;
401 }
402
GetPaddingBitrate(const VideoCodec & codec)403 DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) {
404 auto start_bitrate = GetLayerStartBitrates(codec);
405 if (start_bitrate.empty()) {
406 return DataRate::Zero(); // All layers are deactivated.
407 }
408
409 return start_bitrate.back();
410 }
411
412 absl::InlinedVector<DataRate, kMaxSpatialLayers>
GetLayerStartBitrates(const VideoCodec & codec)413 SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) {
414 absl::InlinedVector<DataRate, kMaxSpatialLayers> start_bitrates;
415 const size_t first_active_layer = GetFirstActiveLayer(codec);
416 const size_t num_layers = GetNumActiveSpatialLayers(codec);
417 DataRate last_rate = DataRate::Zero();
418 for (size_t i = 1; i <= num_layers; ++i) {
419 DataRate layer_toggling_rate =
420 FindLayerTogglingThreshold(codec, first_active_layer, i);
421 start_bitrates.push_back(layer_toggling_rate);
422 RTC_DCHECK_LE(last_rate, layer_toggling_rate);
423 last_rate = layer_toggling_rate;
424 }
425 return start_bitrates;
426 }
427
428 } // namespace webrtc
429