1 /*
2 * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/utility/frame_dropper.h"
12
13 #include <algorithm>
14
15 namespace webrtc {
16
17 namespace {
18
19 const float kDefaultFrameSizeAlpha = 0.9f;
20 const float kDefaultKeyFrameRatioAlpha = 0.99f;
21 // 1 key frame every 10th second in 30 fps.
22 const float kDefaultKeyFrameRatioValue = 1 / 300.0f;
23
24 const float kDefaultDropRatioAlpha = 0.9f;
25 const float kDefaultDropRatioValue = 0.96f;
26 // Maximum duration over which frames are continuously dropped.
27 const float kDefaultMaxDropDurationSecs = 4.0f;
28
29 // Default target bitrate.
30 // TODO(isheriff): Should this be higher to avoid dropping too many packets when
31 // the bandwidth is unknown at the start ?
32 const float kDefaultTargetBitrateKbps = 300.0f;
33 const float kDefaultIncomingFrameRate = 30;
34 const float kLeakyBucketSizeSeconds = 0.5f;
35
36 // A delta frame that is bigger than |kLargeDeltaFactor| times the average
37 // delta frame is a large frame that is spread out for accumulation.
38 const int kLargeDeltaFactor = 3;
39
40 // Cap on the frame size accumulator to prevent excessive drops.
41 const float kAccumulatorCapBufferSizeSecs = 3.0f;
42 } // namespace
43
FrameDropper()44 FrameDropper::FrameDropper()
45 : key_frame_ratio_(kDefaultKeyFrameRatioAlpha),
46 delta_frame_size_avg_kbits_(kDefaultFrameSizeAlpha),
47 drop_ratio_(kDefaultDropRatioAlpha, kDefaultDropRatioValue),
48 enabled_(true),
49 max_drop_duration_secs_(kDefaultMaxDropDurationSecs) {
50 Reset();
51 }
52
53 FrameDropper::~FrameDropper() = default;
54
Reset()55 void FrameDropper::Reset() {
56 key_frame_ratio_.Reset(kDefaultKeyFrameRatioAlpha);
57 key_frame_ratio_.Apply(1.0f, kDefaultKeyFrameRatioValue);
58 delta_frame_size_avg_kbits_.Reset(kDefaultFrameSizeAlpha);
59
60 accumulator_ = 0.0f;
61 accumulator_max_ = kDefaultTargetBitrateKbps / 2;
62 target_bitrate_ = kDefaultTargetBitrateKbps;
63 incoming_frame_rate_ = kDefaultIncomingFrameRate;
64
65 large_frame_accumulation_count_ = 0;
66 large_frame_accumulation_chunk_size_ = 0;
67 large_frame_accumulation_spread_ = 0.5 * kDefaultIncomingFrameRate;
68
69 drop_next_ = false;
70 drop_ratio_.Reset(0.9f);
71 drop_ratio_.Apply(0.0f, 0.0f);
72 drop_count_ = 0;
73 was_below_max_ = true;
74 }
75
Enable(bool enable)76 void FrameDropper::Enable(bool enable) {
77 enabled_ = enable;
78 }
79
Fill(size_t framesize_bytes,bool delta_frame)80 void FrameDropper::Fill(size_t framesize_bytes, bool delta_frame) {
81 if (!enabled_) {
82 return;
83 }
84 float framesize_kbits = 8.0f * static_cast<float>(framesize_bytes) / 1000.0f;
85 if (!delta_frame) {
86 key_frame_ratio_.Apply(1.0, 1.0);
87 // Do not spread if we are already doing it (or we risk dropping bits that
88 // need accumulation). Given we compute the key frame ratio and spread
89 // based on that, this should not normally happen.
90 if (large_frame_accumulation_count_ == 0) {
91 if (key_frame_ratio_.filtered() > 1e-5 &&
92 1 / key_frame_ratio_.filtered() < large_frame_accumulation_spread_) {
93 large_frame_accumulation_count_ =
94 static_cast<int32_t>(1 / key_frame_ratio_.filtered() + 0.5);
95 } else {
96 large_frame_accumulation_count_ =
97 static_cast<int32_t>(large_frame_accumulation_spread_ + 0.5);
98 }
99 large_frame_accumulation_chunk_size_ =
100 framesize_kbits / large_frame_accumulation_count_;
101 framesize_kbits = 0;
102 }
103 } else {
104 // Identify if it is an unusually large delta frame and spread accumulation
105 // if that is the case.
106 if (delta_frame_size_avg_kbits_.filtered() != -1 &&
107 (framesize_kbits >
108 kLargeDeltaFactor * delta_frame_size_avg_kbits_.filtered()) &&
109 large_frame_accumulation_count_ == 0) {
110 large_frame_accumulation_count_ =
111 static_cast<int32_t>(large_frame_accumulation_spread_ + 0.5);
112 large_frame_accumulation_chunk_size_ =
113 framesize_kbits / large_frame_accumulation_count_;
114 framesize_kbits = 0;
115 } else {
116 delta_frame_size_avg_kbits_.Apply(1, framesize_kbits);
117 }
118 key_frame_ratio_.Apply(1.0, 0.0);
119 }
120 // Change the level of the accumulator (bucket)
121 accumulator_ += framesize_kbits;
122 CapAccumulator();
123 }
124
Leak(uint32_t input_framerate)125 void FrameDropper::Leak(uint32_t input_framerate) {
126 if (!enabled_) {
127 return;
128 }
129 if (input_framerate < 1) {
130 return;
131 }
132 if (target_bitrate_ < 0.0f) {
133 return;
134 }
135 // Add lower bound for large frame accumulation spread.
136 large_frame_accumulation_spread_ = std::max(0.5 * input_framerate, 5.0);
137 // Expected bits per frame based on current input frame rate.
138 float expected_bits_per_frame = target_bitrate_ / input_framerate;
139 if (large_frame_accumulation_count_ > 0) {
140 expected_bits_per_frame -= large_frame_accumulation_chunk_size_;
141 --large_frame_accumulation_count_;
142 }
143 accumulator_ -= expected_bits_per_frame;
144 if (accumulator_ < 0.0f) {
145 accumulator_ = 0.0f;
146 }
147 UpdateRatio();
148 }
149
UpdateRatio()150 void FrameDropper::UpdateRatio() {
151 if (accumulator_ > 1.3f * accumulator_max_) {
152 // Too far above accumulator max, react faster.
153 drop_ratio_.UpdateBase(0.8f);
154 } else {
155 // Go back to normal reaction.
156 drop_ratio_.UpdateBase(0.9f);
157 }
158 if (accumulator_ > accumulator_max_) {
159 // We are above accumulator max, and should ideally drop a frame. Increase
160 // the drop_ratio_ and drop the frame later.
161 if (was_below_max_) {
162 drop_next_ = true;
163 }
164 drop_ratio_.Apply(1.0f, 1.0f);
165 drop_ratio_.UpdateBase(0.9f);
166 } else {
167 drop_ratio_.Apply(1.0f, 0.0f);
168 }
169 was_below_max_ = accumulator_ < accumulator_max_;
170 }
171
172 // This function signals when to drop frames to the caller. It makes use of the
173 // drop_ratio_ to smooth out the drops over time.
DropFrame()174 bool FrameDropper::DropFrame() {
175 if (!enabled_) {
176 return false;
177 }
178 if (drop_next_) {
179 drop_next_ = false;
180 drop_count_ = 0;
181 }
182
183 if (drop_ratio_.filtered() >= 0.5f) { // Drops per keep
184 // Limit is the number of frames we should drop between each kept frame
185 // to keep our drop ratio. limit is positive in this case.
186 float denom = 1.0f - drop_ratio_.filtered();
187 if (denom < 1e-5) {
188 denom = 1e-5f;
189 }
190 int32_t limit = static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
191 // Put a bound on the max amount of dropped frames between each kept
192 // frame, in terms of frame rate and window size (secs).
193 int max_limit =
194 static_cast<int>(incoming_frame_rate_ * max_drop_duration_secs_);
195 if (limit > max_limit) {
196 limit = max_limit;
197 }
198 if (drop_count_ < 0) {
199 // Reset the drop_count_ since it was negative and should be positive.
200 drop_count_ = -drop_count_;
201 }
202 if (drop_count_ < limit) {
203 // As long we are below the limit we should drop frames.
204 drop_count_++;
205 return true;
206 } else {
207 // Only when we reset drop_count_ a frame should be kept.
208 drop_count_ = 0;
209 return false;
210 }
211 } else if (drop_ratio_.filtered() > 0.0f &&
212 drop_ratio_.filtered() < 0.5f) { // Keeps per drop
213 // Limit is the number of frames we should keep between each drop
214 // in order to keep the drop ratio. limit is negative in this case,
215 // and the drop_count_ is also negative.
216 float denom = drop_ratio_.filtered();
217 if (denom < 1e-5) {
218 denom = 1e-5f;
219 }
220 int32_t limit = -static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
221 if (drop_count_ > 0) {
222 // Reset the drop_count_ since we have a positive
223 // drop_count_, and it should be negative.
224 drop_count_ = -drop_count_;
225 }
226 if (drop_count_ > limit) {
227 if (drop_count_ == 0) {
228 // Drop frames when we reset drop_count_.
229 drop_count_--;
230 return true;
231 } else {
232 // Keep frames as long as we haven't reached limit.
233 drop_count_--;
234 return false;
235 }
236 } else {
237 drop_count_ = 0;
238 return false;
239 }
240 }
241 drop_count_ = 0;
242 return false;
243 }
244
SetRates(float bitrate,float incoming_frame_rate)245 void FrameDropper::SetRates(float bitrate, float incoming_frame_rate) {
246 // Bit rate of -1 means infinite bandwidth.
247 accumulator_max_ = bitrate * kLeakyBucketSizeSeconds;
248 if (target_bitrate_ > 0.0f && bitrate < target_bitrate_ &&
249 accumulator_ > accumulator_max_) {
250 // Rescale the accumulator level if the accumulator max decreases
251 accumulator_ = bitrate / target_bitrate_ * accumulator_;
252 }
253 target_bitrate_ = bitrate;
254 CapAccumulator();
255 incoming_frame_rate_ = incoming_frame_rate;
256 }
257
258 // Put a cap on the accumulator, i.e., don't let it grow beyond some level.
259 // This is a temporary fix for screencasting where very large frames from
260 // encoder will cause very slow response (too many frame drops).
261 // TODO(isheriff): Remove this now that large delta frames are also spread out ?
CapAccumulator()262 void FrameDropper::CapAccumulator() {
263 float max_accumulator = target_bitrate_ * kAccumulatorCapBufferSizeSecs;
264 if (accumulator_ > max_accumulator) {
265 accumulator_ = max_accumulator;
266 }
267 }
268 } // namespace webrtc
269