• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
12 #define MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
13 
14 #include <math.h>
15 #include <stdlib.h>
16 
17 #include <memory>
18 
19 #include "modules/video_coding/internal_defines.h"
20 #include "rtc_base/experiments/rate_control_settings.h"
21 #include "rtc_base/numerics/exp_filter.h"
22 
23 namespace webrtc {
24 namespace media_optimization {
25 
26 // Number of time periods used for (max) window filter for packet loss
27 // TODO(marpan): set reasonable window size for filtered packet loss,
28 // adjustment should be based on logged/real data of loss stats/correlation.
29 enum { kLossPrHistorySize = 10 };
30 
31 // 1000 ms, total filter length is (kLossPrHistorySize * 1000) ms
32 enum { kLossPrShortFilterWinMs = 1000 };
33 
34 // The type of filter used on the received packet loss reports.
35 enum FilterPacketLossMode {
36   kNoFilter,   // No filtering on received loss.
37   kAvgFilter,  // Recursive average filter.
38   kMaxFilter   // Max-window filter, over the time interval of:
39                // (kLossPrHistorySize * kLossPrShortFilterWinMs) ms.
40 };
41 
42 // Thresholds for hybrid NACK/FEC
43 // common to media optimization and the jitter buffer.
44 const int64_t kLowRttNackMs = 20;
45 
46 // If the RTT is higher than this an extra RTT wont be added to to the jitter
47 // buffer delay.
48 const int kMaxRttDelayThreshold = 500;
49 
50 struct VCMProtectionParameters {
51   VCMProtectionParameters();
52 
53   int64_t rtt;
54   float lossPr;
55   float bitRate;
56   float packetsPerFrame;
57   float packetsPerFrameKey;
58   float frameRate;
59   float keyFrameSize;
60   uint8_t fecRateDelta;
61   uint8_t fecRateKey;
62   uint16_t codecWidth;
63   uint16_t codecHeight;
64   int numLayers;
65 };
66 
67 /******************************/
68 /* VCMProtectionMethod class  */
69 /******************************/
70 
71 enum VCMProtectionMethodEnum { kNack, kFec, kNackFec, kNone };
72 
73 class VCMLossProbabilitySample {
74  public:
VCMLossProbabilitySample()75   VCMLossProbabilitySample() : lossPr255(0), timeMs(-1) {}
76 
77   uint8_t lossPr255;
78   int64_t timeMs;
79 };
80 
81 class VCMProtectionMethod {
82  public:
83   VCMProtectionMethod();
84   virtual ~VCMProtectionMethod();
85 
86   // Updates the efficiency of the method using the parameters provided
87   //
88   // Input:
89   //         - parameters         : Parameters used to calculate efficiency
90   //
91   // Return value                 : True if this method is recommended in
92   //                                the given conditions.
93   virtual bool UpdateParameters(const VCMProtectionParameters* parameters) = 0;
94 
95   // Returns the protection type
96   //
97   // Return value                 : The protection type
98   VCMProtectionMethodEnum Type() const;
99 
100   // Returns the effective packet loss for ER, required by this protection
101   // method
102   //
103   // Return value                 : Required effective packet loss
104   virtual uint8_t RequiredPacketLossER();
105 
106   // Extracts the FEC protection factor for Key frame, required by this
107   // protection method
108   //
109   // Return value                 : Required protectionFactor for Key frame
110   virtual uint8_t RequiredProtectionFactorK();
111 
112   // Extracts the FEC protection factor for Delta frame, required by this
113   // protection method
114   //
115   // Return value                 : Required protectionFactor for delta frame
116   virtual uint8_t RequiredProtectionFactorD();
117 
118   // Extracts whether the FEC Unequal protection (UEP) is used for Key frame.
119   //
120   // Return value                 : Required Unequal protection on/off state.
121   virtual bool RequiredUepProtectionK();
122 
123   // Extracts whether the the FEC Unequal protection (UEP) is used for Delta
124   // frame.
125   //
126   // Return value                 : Required Unequal protection on/off state.
127   virtual bool RequiredUepProtectionD();
128 
129   virtual int MaxFramesFec() const;
130 
131  protected:
132   uint8_t _effectivePacketLoss;
133   uint8_t _protectionFactorK;
134   uint8_t _protectionFactorD;
135   // Estimation of residual loss after the FEC
136   float _scaleProtKey;
137   int32_t _maxPayloadSize;
138 
139   bool _useUepProtectionK;
140   bool _useUepProtectionD;
141   float _corrFecCost;
142   VCMProtectionMethodEnum _type;
143 };
144 
145 class VCMNackMethod : public VCMProtectionMethod {
146  public:
147   VCMNackMethod();
148   ~VCMNackMethod() override;
149   bool UpdateParameters(const VCMProtectionParameters* parameters) override;
150   // Get the effective packet loss
151   bool EffectivePacketLoss(const VCMProtectionParameters* parameter);
152 };
153 
154 class VCMFecMethod : public VCMProtectionMethod {
155  public:
156   VCMFecMethod();
157   ~VCMFecMethod() override;
158   bool UpdateParameters(const VCMProtectionParameters* parameters) override;
159   // Get the effective packet loss for ER
160   bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
161   // Get the FEC protection factors
162   bool ProtectionFactor(const VCMProtectionParameters* parameters);
163   // Get the boost for key frame protection
164   uint8_t BoostCodeRateKey(uint8_t packetFrameDelta,
165                            uint8_t packetFrameKey) const;
166   // Convert the rates: defined relative to total# packets or source# packets
167   uint8_t ConvertFECRate(uint8_t codeRate) const;
168   // Get the average effective recovery from FEC: for random loss model
169   float AvgRecoveryFEC(const VCMProtectionParameters* parameters) const;
170   // Update FEC with protectionFactorD
171   void UpdateProtectionFactorD(uint8_t protectionFactorD);
172   // Update FEC with protectionFactorK
173   void UpdateProtectionFactorK(uint8_t protectionFactorK);
174   // Compute the bits per frame. Account for temporal layers when applicable.
175   int BitsPerFrame(const VCMProtectionParameters* parameters);
176 
177  protected:
178   enum { kUpperLimitFramesFec = 6 };
179   // Thresholds values for the bytes/frame and round trip time, below which we
180   // may turn off FEC, depending on |_numLayers| and |_maxFramesFec|.
181   // Max bytes/frame for VGA, corresponds to ~140k at 25fps.
182   enum { kMaxBytesPerFrameForFec = 700 };
183   // Max bytes/frame for CIF and lower: corresponds to ~80k at 25fps.
184   enum { kMaxBytesPerFrameForFecLow = 400 };
185   // Max bytes/frame for frame size larger than VGA, ~200k at 25fps.
186   enum { kMaxBytesPerFrameForFecHigh = 1000 };
187 
188   const RateControlSettings rate_control_settings_;
189 };
190 
191 class VCMNackFecMethod : public VCMFecMethod {
192  public:
193   VCMNackFecMethod(int64_t lowRttNackThresholdMs,
194                    int64_t highRttNackThresholdMs);
195   ~VCMNackFecMethod() override;
196   bool UpdateParameters(const VCMProtectionParameters* parameters) override;
197   // Get the effective packet loss for ER
198   bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
199   // Get the protection factors
200   bool ProtectionFactor(const VCMProtectionParameters* parameters);
201   // Get the max number of frames the FEC is allowed to be based on.
202   int MaxFramesFec() const override;
203   // Turn off the FEC based on low bitrate and other factors.
204   bool BitRateTooLowForFec(const VCMProtectionParameters* parameters);
205 
206  private:
207   int ComputeMaxFramesFec(const VCMProtectionParameters* parameters);
208 
209   int64_t _lowRttNackMs;
210   int64_t _highRttNackMs;
211   int _maxFramesFec;
212 };
213 
214 class VCMLossProtectionLogic {
215  public:
216   explicit VCMLossProtectionLogic(int64_t nowMs);
217   ~VCMLossProtectionLogic();
218 
219   // Set the protection method to be used
220   //
221   // Input:
222   //        - newMethodType    : New requested protection method type. If one
223   //                           is already set, it will be deleted and replaced
224   void SetMethod(VCMProtectionMethodEnum newMethodType);
225 
226   // Update the round-trip time
227   //
228   // Input:
229   //          - rtt           : Round-trip time in seconds.
230   void UpdateRtt(int64_t rtt);
231 
232   // Update the filtered packet loss.
233   //
234   // Input:
235   //          - packetLossEnc :  The reported packet loss filtered
236   //                             (max window or average)
237   void UpdateFilteredLossPr(uint8_t packetLossEnc);
238 
239   // Update the current target bit rate.
240   //
241   // Input:
242   //          - bitRate          : The current target bit rate in kbits/s
243   void UpdateBitRate(float bitRate);
244 
245   // Update the number of packets per frame estimate, for delta frames
246   //
247   // Input:
248   //          - nPackets         : Number of packets in the latest sent frame.
249   void UpdatePacketsPerFrame(float nPackets, int64_t nowMs);
250 
251   // Update the number of packets per frame estimate, for key frames
252   //
253   // Input:
254   //          - nPackets         : umber of packets in the latest sent frame.
255   void UpdatePacketsPerFrameKey(float nPackets, int64_t nowMs);
256 
257   // Update the keyFrameSize estimate
258   //
259   // Input:
260   //          - keyFrameSize     : The size of the latest sent key frame.
261   void UpdateKeyFrameSize(float keyFrameSize);
262 
263   // Update the frame rate
264   //
265   // Input:
266   //          - frameRate        : The current target frame rate.
UpdateFrameRate(float frameRate)267   void UpdateFrameRate(float frameRate) { _frameRate = frameRate; }
268 
269   // Update the frame size
270   //
271   // Input:
272   //          - width        : The codec frame width.
273   //          - height       : The codec frame height.
274   void UpdateFrameSize(size_t width, size_t height);
275 
276   // Update the number of active layers
277   //
278   // Input:
279   //          - numLayers    : Number of layers used.
280   void UpdateNumLayers(int numLayers);
281 
282   // The amount of packet loss to cover for with FEC.
283   //
284   // Input:
285   //          - fecRateKey      : Packet loss to cover for with FEC when
286   //                              sending key frames.
287   //          - fecRateDelta    : Packet loss to cover for with FEC when
288   //                              sending delta frames.
UpdateFECRates(uint8_t fecRateKey,uint8_t fecRateDelta)289   void UpdateFECRates(uint8_t fecRateKey, uint8_t fecRateDelta) {
290     _fecRateKey = fecRateKey;
291     _fecRateDelta = fecRateDelta;
292   }
293 
294   // Update the protection methods with the current VCMProtectionParameters
295   // and set the requested protection settings.
296   // Return value     : Returns true on update
297   bool UpdateMethod();
298 
299   // Returns the method currently selected.
300   //
301   // Return value                 : The protection method currently selected.
302   VCMProtectionMethod* SelectedMethod() const;
303 
304   // Return the protection type of the currently selected method
305   VCMProtectionMethodEnum SelectedType() const;
306 
307   // Updates the filtered loss for the average and max window packet loss,
308   // and returns the filtered loss probability in the interval [0, 255].
309   // The returned filtered loss value depends on the parameter |filter_mode|.
310   // The input parameter |lossPr255| is the received packet loss.
311 
312   // Return value                 : The filtered loss probability
313   uint8_t FilteredLoss(int64_t nowMs,
314                        FilterPacketLossMode filter_mode,
315                        uint8_t lossPr255);
316 
317   void Reset(int64_t nowMs);
318 
319   void Release();
320 
321  private:
322   // Sets the available loss protection methods.
323   void UpdateMaxLossHistory(uint8_t lossPr255, int64_t now);
324   uint8_t MaxFilteredLossPr(int64_t nowMs) const;
325   std::unique_ptr<VCMProtectionMethod> _selectedMethod;
326   VCMProtectionParameters _currentParameters;
327   int64_t _rtt;
328   float _lossPr;
329   float _bitRate;
330   float _frameRate;
331   float _keyFrameSize;
332   uint8_t _fecRateKey;
333   uint8_t _fecRateDelta;
334   int64_t _lastPrUpdateT;
335   int64_t _lastPacketPerFrameUpdateT;
336   int64_t _lastPacketPerFrameUpdateTKey;
337   rtc::ExpFilter _lossPr255;
338   VCMLossProbabilitySample _lossPrHistory[kLossPrHistorySize];
339   uint8_t _shortMaxLossPr255;
340   rtc::ExpFilter _packetsPerFrame;
341   rtc::ExpFilter _packetsPerFrameKey;
342   size_t _codecWidth;
343   size_t _codecHeight;
344   int _numLayers;
345 };
346 
347 }  // namespace media_optimization
348 }  // namespace webrtc
349 
350 #endif  // MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
351