1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_ 12 #define MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_ 13 14 #include <math.h> 15 #include <stdlib.h> 16 17 #include <memory> 18 19 #include "modules/video_coding/internal_defines.h" 20 #include "rtc_base/experiments/rate_control_settings.h" 21 #include "rtc_base/numerics/exp_filter.h" 22 23 namespace webrtc { 24 namespace media_optimization { 25 26 // Number of time periods used for (max) window filter for packet loss 27 // TODO(marpan): set reasonable window size for filtered packet loss, 28 // adjustment should be based on logged/real data of loss stats/correlation. 29 constexpr int kLossPrHistorySize = 10; 30 31 // 1000 ms, total filter length is (kLossPrHistorySize * 1000) ms 32 constexpr int kLossPrShortFilterWinMs = 1000; 33 34 // The type of filter used on the received packet loss reports. 35 enum FilterPacketLossMode { 36 kNoFilter, // No filtering on received loss. 37 kAvgFilter, // Recursive average filter. 38 kMaxFilter // Max-window filter, over the time interval of: 39 // (kLossPrHistorySize * kLossPrShortFilterWinMs) ms. 40 }; 41 42 // Thresholds for hybrid NACK/FEC 43 // common to media optimization and the jitter buffer. 44 constexpr int64_t kLowRttNackMs = 20; 45 46 // If the RTT is higher than this an extra RTT wont be added to to the jitter 47 // buffer delay. 48 constexpr int kMaxRttDelayThreshold = 500; 49 50 struct VCMProtectionParameters { 51 VCMProtectionParameters(); 52 53 int64_t rtt; 54 float lossPr; 55 float bitRate; 56 float packetsPerFrame; 57 float packetsPerFrameKey; 58 float frameRate; 59 float keyFrameSize; 60 uint8_t fecRateDelta; 61 uint8_t fecRateKey; 62 uint16_t codecWidth; 63 uint16_t codecHeight; 64 int numLayers; 65 }; 66 67 /******************************/ 68 /* VCMProtectionMethod class */ 69 /******************************/ 70 71 enum VCMProtectionMethodEnum { kNack, kFec, kNackFec, kNone }; 72 73 class VCMLossProbabilitySample { 74 public: VCMLossProbabilitySample()75 VCMLossProbabilitySample() : lossPr255(0), timeMs(-1) {} 76 77 uint8_t lossPr255; 78 int64_t timeMs; 79 }; 80 81 class VCMProtectionMethod { 82 public: 83 VCMProtectionMethod(); 84 virtual ~VCMProtectionMethod(); 85 86 // Updates the efficiency of the method using the parameters provided 87 // 88 // Input: 89 // - parameters : Parameters used to calculate efficiency 90 // 91 // Return value : True if this method is recommended in 92 // the given conditions. 93 virtual bool UpdateParameters(const VCMProtectionParameters* parameters) = 0; 94 95 // Returns the protection type 96 // 97 // Return value : The protection type 98 VCMProtectionMethodEnum Type() const; 99 100 // Returns the effective packet loss for ER, required by this protection 101 // method 102 // 103 // Return value : Required effective packet loss 104 virtual uint8_t RequiredPacketLossER(); 105 106 // Extracts the FEC protection factor for Key frame, required by this 107 // protection method 108 // 109 // Return value : Required protectionFactor for Key frame 110 virtual uint8_t RequiredProtectionFactorK(); 111 112 // Extracts the FEC protection factor for Delta frame, required by this 113 // protection method 114 // 115 // Return value : Required protectionFactor for delta frame 116 virtual uint8_t RequiredProtectionFactorD(); 117 118 // Extracts whether the FEC Unequal protection (UEP) is used for Key frame. 119 // 120 // Return value : Required Unequal protection on/off state. 121 virtual bool RequiredUepProtectionK(); 122 123 // Extracts whether the the FEC Unequal protection (UEP) is used for Delta 124 // frame. 125 // 126 // Return value : Required Unequal protection on/off state. 127 virtual bool RequiredUepProtectionD(); 128 129 virtual int MaxFramesFec() const; 130 131 protected: 132 uint8_t _effectivePacketLoss; 133 uint8_t _protectionFactorK; 134 uint8_t _protectionFactorD; 135 // Estimation of residual loss after the FEC 136 float _scaleProtKey; 137 int32_t _maxPayloadSize; 138 139 bool _useUepProtectionK; 140 bool _useUepProtectionD; 141 float _corrFecCost; 142 VCMProtectionMethodEnum _type; 143 }; 144 145 class VCMNackMethod : public VCMProtectionMethod { 146 public: 147 VCMNackMethod(); 148 ~VCMNackMethod() override; 149 bool UpdateParameters(const VCMProtectionParameters* parameters) override; 150 // Get the effective packet loss 151 bool EffectivePacketLoss(const VCMProtectionParameters* parameter); 152 }; 153 154 class VCMFecMethod : public VCMProtectionMethod { 155 public: 156 VCMFecMethod(); 157 ~VCMFecMethod() override; 158 bool UpdateParameters(const VCMProtectionParameters* parameters) override; 159 // Get the effective packet loss for ER 160 bool EffectivePacketLoss(const VCMProtectionParameters* parameters); 161 // Get the FEC protection factors 162 bool ProtectionFactor(const VCMProtectionParameters* parameters); 163 // Get the boost for key frame protection 164 uint8_t BoostCodeRateKey(uint8_t packetFrameDelta, 165 uint8_t packetFrameKey) const; 166 // Convert the rates: defined relative to total# packets or source# packets 167 uint8_t ConvertFECRate(uint8_t codeRate) const; 168 // Get the average effective recovery from FEC: for random loss model 169 float AvgRecoveryFEC(const VCMProtectionParameters* parameters) const; 170 // Update FEC with protectionFactorD 171 void UpdateProtectionFactorD(uint8_t protectionFactorD); 172 // Update FEC with protectionFactorK 173 void UpdateProtectionFactorK(uint8_t protectionFactorK); 174 // Compute the bits per frame. Account for temporal layers when applicable. 175 int BitsPerFrame(const VCMProtectionParameters* parameters); 176 177 protected: 178 static constexpr int kUpperLimitFramesFec = 6; 179 // Thresholds values for the bytes/frame and round trip time, below which we 180 // may turn off FEC, depending on `_numLayers` and `_maxFramesFec`. 181 // Max bytes/frame for VGA, corresponds to ~140k at 25fps. 182 static constexpr int kMaxBytesPerFrameForFec = 700; 183 // Max bytes/frame for CIF and lower: corresponds to ~80k at 25fps. 184 static constexpr int kMaxBytesPerFrameForFecLow = 400; 185 // Max bytes/frame for frame size larger than VGA, ~200k at 25fps. 186 static constexpr int kMaxBytesPerFrameForFecHigh = 1000; 187 188 const RateControlSettings rate_control_settings_; 189 }; 190 191 class VCMNackFecMethod : public VCMFecMethod { 192 public: 193 VCMNackFecMethod(int64_t lowRttNackThresholdMs, 194 int64_t highRttNackThresholdMs); 195 ~VCMNackFecMethod() override; 196 bool UpdateParameters(const VCMProtectionParameters* parameters) override; 197 // Get the effective packet loss for ER 198 bool EffectivePacketLoss(const VCMProtectionParameters* parameters); 199 // Get the protection factors 200 bool ProtectionFactor(const VCMProtectionParameters* parameters); 201 // Get the max number of frames the FEC is allowed to be based on. 202 int MaxFramesFec() const override; 203 // Turn off the FEC based on low bitrate and other factors. 204 bool BitRateTooLowForFec(const VCMProtectionParameters* parameters); 205 206 private: 207 int ComputeMaxFramesFec(const VCMProtectionParameters* parameters); 208 209 int64_t _lowRttNackMs; 210 int64_t _highRttNackMs; 211 int _maxFramesFec; 212 }; 213 214 class VCMLossProtectionLogic { 215 public: 216 explicit VCMLossProtectionLogic(int64_t nowMs); 217 ~VCMLossProtectionLogic(); 218 219 // Set the protection method to be used 220 // 221 // Input: 222 // - newMethodType : New requested protection method type. If one 223 // is already set, it will be deleted and replaced 224 void SetMethod(VCMProtectionMethodEnum newMethodType); 225 226 // Update the round-trip time 227 // 228 // Input: 229 // - rtt : Round-trip time in seconds. 230 void UpdateRtt(int64_t rtt); 231 232 // Update the filtered packet loss. 233 // 234 // Input: 235 // - packetLossEnc : The reported packet loss filtered 236 // (max window or average) 237 void UpdateFilteredLossPr(uint8_t packetLossEnc); 238 239 // Update the current target bit rate. 240 // 241 // Input: 242 // - bitRate : The current target bit rate in kbits/s 243 void UpdateBitRate(float bitRate); 244 245 // Update the number of packets per frame estimate, for delta frames 246 // 247 // Input: 248 // - nPackets : Number of packets in the latest sent frame. 249 void UpdatePacketsPerFrame(float nPackets, int64_t nowMs); 250 251 // Update the number of packets per frame estimate, for key frames 252 // 253 // Input: 254 // - nPackets : umber of packets in the latest sent frame. 255 void UpdatePacketsPerFrameKey(float nPackets, int64_t nowMs); 256 257 // Update the keyFrameSize estimate 258 // 259 // Input: 260 // - keyFrameSize : The size of the latest sent key frame. 261 void UpdateKeyFrameSize(float keyFrameSize); 262 263 // Update the frame rate 264 // 265 // Input: 266 // - frameRate : The current target frame rate. UpdateFrameRate(float frameRate)267 void UpdateFrameRate(float frameRate) { _frameRate = frameRate; } 268 269 // Update the frame size 270 // 271 // Input: 272 // - width : The codec frame width. 273 // - height : The codec frame height. 274 void UpdateFrameSize(size_t width, size_t height); 275 276 // Update the number of active layers 277 // 278 // Input: 279 // - numLayers : Number of layers used. 280 void UpdateNumLayers(int numLayers); 281 282 // The amount of packet loss to cover for with FEC. 283 // 284 // Input: 285 // - fecRateKey : Packet loss to cover for with FEC when 286 // sending key frames. 287 // - fecRateDelta : Packet loss to cover for with FEC when 288 // sending delta frames. UpdateFECRates(uint8_t fecRateKey,uint8_t fecRateDelta)289 void UpdateFECRates(uint8_t fecRateKey, uint8_t fecRateDelta) { 290 _fecRateKey = fecRateKey; 291 _fecRateDelta = fecRateDelta; 292 } 293 294 // Update the protection methods with the current VCMProtectionParameters 295 // and set the requested protection settings. 296 // Return value : Returns true on update 297 bool UpdateMethod(); 298 299 // Returns the method currently selected. 300 // 301 // Return value : The protection method currently selected. 302 VCMProtectionMethod* SelectedMethod() const; 303 304 // Return the protection type of the currently selected method 305 VCMProtectionMethodEnum SelectedType() const; 306 307 // Updates the filtered loss for the average and max window packet loss, 308 // and returns the filtered loss probability in the interval [0, 255]. 309 // The returned filtered loss value depends on the parameter `filter_mode`. 310 // The input parameter `lossPr255` is the received packet loss. 311 312 // Return value : The filtered loss probability 313 uint8_t FilteredLoss(int64_t nowMs, 314 FilterPacketLossMode filter_mode, 315 uint8_t lossPr255); 316 317 void Reset(int64_t nowMs); 318 319 void Release(); 320 321 private: 322 // Sets the available loss protection methods. 323 void UpdateMaxLossHistory(uint8_t lossPr255, int64_t now); 324 uint8_t MaxFilteredLossPr(int64_t nowMs) const; 325 std::unique_ptr<VCMProtectionMethod> _selectedMethod; 326 VCMProtectionParameters _currentParameters; 327 int64_t _rtt; 328 float _lossPr; 329 float _bitRate; 330 float _frameRate; 331 float _keyFrameSize; 332 uint8_t _fecRateKey; 333 uint8_t _fecRateDelta; 334 int64_t _lastPrUpdateT; 335 int64_t _lastPacketPerFrameUpdateT; 336 int64_t _lastPacketPerFrameUpdateTKey; 337 rtc::ExpFilter _lossPr255; 338 VCMLossProbabilitySample _lossPrHistory[kLossPrHistorySize]; 339 uint8_t _shortMaxLossPr255; 340 rtc::ExpFilter _packetsPerFrame; 341 rtc::ExpFilter _packetsPerFrameKey; 342 size_t _codecWidth; 343 size_t _codecHeight; 344 int _numLayers; 345 }; 346 347 } // namespace media_optimization 348 } // namespace webrtc 349 350 #endif // MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_ 351