1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_SEND_STATISTICS_PROXY_H_ 12 #define VIDEO_SEND_STATISTICS_PROXY_H_ 13 14 #include <array> 15 #include <map> 16 #include <memory> 17 #include <string> 18 #include <vector> 19 20 #include "api/video/video_codec_constants.h" 21 #include "api/video/video_stream_encoder_observer.h" 22 #include "api/video_codecs/video_encoder_config.h" 23 #include "call/video_send_stream.h" 24 #include "modules/include/module_common_types_public.h" 25 #include "modules/rtp_rtcp/include/report_block_data.h" 26 #include "modules/video_coding/include/video_codec_interface.h" 27 #include "modules/video_coding/include/video_coding_defines.h" 28 #include "rtc_base/numerics/exp_filter.h" 29 #include "rtc_base/rate_tracker.h" 30 #include "rtc_base/synchronization/mutex.h" 31 #include "rtc_base/thread_annotations.h" 32 #include "system_wrappers/include/clock.h" 33 #include "video/quality_limitation_reason_tracker.h" 34 #include "video/report_block_stats.h" 35 #include "video/stats_counter.h" 36 37 namespace webrtc { 38 39 class SendStatisticsProxy : public VideoStreamEncoderObserver, 40 public RtcpStatisticsCallback, 41 public ReportBlockDataObserver, 42 public RtcpPacketTypeCounterObserver, 43 public StreamDataCountersCallback, 44 public BitrateStatisticsObserver, 45 public FrameCountObserver, 46 public SendSideDelayObserver { 47 public: 48 static const int kStatsTimeoutMs; 49 // Number of required samples to be collected before a metric is added 50 // to a rtc histogram. 51 static const int kMinRequiredMetricsSamples = 200; 52 53 SendStatisticsProxy(Clock* clock, 54 const VideoSendStream::Config& config, 55 VideoEncoderConfig::ContentType content_type); 56 ~SendStatisticsProxy() override; 57 58 virtual VideoSendStream::Stats GetStats(); 59 60 void OnSendEncodedImage(const EncodedImage& encoded_image, 61 const CodecSpecificInfo* codec_info) override; 62 63 void OnEncoderImplementationChanged( 64 const std::string& implementation_name) override; 65 66 // Used to update incoming frame rate. 67 void OnIncomingFrame(int width, int height) override; 68 69 // Dropped frame stats. 70 void OnFrameDropped(DropReason) override; 71 72 // Adaptation stats. 73 void OnAdaptationChanged( 74 VideoAdaptationReason reason, 75 const VideoAdaptationCounters& cpu_counters, 76 const VideoAdaptationCounters& quality_counters) override; 77 void ClearAdaptationStats() override; 78 void UpdateAdaptationSettings(AdaptationSettings cpu_settings, 79 AdaptationSettings quality_settings) override; 80 81 void OnBitrateAllocationUpdated( 82 const VideoCodec& codec, 83 const VideoBitrateAllocation& allocation) override; 84 85 void OnEncoderInternalScalerUpdate(bool is_scaled) override; 86 87 void OnMinPixelLimitReached() override; 88 void OnInitialQualityResolutionAdaptDown() override; 89 90 void OnSuspendChange(bool is_suspended) override; 91 void OnInactiveSsrc(uint32_t ssrc); 92 93 // Used to indicate change in content type, which may require a change in 94 // how stats are collected. 95 void OnEncoderReconfigured(const VideoEncoderConfig& encoder_config, 96 const std::vector<VideoStream>& streams) override; 97 98 // Used to update the encoder target rate. 99 void OnSetEncoderTargetRate(uint32_t bitrate_bps); 100 101 // Implements CpuOveruseMetricsObserver. 102 void OnEncodedFrameTimeMeasured(int encode_time_ms, 103 int encode_usage_percent) override; 104 105 int GetInputFrameRate() const override; 106 int GetSendFrameRate() const; 107 108 protected: 109 // From RtcpStatisticsCallback. 110 void StatisticsUpdated(const RtcpStatistics& statistics, 111 uint32_t ssrc) override; 112 // From ReportBlockDataObserver. 113 void OnReportBlockDataUpdated(ReportBlockData report_block_data) override; 114 // From RtcpPacketTypeCounterObserver. 115 void RtcpPacketTypesCounterUpdated( 116 uint32_t ssrc, 117 const RtcpPacketTypeCounter& packet_counter) override; 118 // From StreamDataCountersCallback. 119 void DataCountersUpdated(const StreamDataCounters& counters, 120 uint32_t ssrc) override; 121 122 // From BitrateStatisticsObserver. 123 void Notify(uint32_t total_bitrate_bps, 124 uint32_t retransmit_bitrate_bps, 125 uint32_t ssrc) override; 126 127 // From FrameCountObserver. 128 void FrameCountUpdated(const FrameCounts& frame_counts, 129 uint32_t ssrc) override; 130 131 void SendSideDelayUpdated(int avg_delay_ms, 132 int max_delay_ms, 133 uint64_t total_delay_ms, 134 uint32_t ssrc) override; 135 136 private: 137 class SampleCounter { 138 public: SampleCounter()139 SampleCounter() : sum(0), num_samples(0) {} ~SampleCounter()140 ~SampleCounter() {} 141 void Add(int sample); 142 int Avg(int64_t min_required_samples) const; 143 144 private: 145 int64_t sum; 146 int64_t num_samples; 147 }; 148 class BoolSampleCounter { 149 public: BoolSampleCounter()150 BoolSampleCounter() : sum(0), num_samples(0) {} ~BoolSampleCounter()151 ~BoolSampleCounter() {} 152 void Add(bool sample); 153 void Add(bool sample, int64_t count); 154 int Percent(int64_t min_required_samples) const; 155 int Permille(int64_t min_required_samples) const; 156 157 private: 158 int Fraction(int64_t min_required_samples, float multiplier) const; 159 int64_t sum; 160 int64_t num_samples; 161 }; 162 struct StatsUpdateTimes { StatsUpdateTimesStatsUpdateTimes163 StatsUpdateTimes() : resolution_update_ms(0), bitrate_update_ms(0) {} 164 int64_t resolution_update_ms; 165 int64_t bitrate_update_ms; 166 }; 167 struct TargetRateUpdates { TargetRateUpdatesTargetRateUpdates168 TargetRateUpdates() 169 : pause_resume_events(0), last_paused_or_resumed(false), last_ms(-1) {} 170 int pause_resume_events; 171 bool last_paused_or_resumed; 172 int64_t last_ms; 173 }; 174 struct FallbackEncoderInfo { 175 FallbackEncoderInfo(); 176 bool is_possible = true; 177 bool is_active = false; 178 int on_off_events = 0; 179 int64_t elapsed_ms = 0; 180 absl::optional<int64_t> last_update_ms; 181 const int max_frame_diff_ms = 2000; 182 }; 183 struct FallbackEncoderInfoDisabled { 184 bool is_possible = true; 185 bool min_pixel_limit_reached = false; 186 }; 187 struct StatsTimer { 188 void Start(int64_t now_ms); 189 void Stop(int64_t now_ms); 190 void Restart(int64_t now_ms); 191 int64_t start_ms = -1; 192 int64_t total_ms = 0; 193 }; 194 struct QpCounters { 195 SampleCounter vp8; // QP range: 0-127. 196 SampleCounter vp9; // QP range: 0-255. 197 SampleCounter h264; // QP range: 0-51. 198 }; 199 struct AdaptChanges { 200 int down = 0; 201 int up = 0; 202 }; 203 204 // Map holding encoded frames (mapped by timestamp). 205 // If simulcast layers are encoded on different threads, there is no guarantee 206 // that one frame of all layers are encoded before the next start. 207 struct TimestampOlderThan { operatorTimestampOlderThan208 bool operator()(uint32_t ts1, uint32_t ts2) const { 209 return IsNewerTimestamp(ts2, ts1); 210 } 211 }; 212 struct Frame { FrameFrame213 Frame(int64_t send_ms, uint32_t width, uint32_t height, int simulcast_idx) 214 : send_ms(send_ms), 215 max_width(width), 216 max_height(height), 217 max_simulcast_idx(simulcast_idx) {} 218 const int64_t 219 send_ms; // Time when first frame with this timestamp is sent. 220 uint32_t max_width; // Max width with this timestamp. 221 uint32_t max_height; // Max height with this timestamp. 222 int max_simulcast_idx; // Max simulcast index with this timestamp. 223 }; 224 typedef std::map<uint32_t, Frame, TimestampOlderThan> EncodedFrameMap; 225 226 void PurgeOldStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 227 VideoSendStream::StreamStats* GetStatsEntry(uint32_t ssrc) 228 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 229 230 struct MaskedAdaptationCounts { 231 absl::optional<int> resolution_adaptations = absl::nullopt; 232 absl::optional<int> num_framerate_reductions = absl::nullopt; 233 }; 234 235 struct Adaptations { 236 public: 237 MaskedAdaptationCounts MaskedCpuCounts() const; 238 MaskedAdaptationCounts MaskedQualityCounts() const; 239 240 void set_cpu_counts(const VideoAdaptationCounters& cpu_counts); 241 void set_quality_counts(const VideoAdaptationCounters& quality_counts); 242 243 VideoAdaptationCounters cpu_counts() const; 244 VideoAdaptationCounters quality_counts() const; 245 246 void UpdateMaskingSettings(AdaptationSettings cpu_settings, 247 AdaptationSettings quality_settings); 248 249 private: 250 VideoAdaptationCounters cpu_counts_; 251 AdaptationSettings cpu_settings_; 252 VideoAdaptationCounters quality_counts_; 253 AdaptationSettings quality_settings_; 254 255 MaskedAdaptationCounts Mask(const VideoAdaptationCounters& counters, 256 const AdaptationSettings& settings) const; 257 }; 258 259 void SetAdaptTimer(const MaskedAdaptationCounts& counts, StatsTimer* timer) 260 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 261 void UpdateAdaptationStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 262 void TryUpdateInitialQualityResolutionAdaptUp( 263 absl::optional<int> old_quality_downscales, 264 absl::optional<int> updated_quality_downscales) 265 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 266 267 void UpdateEncoderFallbackStats(const CodecSpecificInfo* codec_info, 268 int pixels, 269 int simulcast_index) 270 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 271 void UpdateFallbackDisabledStats(const CodecSpecificInfo* codec_info, 272 int pixels, 273 int simulcast_index) 274 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 275 276 Clock* const clock_; 277 const std::string payload_name_; 278 const RtpConfig rtp_config_; 279 const absl::optional<int> fallback_max_pixels_; 280 const absl::optional<int> fallback_max_pixels_disabled_; 281 mutable Mutex mutex_; 282 VideoEncoderConfig::ContentType content_type_ RTC_GUARDED_BY(mutex_); 283 const int64_t start_ms_; 284 VideoSendStream::Stats stats_ RTC_GUARDED_BY(mutex_); 285 std::map<uint32_t, StatsUpdateTimes> update_times_ RTC_GUARDED_BY(mutex_); 286 rtc::ExpFilter encode_time_ RTC_GUARDED_BY(mutex_); 287 QualityLimitationReasonTracker quality_limitation_reason_tracker_ 288 RTC_GUARDED_BY(mutex_); 289 rtc::RateTracker media_byte_rate_tracker_ RTC_GUARDED_BY(mutex_); 290 rtc::RateTracker encoded_frame_rate_tracker_ RTC_GUARDED_BY(mutex_); 291 std::map<uint32_t, std::unique_ptr<rtc::RateTracker>> 292 encoded_frame_rate_trackers_ RTC_GUARDED_BY(mutex_); 293 294 absl::optional<int64_t> last_outlier_timestamp_ RTC_GUARDED_BY(mutex_); 295 296 int last_num_spatial_layers_ RTC_GUARDED_BY(mutex_); 297 int last_num_simulcast_streams_ RTC_GUARDED_BY(mutex_); 298 std::array<bool, kMaxSpatialLayers> last_spatial_layer_use_ 299 RTC_GUARDED_BY(mutex_); 300 // Indicates if the latest bitrate allocation had layers disabled by low 301 // available bandwidth. 302 bool bw_limited_layers_ RTC_GUARDED_BY(mutex_); 303 // Indicastes if the encoder internally downscales input image. 304 bool internal_encoder_scaler_ RTC_GUARDED_BY(mutex_); 305 Adaptations adaptation_limitations_ RTC_GUARDED_BY(mutex_); 306 307 struct EncoderChangeEvent { 308 std::string previous_encoder_implementation; 309 std::string new_encoder_implementation; 310 }; 311 // Stores the last change in encoder implementation in an optional, so that 312 // the event can be consumed. 313 absl::optional<EncoderChangeEvent> encoder_changed_; 314 315 // Contains stats used for UMA histograms. These stats will be reset if 316 // content type changes between real-time video and screenshare, since these 317 // will be reported separately. 318 struct UmaSamplesContainer { 319 UmaSamplesContainer(const char* prefix, 320 const VideoSendStream::Stats& start_stats, 321 Clock* clock); 322 ~UmaSamplesContainer(); 323 324 void UpdateHistograms(const RtpConfig& rtp_config, 325 const VideoSendStream::Stats& current_stats); 326 327 void InitializeBitrateCounters(const VideoSendStream::Stats& stats); 328 329 bool InsertEncodedFrame(const EncodedImage& encoded_frame, 330 int simulcast_idx); 331 void RemoveOld(int64_t now_ms); 332 333 const std::string uma_prefix_; 334 Clock* const clock_; 335 SampleCounter input_width_counter_; 336 SampleCounter input_height_counter_; 337 SampleCounter sent_width_counter_; 338 SampleCounter sent_height_counter_; 339 SampleCounter encode_time_counter_; 340 BoolSampleCounter key_frame_counter_; 341 BoolSampleCounter quality_limited_frame_counter_; 342 SampleCounter quality_downscales_counter_; 343 BoolSampleCounter cpu_limited_frame_counter_; 344 BoolSampleCounter bw_limited_frame_counter_; 345 SampleCounter bw_resolutions_disabled_counter_; 346 SampleCounter delay_counter_; 347 SampleCounter max_delay_counter_; 348 rtc::RateTracker input_frame_rate_tracker_; 349 RateCounter input_fps_counter_; 350 RateCounter sent_fps_counter_; 351 RateAccCounter total_byte_counter_; 352 RateAccCounter media_byte_counter_; 353 RateAccCounter rtx_byte_counter_; 354 RateAccCounter padding_byte_counter_; 355 RateAccCounter retransmit_byte_counter_; 356 RateAccCounter fec_byte_counter_; 357 int64_t first_rtcp_stats_time_ms_; 358 int64_t first_rtp_stats_time_ms_; 359 StatsTimer cpu_adapt_timer_; 360 StatsTimer quality_adapt_timer_; 361 BoolSampleCounter paused_time_counter_; 362 TargetRateUpdates target_rate_updates_; 363 BoolSampleCounter fallback_active_counter_; 364 FallbackEncoderInfo fallback_info_; 365 FallbackEncoderInfoDisabled fallback_info_disabled_; 366 ReportBlockStats report_block_stats_; 367 const VideoSendStream::Stats start_stats_; 368 size_t num_streams_; // Number of configured streams to encoder. 369 size_t num_pixels_highest_stream_; 370 EncodedFrameMap encoded_frames_; 371 AdaptChanges initial_quality_changes_; 372 373 std::map<int, QpCounters> 374 qp_counters_; // QP counters mapped by spatial idx. 375 }; 376 377 std::unique_ptr<UmaSamplesContainer> uma_container_ RTC_GUARDED_BY(mutex_); 378 }; 379 380 } // namespace webrtc 381 #endif // VIDEO_SEND_STATISTICS_PROXY_H_ 382