1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_SEND_STATISTICS_PROXY_H_ 12 #define VIDEO_SEND_STATISTICS_PROXY_H_ 13 14 #include <array> 15 #include <map> 16 #include <memory> 17 #include <string> 18 #include <vector> 19 20 #include "api/field_trials_view.h" 21 #include "api/video/video_codec_constants.h" 22 #include "call/video_send_stream.h" 23 #include "modules/include/module_common_types_public.h" 24 #include "modules/rtp_rtcp/include/report_block_data.h" 25 #include "modules/video_coding/include/video_codec_interface.h" 26 #include "modules/video_coding/include/video_coding_defines.h" 27 #include "rtc_base/numerics/exp_filter.h" 28 #include "rtc_base/rate_tracker.h" 29 #include "rtc_base/synchronization/mutex.h" 30 #include "rtc_base/thread_annotations.h" 31 #include "system_wrappers/include/clock.h" 32 #include "video/config/video_encoder_config.h" 33 #include "video/quality_limitation_reason_tracker.h" 34 #include "video/report_block_stats.h" 35 #include "video/stats_counter.h" 36 #include "video/video_stream_encoder_observer.h" 37 38 namespace webrtc { 39 40 class SendStatisticsProxy : public VideoStreamEncoderObserver, 41 public ReportBlockDataObserver, 42 public RtcpPacketTypeCounterObserver, 43 public StreamDataCountersCallback, 44 public BitrateStatisticsObserver, 45 public FrameCountObserver, 46 public SendSideDelayObserver { 47 public: 48 static const int kStatsTimeoutMs; 49 // Number of required samples to be collected before a metric is added 50 // to a rtc histogram. 51 static const int kMinRequiredMetricsSamples = 200; 52 53 SendStatisticsProxy(Clock* clock, 54 const VideoSendStream::Config& config, 55 VideoEncoderConfig::ContentType content_type, 56 const FieldTrialsView& field_trials); 57 ~SendStatisticsProxy() override; 58 59 virtual VideoSendStream::Stats GetStats(); 60 61 void OnSendEncodedImage(const EncodedImage& encoded_image, 62 const CodecSpecificInfo* codec_info) override; 63 64 void OnEncoderImplementationChanged( 65 EncoderImplementation implementation) override; 66 67 // Used to update incoming frame rate. 68 void OnIncomingFrame(int width, int height) override; 69 70 // Dropped frame stats. 71 void OnFrameDropped(DropReason) override; 72 73 // Adaptation stats. 74 void OnAdaptationChanged( 75 VideoAdaptationReason reason, 76 const VideoAdaptationCounters& cpu_counters, 77 const VideoAdaptationCounters& quality_counters) override; 78 void ClearAdaptationStats() override; 79 void UpdateAdaptationSettings(AdaptationSettings cpu_settings, 80 AdaptationSettings quality_settings) override; 81 82 void OnBitrateAllocationUpdated( 83 const VideoCodec& codec, 84 const VideoBitrateAllocation& allocation) override; 85 86 void OnEncoderInternalScalerUpdate(bool is_scaled) override; 87 88 void OnMinPixelLimitReached() override; 89 void OnInitialQualityResolutionAdaptDown() override; 90 91 void OnSuspendChange(bool is_suspended) override; 92 void OnInactiveSsrc(uint32_t ssrc); 93 94 // Used to indicate change in content type, which may require a change in 95 // how stats are collected. 96 void OnEncoderReconfigured(const VideoEncoderConfig& encoder_config, 97 const std::vector<VideoStream>& streams) override; 98 99 // Used to update the encoder target rate. 100 void OnSetEncoderTargetRate(uint32_t bitrate_bps); 101 102 // Implements CpuOveruseMetricsObserver. 103 void OnEncodedFrameTimeMeasured(int encode_time_ms, 104 int encode_usage_percent) override; 105 106 int GetInputFrameRate() const override; 107 int GetSendFrameRate() const; 108 109 protected: 110 // From ReportBlockDataObserver. 111 void OnReportBlockDataUpdated(ReportBlockData report_block_data) override; 112 // From RtcpPacketTypeCounterObserver. 113 void RtcpPacketTypesCounterUpdated( 114 uint32_t ssrc, 115 const RtcpPacketTypeCounter& packet_counter) override; 116 // From StreamDataCountersCallback. 117 void DataCountersUpdated(const StreamDataCounters& counters, 118 uint32_t ssrc) override; 119 120 // From BitrateStatisticsObserver. 121 void Notify(uint32_t total_bitrate_bps, 122 uint32_t retransmit_bitrate_bps, 123 uint32_t ssrc) override; 124 125 // From FrameCountObserver. 126 void FrameCountUpdated(const FrameCounts& frame_counts, 127 uint32_t ssrc) override; 128 129 // From SendSideDelayObserver. 130 void SendSideDelayUpdated(int avg_delay_ms, 131 int max_delay_ms, 132 uint32_t ssrc) override; 133 134 private: 135 class SampleCounter { 136 public: SampleCounter()137 SampleCounter() : sum(0), num_samples(0) {} ~SampleCounter()138 ~SampleCounter() {} 139 void Add(int sample); 140 int Avg(int64_t min_required_samples) const; 141 142 private: 143 int64_t sum; 144 int64_t num_samples; 145 }; 146 class BoolSampleCounter { 147 public: BoolSampleCounter()148 BoolSampleCounter() : sum(0), num_samples(0) {} ~BoolSampleCounter()149 ~BoolSampleCounter() {} 150 void Add(bool sample); 151 void Add(bool sample, int64_t count); 152 int Percent(int64_t min_required_samples) const; 153 int Permille(int64_t min_required_samples) const; 154 155 private: 156 int Fraction(int64_t min_required_samples, float multiplier) const; 157 int64_t sum; 158 int64_t num_samples; 159 }; 160 struct StatsUpdateTimes { StatsUpdateTimesStatsUpdateTimes161 StatsUpdateTimes() : resolution_update_ms(0), bitrate_update_ms(0) {} 162 int64_t resolution_update_ms; 163 int64_t bitrate_update_ms; 164 }; 165 struct TargetRateUpdates { TargetRateUpdatesTargetRateUpdates166 TargetRateUpdates() 167 : pause_resume_events(0), last_paused_or_resumed(false), last_ms(-1) {} 168 int pause_resume_events; 169 bool last_paused_or_resumed; 170 int64_t last_ms; 171 }; 172 struct FallbackEncoderInfo { 173 FallbackEncoderInfo(); 174 bool is_possible = true; 175 bool is_active = false; 176 int on_off_events = 0; 177 int64_t elapsed_ms = 0; 178 absl::optional<int64_t> last_update_ms; 179 const int max_frame_diff_ms = 2000; 180 }; 181 struct FallbackEncoderInfoDisabled { 182 bool is_possible = true; 183 bool min_pixel_limit_reached = false; 184 }; 185 struct StatsTimer { 186 void Start(int64_t now_ms); 187 void Stop(int64_t now_ms); 188 void Restart(int64_t now_ms); 189 int64_t start_ms = -1; 190 int64_t total_ms = 0; 191 }; 192 struct QpCounters { 193 SampleCounter vp8; // QP range: 0-127. 194 SampleCounter vp9; // QP range: 0-255. 195 SampleCounter h264; // QP range: 0-51. 196 }; 197 struct AdaptChanges { 198 int down = 0; 199 int up = 0; 200 }; 201 202 // Map holding encoded frames (mapped by timestamp). 203 // If simulcast layers are encoded on different threads, there is no guarantee 204 // that one frame of all layers are encoded before the next start. 205 struct TimestampOlderThan { operatorTimestampOlderThan206 bool operator()(uint32_t ts1, uint32_t ts2) const { 207 return IsNewerTimestamp(ts2, ts1); 208 } 209 }; 210 struct Frame { FrameFrame211 Frame(int64_t send_ms, uint32_t width, uint32_t height, int simulcast_idx) 212 : send_ms(send_ms), 213 max_width(width), 214 max_height(height), 215 max_simulcast_idx(simulcast_idx) {} 216 const int64_t 217 send_ms; // Time when first frame with this timestamp is sent. 218 uint32_t max_width; // Max width with this timestamp. 219 uint32_t max_height; // Max height with this timestamp. 220 int max_simulcast_idx; // Max simulcast index with this timestamp. 221 }; 222 typedef std::map<uint32_t, Frame, TimestampOlderThan> EncodedFrameMap; 223 224 void PurgeOldStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 225 VideoSendStream::StreamStats* GetStatsEntry(uint32_t ssrc) 226 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 227 228 struct MaskedAdaptationCounts { 229 absl::optional<int> resolution_adaptations = absl::nullopt; 230 absl::optional<int> num_framerate_reductions = absl::nullopt; 231 }; 232 233 struct Adaptations { 234 public: 235 MaskedAdaptationCounts MaskedCpuCounts() const; 236 MaskedAdaptationCounts MaskedQualityCounts() const; 237 238 void set_cpu_counts(const VideoAdaptationCounters& cpu_counts); 239 void set_quality_counts(const VideoAdaptationCounters& quality_counts); 240 241 VideoAdaptationCounters cpu_counts() const; 242 VideoAdaptationCounters quality_counts() const; 243 244 void UpdateMaskingSettings(AdaptationSettings cpu_settings, 245 AdaptationSettings quality_settings); 246 247 private: 248 VideoAdaptationCounters cpu_counts_; 249 AdaptationSettings cpu_settings_; 250 VideoAdaptationCounters quality_counts_; 251 AdaptationSettings quality_settings_; 252 253 MaskedAdaptationCounts Mask(const VideoAdaptationCounters& counters, 254 const AdaptationSettings& settings) const; 255 }; 256 257 void SetAdaptTimer(const MaskedAdaptationCounts& counts, StatsTimer* timer) 258 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 259 void UpdateAdaptationStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 260 void TryUpdateInitialQualityResolutionAdaptUp( 261 absl::optional<int> old_quality_downscales, 262 absl::optional<int> updated_quality_downscales) 263 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 264 265 void UpdateEncoderFallbackStats(const CodecSpecificInfo* codec_info, 266 int pixels, 267 int simulcast_index) 268 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 269 void UpdateFallbackDisabledStats(const CodecSpecificInfo* codec_info, 270 int pixels, 271 int simulcast_index) 272 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 273 274 Clock* const clock_; 275 const std::string payload_name_; 276 const RtpConfig rtp_config_; 277 const absl::optional<int> fallback_max_pixels_; 278 const absl::optional<int> fallback_max_pixels_disabled_; 279 mutable Mutex mutex_; 280 VideoEncoderConfig::ContentType content_type_ RTC_GUARDED_BY(mutex_); 281 const int64_t start_ms_; 282 VideoSendStream::Stats stats_ RTC_GUARDED_BY(mutex_); 283 std::map<uint32_t, StatsUpdateTimes> update_times_ RTC_GUARDED_BY(mutex_); 284 rtc::ExpFilter encode_time_ RTC_GUARDED_BY(mutex_); 285 QualityLimitationReasonTracker quality_limitation_reason_tracker_ 286 RTC_GUARDED_BY(mutex_); 287 rtc::RateTracker media_byte_rate_tracker_ RTC_GUARDED_BY(mutex_); 288 rtc::RateTracker encoded_frame_rate_tracker_ RTC_GUARDED_BY(mutex_); 289 // Rate trackers mapped by ssrc. 290 std::map<uint32_t, std::unique_ptr<rtc::RateTracker>> 291 encoded_frame_rate_trackers_ RTC_GUARDED_BY(mutex_); 292 293 absl::optional<int64_t> last_outlier_timestamp_ RTC_GUARDED_BY(mutex_); 294 295 int last_num_spatial_layers_ RTC_GUARDED_BY(mutex_); 296 int last_num_simulcast_streams_ RTC_GUARDED_BY(mutex_); 297 std::array<bool, kMaxSpatialLayers> last_spatial_layer_use_ 298 RTC_GUARDED_BY(mutex_); 299 // Indicates if the latest bitrate allocation had layers disabled by low 300 // available bandwidth. 301 bool bw_limited_layers_ RTC_GUARDED_BY(mutex_); 302 // Indicastes if the encoder internally downscales input image. 303 bool internal_encoder_scaler_ RTC_GUARDED_BY(mutex_); 304 Adaptations adaptation_limitations_ RTC_GUARDED_BY(mutex_); 305 306 struct EncoderChangeEvent { 307 std::string previous_encoder_implementation; 308 std::string new_encoder_implementation; 309 }; 310 // Stores the last change in encoder implementation in an optional, so that 311 // the event can be consumed. 312 absl::optional<EncoderChangeEvent> encoder_changed_; 313 314 // Contains stats used for UMA histograms. These stats will be reset if 315 // content type changes between real-time video and screenshare, since these 316 // will be reported separately. 317 struct UmaSamplesContainer { 318 UmaSamplesContainer(const char* prefix, 319 const VideoSendStream::Stats& start_stats, 320 Clock* clock); 321 ~UmaSamplesContainer(); 322 323 void UpdateHistograms(const RtpConfig& rtp_config, 324 const VideoSendStream::Stats& current_stats); 325 326 void InitializeBitrateCounters(const VideoSendStream::Stats& stats); 327 328 bool InsertEncodedFrame(const EncodedImage& encoded_frame, 329 int simulcast_idx); 330 void RemoveOld(int64_t now_ms); 331 332 const std::string uma_prefix_; 333 Clock* const clock_; 334 SampleCounter input_width_counter_; 335 SampleCounter input_height_counter_; 336 SampleCounter sent_width_counter_; 337 SampleCounter sent_height_counter_; 338 SampleCounter encode_time_counter_; 339 BoolSampleCounter key_frame_counter_; 340 BoolSampleCounter quality_limited_frame_counter_; 341 SampleCounter quality_downscales_counter_; 342 BoolSampleCounter cpu_limited_frame_counter_; 343 BoolSampleCounter bw_limited_frame_counter_; 344 SampleCounter bw_resolutions_disabled_counter_; 345 SampleCounter delay_counter_; 346 SampleCounter max_delay_counter_; 347 rtc::RateTracker input_frame_rate_tracker_; 348 RateCounter input_fps_counter_; 349 RateCounter sent_fps_counter_; 350 RateAccCounter total_byte_counter_; 351 RateAccCounter media_byte_counter_; 352 RateAccCounter rtx_byte_counter_; 353 RateAccCounter padding_byte_counter_; 354 RateAccCounter retransmit_byte_counter_; 355 RateAccCounter fec_byte_counter_; 356 int64_t first_rtcp_stats_time_ms_; 357 int64_t first_rtp_stats_time_ms_; 358 StatsTimer cpu_adapt_timer_; 359 StatsTimer quality_adapt_timer_; 360 BoolSampleCounter paused_time_counter_; 361 TargetRateUpdates target_rate_updates_; 362 BoolSampleCounter fallback_active_counter_; 363 FallbackEncoderInfo fallback_info_; 364 FallbackEncoderInfoDisabled fallback_info_disabled_; 365 ReportBlockStats report_block_stats_; 366 const VideoSendStream::Stats start_stats_; 367 size_t num_streams_; // Number of configured streams to encoder. 368 size_t num_pixels_highest_stream_; 369 EncodedFrameMap encoded_frames_; 370 AdaptChanges initial_quality_changes_; 371 372 std::map<int, QpCounters> 373 qp_counters_; // QP counters mapped by spatial idx. 374 }; 375 376 std::unique_ptr<UmaSamplesContainer> uma_container_ RTC_GUARDED_BY(mutex_); 377 }; 378 379 } // namespace webrtc 380 #endif // VIDEO_SEND_STATISTICS_PROXY_H_ 381