1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_RECEIVE_STATISTICS_PROXY2_H_ 12 #define VIDEO_RECEIVE_STATISTICS_PROXY2_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "absl/types/optional.h" 20 #include "api/sequence_checker.h" 21 #include "api/task_queue/pending_task_safety_flag.h" 22 #include "api/task_queue/task_queue_base.h" 23 #include "api/units/timestamp.h" 24 #include "api/video_codecs/video_decoder.h" 25 #include "call/video_receive_stream.h" 26 #include "modules/include/module_common_types.h" 27 #include "modules/video_coding/include/video_coding_defines.h" 28 #include "rtc_base/numerics/histogram_percentile_counter.h" 29 #include "rtc_base/numerics/moving_max_counter.h" 30 #include "rtc_base/numerics/sample_counter.h" 31 #include "rtc_base/rate_statistics.h" 32 #include "rtc_base/rate_tracker.h" 33 #include "rtc_base/system/no_unique_address.h" 34 #include "rtc_base/thread_annotations.h" 35 #include "video/quality_threshold.h" 36 #include "video/stats_counter.h" 37 #include "video/video_quality_observer2.h" 38 39 namespace webrtc { 40 41 class Clock; 42 struct CodecSpecificInfo; 43 44 namespace internal { 45 // Declared in video_receive_stream2.h. 46 struct VideoFrameMetaData; 47 48 class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, 49 public RtcpCnameCallback, 50 public RtcpPacketTypeCounterObserver { 51 public: 52 ReceiveStatisticsProxy(uint32_t remote_ssrc, 53 Clock* clock, 54 TaskQueueBase* worker_thread); 55 ~ReceiveStatisticsProxy() override; 56 57 VideoReceiveStreamInterface::Stats GetStats() const; 58 59 void OnDecodedFrame(const VideoFrame& frame, 60 absl::optional<uint8_t> qp, 61 TimeDelta decode_time, 62 VideoContentType content_type); 63 64 // Called asyncronously on the worker thread as a result of a call to the 65 // above OnDecodedFrame method, which is called back on the thread where 66 // the actual decoding happens. 67 void OnDecodedFrame(const VideoFrameMetaData& frame_meta, 68 absl::optional<uint8_t> qp, 69 TimeDelta decode_time, 70 TimeDelta processing_delay, 71 TimeDelta assembly_time, 72 VideoContentType content_type); 73 74 void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms, 75 int64_t sync_offset_ms, 76 double estimated_freq_khz); 77 void OnRenderedFrame(const VideoFrameMetaData& frame_meta); 78 void OnIncomingPayloadType(int payload_type); 79 void OnDecoderInfo(const VideoDecoder::DecoderInfo& decoder_info); 80 81 void OnPreDecode(VideoCodecType codec_type, int qp); 82 83 void OnUniqueFramesCounted(int num_unique_frames); 84 85 // Indicates video stream has been paused (no incoming packets). 86 void OnStreamInactive(); 87 88 // Overrides VCMReceiveStatisticsCallback. 89 void OnCompleteFrame(bool is_keyframe, 90 size_t size_bytes, 91 VideoContentType content_type) override; 92 void OnDroppedFrames(uint32_t frames_dropped) override; 93 void OnFrameBufferTimingsUpdated(int max_decode_ms, 94 int current_delay_ms, 95 int target_delay_ms, 96 int jitter_buffer_ms, 97 int min_playout_delay_ms, 98 int render_delay_ms) override; 99 100 void OnTimingFrameInfoUpdated(const TimingFrameInfo& info) override; 101 102 // Overrides RtcpCnameCallback. 103 void OnCname(uint32_t ssrc, absl::string_view cname) override; 104 105 // Overrides RtcpPacketTypeCounterObserver. 106 void RtcpPacketTypesCounterUpdated( 107 uint32_t ssrc, 108 const RtcpPacketTypeCounter& packet_counter) override; 109 110 void OnRttUpdate(int64_t avg_rtt_ms); 111 112 // Notification methods that are used to check our internal state and validate 113 // threading assumptions. These are called by VideoReceiveStreamInterface. 114 void DecoderThreadStarting(); 115 void DecoderThreadStopped(); 116 117 // Produce histograms. Must be called after DecoderThreadStopped(), typically 118 // at the end of the call. 119 void UpdateHistograms(absl::optional<int> fraction_lost, 120 const StreamDataCounters& rtp_stats, 121 const StreamDataCounters* rtx_stats); 122 123 private: 124 struct QpCounters { 125 rtc::SampleCounter vp8; 126 }; 127 128 struct ContentSpecificStats { 129 ContentSpecificStats(); 130 ~ContentSpecificStats(); 131 132 void Add(const ContentSpecificStats& other); 133 134 rtc::SampleCounter e2e_delay_counter; 135 rtc::SampleCounter interframe_delay_counter; 136 int64_t flow_duration_ms = 0; 137 int64_t total_media_bytes = 0; 138 rtc::SampleCounter received_width; 139 rtc::SampleCounter received_height; 140 rtc::SampleCounter qp_counter; 141 FrameCounts frame_counts; 142 rtc::HistogramPercentileCounter interframe_delay_percentiles; 143 }; 144 145 void QualitySample(Timestamp now); 146 147 // Removes info about old frames and then updates the framerate. 148 void UpdateFramerate(int64_t now_ms) const; 149 150 absl::optional<int64_t> GetCurrentEstimatedPlayoutNtpTimestampMs( 151 int64_t now_ms) const; 152 153 Clock* const clock_; 154 const int64_t start_ms_; 155 156 int64_t last_sample_time_ RTC_GUARDED_BY(main_thread_); 157 158 QualityThreshold fps_threshold_ RTC_GUARDED_BY(main_thread_); 159 QualityThreshold qp_threshold_ RTC_GUARDED_BY(main_thread_); 160 QualityThreshold variance_threshold_ RTC_GUARDED_BY(main_thread_); 161 rtc::SampleCounter qp_sample_ RTC_GUARDED_BY(main_thread_); 162 int num_bad_states_ RTC_GUARDED_BY(main_thread_); 163 int num_certain_states_ RTC_GUARDED_BY(main_thread_); 164 // Note: The `stats_.rtp_stats` member is not used or populated by this class. 165 mutable VideoReceiveStreamInterface::Stats stats_ 166 RTC_GUARDED_BY(main_thread_); 167 // Same as stats_.ssrc, but const (no lock required). 168 const uint32_t remote_ssrc_; 169 RateStatistics decode_fps_estimator_ RTC_GUARDED_BY(main_thread_); 170 RateStatistics renders_fps_estimator_ RTC_GUARDED_BY(main_thread_); 171 rtc::RateTracker render_fps_tracker_ RTC_GUARDED_BY(main_thread_); 172 rtc::RateTracker render_pixel_tracker_ RTC_GUARDED_BY(main_thread_); 173 rtc::SampleCounter sync_offset_counter_ RTC_GUARDED_BY(main_thread_); 174 rtc::SampleCounter decode_time_counter_ RTC_GUARDED_BY(main_thread_); 175 rtc::SampleCounter jitter_buffer_delay_counter_ RTC_GUARDED_BY(main_thread_); 176 rtc::SampleCounter target_delay_counter_ RTC_GUARDED_BY(main_thread_); 177 rtc::SampleCounter current_delay_counter_ RTC_GUARDED_BY(main_thread_); 178 rtc::SampleCounter delay_counter_ RTC_GUARDED_BY(main_thread_); 179 std::unique_ptr<VideoQualityObserver> video_quality_observer_ 180 RTC_GUARDED_BY(main_thread_); 181 mutable rtc::MovingMaxCounter<int> interframe_delay_max_moving_ 182 RTC_GUARDED_BY(main_thread_); 183 std::map<VideoContentType, ContentSpecificStats> content_specific_stats_ 184 RTC_GUARDED_BY(main_thread_); 185 MaxCounter freq_offset_counter_ RTC_GUARDED_BY(main_thread_); 186 QpCounters qp_counters_ RTC_GUARDED_BY(main_thread_); 187 int64_t avg_rtt_ms_ RTC_GUARDED_BY(main_thread_) = 0; 188 mutable std::map<int64_t, size_t> frame_window_ RTC_GUARDED_BY(main_thread_); 189 VideoContentType last_content_type_ RTC_GUARDED_BY(&main_thread_); 190 VideoCodecType last_codec_type_ RTC_GUARDED_BY(main_thread_); 191 absl::optional<int64_t> first_frame_received_time_ms_ 192 RTC_GUARDED_BY(main_thread_); 193 absl::optional<int64_t> first_decoded_frame_time_ms_ 194 RTC_GUARDED_BY(main_thread_); 195 absl::optional<int64_t> last_decoded_frame_time_ms_ 196 RTC_GUARDED_BY(main_thread_); 197 size_t num_delayed_frames_rendered_ RTC_GUARDED_BY(main_thread_); 198 int64_t sum_missed_render_deadline_ms_ RTC_GUARDED_BY(main_thread_); 199 // Mutable because calling Max() on MovingMaxCounter is not const. Yet it is 200 // called from const GetStats(). 201 mutable rtc::MovingMaxCounter<TimingFrameInfo> timing_frame_info_counter_ 202 RTC_GUARDED_BY(main_thread_); 203 absl::optional<int> num_unique_frames_ RTC_GUARDED_BY(main_thread_); 204 absl::optional<int64_t> last_estimated_playout_ntp_timestamp_ms_ 205 RTC_GUARDED_BY(main_thread_); 206 absl::optional<int64_t> last_estimated_playout_time_ms_ 207 RTC_GUARDED_BY(main_thread_); 208 209 // The thread on which this instance is constructed and some of its main 210 // methods are invoked on such as GetStats(). 211 TaskQueueBase* const worker_thread_; 212 213 ScopedTaskSafety task_safety_; 214 215 RTC_NO_UNIQUE_ADDRESS SequenceChecker decode_queue_; 216 SequenceChecker main_thread_; 217 RTC_NO_UNIQUE_ADDRESS SequenceChecker incoming_render_queue_; 218 }; 219 220 } // namespace internal 221 } // namespace webrtc 222 #endif // VIDEO_RECEIVE_STATISTICS_PROXY2_H_ 223