1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/video_quality_observer.h"
12
13 #include <algorithm>
14 #include <cmath>
15 #include <cstdint>
16 #include <string>
17
18 #include "rtc_base/logging.h"
19 #include "rtc_base/strings/string_builder.h"
20 #include "system_wrappers/include/metrics.h"
21
22 namespace webrtc {
23 const uint32_t VideoQualityObserver::kMinFrameSamplesToDetectFreeze = 5;
24 const uint32_t VideoQualityObserver::kMinIncreaseForFreezeMs = 150;
25 const uint32_t VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames = 30;
26
27 namespace {
28 constexpr int kMinVideoDurationMs = 3000;
29 constexpr int kMinRequiredSamples = 1;
30 constexpr int kPixelsInHighResolution =
31 960 * 540; // CPU-adapted HD still counts.
32 constexpr int kPixelsInMediumResolution = 640 * 360;
33 constexpr int kBlockyQpThresholdVp8 = 70;
34 constexpr int kBlockyQpThresholdVp9 = 180;
35 constexpr int kMaxNumCachedBlockyFrames = 100;
36 // TODO(ilnik): Add H264/HEVC thresholds.
37 } // namespace
38
VideoQualityObserver(VideoContentType content_type)39 VideoQualityObserver::VideoQualityObserver(VideoContentType content_type)
40 : last_frame_rendered_ms_(-1),
41 num_frames_rendered_(0),
42 first_frame_rendered_ms_(-1),
43 last_frame_pixels_(0),
44 is_last_frame_blocky_(false),
45 last_unfreeze_time_ms_(0),
46 render_interframe_delays_(kAvgInterframeDelaysWindowSizeFrames),
47 sum_squared_interframe_delays_secs_(0.0),
48 time_in_resolution_ms_(3, 0),
49 current_resolution_(Resolution::Low),
50 num_resolution_downgrades_(0),
51 time_in_blocky_video_ms_(0),
52 content_type_(content_type),
53 is_paused_(false) {}
54
UpdateHistograms()55 void VideoQualityObserver::UpdateHistograms() {
56 // Don't report anything on an empty video stream.
57 if (num_frames_rendered_ == 0) {
58 return;
59 }
60
61 char log_stream_buf[2 * 1024];
62 rtc::SimpleStringBuilder log_stream(log_stream_buf);
63
64 if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
65 smooth_playback_durations_.Add(last_frame_rendered_ms_ -
66 last_unfreeze_time_ms_);
67 }
68
69 std::string uma_prefix = videocontenttypehelpers::IsScreenshare(content_type_)
70 ? "WebRTC.Video.Screenshare"
71 : "WebRTC.Video";
72
73 auto mean_time_between_freezes =
74 smooth_playback_durations_.Avg(kMinRequiredSamples);
75 if (mean_time_between_freezes) {
76 RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs",
77 *mean_time_between_freezes);
78 log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs "
79 << *mean_time_between_freezes << "\n";
80 }
81 auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples);
82 if (avg_freeze_length) {
83 RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs",
84 *avg_freeze_length);
85 log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length
86 << "\n";
87 }
88
89 int64_t video_duration_ms =
90 last_frame_rendered_ms_ - first_frame_rendered_ms_;
91
92 if (video_duration_ms >= kMinVideoDurationMs) {
93 int time_spent_in_hd_percentage = static_cast<int>(
94 time_in_resolution_ms_[Resolution::High] * 100 / video_duration_ms);
95 RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage",
96 time_spent_in_hd_percentage);
97 log_stream << uma_prefix << ".TimeInHdPercentage "
98 << time_spent_in_hd_percentage << "\n";
99
100 int time_with_blocky_video_percentage =
101 static_cast<int>(time_in_blocky_video_ms_ * 100 / video_duration_ms);
102 RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage",
103 time_with_blocky_video_percentage);
104 log_stream << uma_prefix << ".TimeInBlockyVideoPercentage "
105 << time_with_blocky_video_percentage << "\n";
106
107 int num_resolution_downgrades_per_minute =
108 num_resolution_downgrades_ * 60000 / video_duration_ms;
109 RTC_HISTOGRAM_COUNTS_SPARSE_100(
110 uma_prefix + ".NumberResolutionDownswitchesPerMinute",
111 num_resolution_downgrades_per_minute);
112 log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute "
113 << num_resolution_downgrades_per_minute << "\n";
114
115 int num_freezes_per_minute =
116 freezes_durations_.NumSamples() * 60000 / video_duration_ms;
117 RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".NumberFreezesPerMinute",
118 num_freezes_per_minute);
119 log_stream << uma_prefix << ".NumberFreezesPerMinute "
120 << num_freezes_per_minute << "\n";
121
122 if (sum_squared_interframe_delays_secs_ > 0.0) {
123 int harmonic_framerate_fps = std::round(
124 video_duration_ms / (1000 * sum_squared_interframe_delays_secs_));
125 RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".HarmonicFrameRate",
126 harmonic_framerate_fps);
127 log_stream << uma_prefix << ".HarmonicFrameRate "
128 << harmonic_framerate_fps << "\n";
129 }
130 }
131 RTC_LOG(LS_INFO) << log_stream.str();
132 }
133
OnRenderedFrame(const VideoFrame & frame,int64_t now_ms)134 void VideoQualityObserver::OnRenderedFrame(const VideoFrame& frame,
135 int64_t now_ms) {
136 RTC_DCHECK_LE(last_frame_rendered_ms_, now_ms);
137 RTC_DCHECK_LE(last_unfreeze_time_ms_, now_ms);
138
139 if (num_frames_rendered_ == 0) {
140 first_frame_rendered_ms_ = last_unfreeze_time_ms_ = now_ms;
141 }
142
143 auto blocky_frame_it = blocky_frames_.find(frame.timestamp());
144
145 if (num_frames_rendered_ > 0) {
146 // Process inter-frame delay.
147 const int64_t interframe_delay_ms = now_ms - last_frame_rendered_ms_;
148 const double interframe_delays_secs = interframe_delay_ms / 1000.0;
149
150 // Sum of squared inter frame intervals is used to calculate the harmonic
151 // frame rate metric. The metric aims to reflect overall experience related
152 // to smoothness of video playback and includes both freezes and pauses.
153 sum_squared_interframe_delays_secs_ +=
154 interframe_delays_secs * interframe_delays_secs;
155
156 if (!is_paused_) {
157 render_interframe_delays_.AddSample(interframe_delay_ms);
158
159 bool was_freeze = false;
160 if (render_interframe_delays_.Size() >= kMinFrameSamplesToDetectFreeze) {
161 const absl::optional<int64_t> avg_interframe_delay =
162 render_interframe_delays_.GetAverageRoundedDown();
163 RTC_DCHECK(avg_interframe_delay);
164 was_freeze = interframe_delay_ms >=
165 std::max(3 * *avg_interframe_delay,
166 *avg_interframe_delay + kMinIncreaseForFreezeMs);
167 }
168
169 if (was_freeze) {
170 freezes_durations_.Add(interframe_delay_ms);
171 smooth_playback_durations_.Add(last_frame_rendered_ms_ -
172 last_unfreeze_time_ms_);
173 last_unfreeze_time_ms_ = now_ms;
174 } else {
175 // Count spatial metrics if there were no freeze.
176 time_in_resolution_ms_[current_resolution_] += interframe_delay_ms;
177
178 if (is_last_frame_blocky_) {
179 time_in_blocky_video_ms_ += interframe_delay_ms;
180 }
181 }
182 }
183 }
184
185 if (is_paused_) {
186 // If the stream was paused since the previous frame, do not count the
187 // pause toward smooth playback. Explicitly count the part before it and
188 // start the new smooth playback interval from this frame.
189 is_paused_ = false;
190 if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
191 smooth_playback_durations_.Add(last_frame_rendered_ms_ -
192 last_unfreeze_time_ms_);
193 }
194 last_unfreeze_time_ms_ = now_ms;
195
196 if (num_frames_rendered_ > 0) {
197 pauses_durations_.Add(now_ms - last_frame_rendered_ms_);
198 }
199 }
200
201 int64_t pixels = frame.width() * frame.height();
202 if (pixels >= kPixelsInHighResolution) {
203 current_resolution_ = Resolution::High;
204 } else if (pixels >= kPixelsInMediumResolution) {
205 current_resolution_ = Resolution::Medium;
206 } else {
207 current_resolution_ = Resolution::Low;
208 }
209
210 if (pixels < last_frame_pixels_) {
211 ++num_resolution_downgrades_;
212 }
213
214 last_frame_pixels_ = pixels;
215 last_frame_rendered_ms_ = now_ms;
216
217 is_last_frame_blocky_ = blocky_frame_it != blocky_frames_.end();
218 if (is_last_frame_blocky_) {
219 blocky_frames_.erase(blocky_frames_.begin(), ++blocky_frame_it);
220 }
221
222 ++num_frames_rendered_;
223 }
224
OnDecodedFrame(const VideoFrame & frame,absl::optional<uint8_t> qp,VideoCodecType codec)225 void VideoQualityObserver::OnDecodedFrame(const VideoFrame& frame,
226 absl::optional<uint8_t> qp,
227 VideoCodecType codec) {
228 if (qp) {
229 absl::optional<int> qp_blocky_threshold;
230 // TODO(ilnik): add other codec types when we have QP for them.
231 switch (codec) {
232 case kVideoCodecVP8:
233 qp_blocky_threshold = kBlockyQpThresholdVp8;
234 break;
235 case kVideoCodecVP9:
236 qp_blocky_threshold = kBlockyQpThresholdVp9;
237 break;
238 default:
239 qp_blocky_threshold = absl::nullopt;
240 }
241
242 RTC_DCHECK(blocky_frames_.find(frame.timestamp()) == blocky_frames_.end());
243
244 if (qp_blocky_threshold && *qp > *qp_blocky_threshold) {
245 // Cache blocky frame. Its duration will be calculated in render callback.
246 if (blocky_frames_.size() > kMaxNumCachedBlockyFrames) {
247 RTC_LOG(LS_WARNING) << "Overflow of blocky frames cache.";
248 blocky_frames_.erase(
249 blocky_frames_.begin(),
250 std::next(blocky_frames_.begin(), kMaxNumCachedBlockyFrames / 2));
251 }
252
253 blocky_frames_.insert(frame.timestamp());
254 }
255 }
256 }
257
OnStreamInactive()258 void VideoQualityObserver::OnStreamInactive() {
259 is_paused_ = true;
260 }
261
NumFreezes() const262 uint32_t VideoQualityObserver::NumFreezes() const {
263 return freezes_durations_.NumSamples();
264 }
265
NumPauses() const266 uint32_t VideoQualityObserver::NumPauses() const {
267 return pauses_durations_.NumSamples();
268 }
269
TotalFreezesDurationMs() const270 uint32_t VideoQualityObserver::TotalFreezesDurationMs() const {
271 return freezes_durations_.Sum(kMinRequiredSamples).value_or(0);
272 }
273
TotalPausesDurationMs() const274 uint32_t VideoQualityObserver::TotalPausesDurationMs() const {
275 return pauses_durations_.Sum(kMinRequiredSamples).value_or(0);
276 }
277
TotalFramesDurationMs() const278 uint32_t VideoQualityObserver::TotalFramesDurationMs() const {
279 return last_frame_rendered_ms_ - first_frame_rendered_ms_;
280 }
281
SumSquaredFrameDurationsSec() const282 double VideoQualityObserver::SumSquaredFrameDurationsSec() const {
283 return sum_squared_interframe_delays_secs_;
284 }
285
286 } // namespace webrtc
287