1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/utility/quality_scaler.h"
12
13 #include <memory>
14 #include <utility>
15
16 #include "api/video/video_adaptation_reason.h"
17 #include "rtc_base/checks.h"
18 #include "rtc_base/experiments/quality_scaler_settings.h"
19 #include "rtc_base/logging.h"
20 #include "rtc_base/numerics/exp_filter.h"
21 #include "rtc_base/task_queue.h"
22 #include "rtc_base/task_utils/to_queued_task.h"
23 #include "rtc_base/weak_ptr.h"
24
25 // TODO(kthelgason): Some versions of Android have issues with log2.
26 // See https://code.google.com/p/android/issues/detail?id=212634 for details
27 #if defined(WEBRTC_ANDROID)
28 #define log2(x) (log(x) / log(2))
29 #endif
30
31 namespace webrtc {
32
33 namespace {
34 // TODO(nisse): Delete, delegate to encoders.
35 // Threshold constant used until first downscale (to permit fast rampup).
36 static const int kMeasureMs = 2000;
37 static const float kSamplePeriodScaleFactor = 2.5;
38 static const int kFramedropPercentThreshold = 60;
39 static const size_t kMinFramesNeededToScale = 2 * 30;
40
41 } // namespace
42
43 class QualityScaler::QpSmoother {
44 public:
QpSmoother(float alpha)45 explicit QpSmoother(float alpha)
46 : alpha_(alpha),
47 // The initial value of last_sample_ms doesn't matter since the smoother
48 // will ignore the time delta for the first update.
49 last_sample_ms_(0),
50 smoother_(alpha) {}
51
GetAvg() const52 absl::optional<int> GetAvg() const {
53 float value = smoother_.filtered();
54 if (value == rtc::ExpFilter::kValueUndefined) {
55 return absl::nullopt;
56 }
57 return static_cast<int>(value);
58 }
59
Add(float sample,int64_t time_sent_us)60 void Add(float sample, int64_t time_sent_us) {
61 int64_t now_ms = time_sent_us / 1000;
62 smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
63 last_sample_ms_ = now_ms;
64 }
65
Reset()66 void Reset() { smoother_.Reset(alpha_); }
67
68 private:
69 const float alpha_;
70 int64_t last_sample_ms_;
71 rtc::ExpFilter smoother_;
72 };
73
74 // The QualityScaler checks for QP periodically by queuing CheckQpTasks. The
75 // task will either run to completion and trigger a new task being queued, or it
76 // will be destroyed because the QualityScaler is destroyed.
77 //
78 // When high or low QP is reported, the task will be pending until a callback is
79 // invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage
80 // asynchronously and prevents checking for QP until the stream has potentially
81 // been reconfigured.
82 class QualityScaler::CheckQpTask {
83 public:
84 // The result of one CheckQpTask may influence the delay of the next
85 // CheckQpTask.
86 struct Result {
87 bool observed_enough_frames = false;
88 bool qp_usage_reported = false;
89 bool clear_qp_samples = false;
90 };
91
CheckQpTask(QualityScaler * quality_scaler,Result previous_task_result)92 CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result)
93 : quality_scaler_(quality_scaler),
94 state_(State::kNotStarted),
95 previous_task_result_(previous_task_result),
96 weak_ptr_factory_(this) {}
97
StartDelayedTask()98 void StartDelayedTask() {
99 RTC_DCHECK_EQ(state_, State::kNotStarted);
100 state_ = State::kCheckingQp;
101 TaskQueueBase::Current()->PostDelayedTask(
102 ToQueuedTask([this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
103 if (!this_weak_ptr) {
104 // The task has been cancelled through destruction.
105 return;
106 }
107 RTC_DCHECK_EQ(state_, State::kCheckingQp);
108 RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
109 switch (quality_scaler_->CheckQp()) {
110 case QualityScaler::CheckQpResult::kInsufficientSamples: {
111 result_.observed_enough_frames = false;
112 // After this line, |this| may be deleted.
113 DoCompleteTask();
114 return;
115 }
116 case QualityScaler::CheckQpResult::kNormalQp: {
117 result_.observed_enough_frames = true;
118 // After this line, |this| may be deleted.
119 DoCompleteTask();
120 return;
121 }
122 case QualityScaler::CheckQpResult::kHighQp: {
123 result_.observed_enough_frames = true;
124 result_.qp_usage_reported = true;
125 state_ = State::kAwaitingQpUsageHandled;
126 rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
127 callback = ConstructCallback();
128 quality_scaler_->fast_rampup_ = false;
129 // After this line, |this| may be deleted.
130 quality_scaler_->handler_->OnReportQpUsageHigh(callback);
131 return;
132 }
133 case QualityScaler::CheckQpResult::kLowQp: {
134 result_.observed_enough_frames = true;
135 result_.qp_usage_reported = true;
136 state_ = State::kAwaitingQpUsageHandled;
137 rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
138 callback = ConstructCallback();
139 // After this line, |this| may be deleted.
140 quality_scaler_->handler_->OnReportQpUsageLow(callback);
141 return;
142 }
143 }
144 }),
145 GetCheckingQpDelayMs());
146 }
147
OnQpUsageHandled(bool clear_qp_samples)148 void OnQpUsageHandled(bool clear_qp_samples) {
149 RTC_DCHECK_EQ(state_, State::kAwaitingQpUsageHandled);
150 result_.clear_qp_samples = clear_qp_samples;
151 if (clear_qp_samples)
152 quality_scaler_->ClearSamples();
153 DoCompleteTask();
154 }
155
HasCompletedTask() const156 bool HasCompletedTask() const { return state_ == State::kCompleted; }
157
result() const158 Result result() const {
159 RTC_DCHECK(HasCompletedTask());
160 return result_;
161 }
162
163 private:
164 enum class State {
165 kNotStarted,
166 kCheckingQp,
167 kAwaitingQpUsageHandled,
168 kCompleted,
169 };
170
171 // Defined after the definition of QualityScaler::CheckQpTaskHandlerCallback.
172 // Gets around a forward declaration issue.
173 rtc::scoped_refptr<QualityScaler::CheckQpTaskHandlerCallback>
174 ConstructCallback();
175
176 // Determines the sampling period of CheckQpTasks.
GetCheckingQpDelayMs() const177 int64_t GetCheckingQpDelayMs() const {
178 RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
179 if (quality_scaler_->fast_rampup_) {
180 return quality_scaler_->sampling_period_ms_;
181 }
182 if (quality_scaler_->experiment_enabled_ &&
183 !previous_task_result_.observed_enough_frames) {
184 // Use half the interval while waiting for enough frames.
185 return quality_scaler_->sampling_period_ms_ / 2;
186 }
187 if (!previous_task_result_.clear_qp_samples) {
188 // Check shortly again.
189 return quality_scaler_->sampling_period_ms_ / 8;
190 }
191 if (quality_scaler_->scale_factor_ &&
192 !previous_task_result_.qp_usage_reported) {
193 // Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
194 return quality_scaler_->sampling_period_ms_ *
195 quality_scaler_->scale_factor_.value();
196 }
197 return quality_scaler_->sampling_period_ms_ *
198 quality_scaler_->initial_scale_factor_;
199 }
200
DoCompleteTask()201 void DoCompleteTask() {
202 RTC_DCHECK(state_ == State::kCheckingQp ||
203 state_ == State::kAwaitingQpUsageHandled);
204 state_ = State::kCompleted;
205 // Starting the next task deletes the pending task. After this line, |this|
206 // has been deleted.
207 quality_scaler_->StartNextCheckQpTask();
208 }
209
210 QualityScaler* const quality_scaler_;
211 State state_;
212 const Result previous_task_result_;
213 Result result_;
214
215 rtc::WeakPtrFactory<CheckQpTask> weak_ptr_factory_;
216 };
217
218 class QualityScaler::CheckQpTaskHandlerCallback
219 : public QualityScalerQpUsageHandlerCallbackInterface {
220 public:
CheckQpTaskHandlerCallback(rtc::WeakPtr<QualityScaler::CheckQpTask> check_qp_task)221 CheckQpTaskHandlerCallback(
222 rtc::WeakPtr<QualityScaler::CheckQpTask> check_qp_task)
223 : QualityScalerQpUsageHandlerCallbackInterface(),
224 check_qp_task_(std::move(check_qp_task)),
225 was_handled_(false) {}
226
~CheckQpTaskHandlerCallback()227 ~CheckQpTaskHandlerCallback() { RTC_DCHECK(was_handled_); }
228
OnQpUsageHandled(bool clear_qp_samples)229 void OnQpUsageHandled(bool clear_qp_samples) {
230 RTC_DCHECK(!was_handled_);
231 was_handled_ = true;
232 if (!check_qp_task_) {
233 // The task has been cancelled through destruction; the result of the
234 // operation is ignored.
235 return;
236 }
237 check_qp_task_->OnQpUsageHandled(clear_qp_samples);
238 }
239
240 private:
241 // The callback may outlive the QualityScaler and its task.
242 rtc::WeakPtr<QualityScaler::CheckQpTask> const check_qp_task_;
243 bool was_handled_;
244 };
245
246 rtc::scoped_refptr<QualityScaler::CheckQpTaskHandlerCallback>
ConstructCallback()247 QualityScaler::CheckQpTask::ConstructCallback() {
248 return new CheckQpTaskHandlerCallback(weak_ptr_factory_.GetWeakPtr());
249 }
250
QualityScaler(QualityScalerQpUsageHandlerInterface * handler,VideoEncoder::QpThresholds thresholds)251 QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
252 VideoEncoder::QpThresholds thresholds)
253 : QualityScaler(handler, thresholds, kMeasureMs) {}
254
255 // Protected ctor, should not be called directly.
QualityScaler(QualityScalerQpUsageHandlerInterface * handler,VideoEncoder::QpThresholds thresholds,int64_t sampling_period_ms)256 QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
257 VideoEncoder::QpThresholds thresholds,
258 int64_t sampling_period_ms)
259 : handler_(handler),
260 thresholds_(thresholds),
261 sampling_period_ms_(sampling_period_ms),
262 fast_rampup_(true),
263 // Arbitrarily choose size based on 30 fps for 5 seconds.
264 average_qp_(5 * 30),
265 framedrop_percent_media_opt_(5 * 30),
266 framedrop_percent_all_(5 * 30),
267 experiment_enabled_(QualityScalingExperiment::Enabled()),
268 min_frames_needed_(
269 QualityScalerSettings::ParseFromFieldTrials().MinFrames().value_or(
270 kMinFramesNeededToScale)),
271 initial_scale_factor_(QualityScalerSettings::ParseFromFieldTrials()
272 .InitialScaleFactor()
273 .value_or(kSamplePeriodScaleFactor)),
274 scale_factor_(
275 QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()) {
276 RTC_DCHECK_RUN_ON(&task_checker_);
277 if (experiment_enabled_) {
278 config_ = QualityScalingExperiment::GetConfig();
279 qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
280 qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
281 }
282 RTC_DCHECK(handler_ != nullptr);
283 StartNextCheckQpTask();
284 RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
285 << ", high: " << thresholds_.high;
286 }
287
~QualityScaler()288 QualityScaler::~QualityScaler() {
289 RTC_DCHECK_RUN_ON(&task_checker_);
290 }
291
StartNextCheckQpTask()292 void QualityScaler::StartNextCheckQpTask() {
293 RTC_DCHECK_RUN_ON(&task_checker_);
294 RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask())
295 << "A previous CheckQpTask has not completed yet!";
296 CheckQpTask::Result previous_task_result;
297 if (pending_qp_task_) {
298 previous_task_result = pending_qp_task_->result();
299 }
300 pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
301 pending_qp_task_->StartDelayedTask();
302 }
303
SetQpThresholds(VideoEncoder::QpThresholds thresholds)304 void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
305 RTC_DCHECK_RUN_ON(&task_checker_);
306 thresholds_ = thresholds;
307 }
308
ReportDroppedFrameByMediaOpt()309 void QualityScaler::ReportDroppedFrameByMediaOpt() {
310 RTC_DCHECK_RUN_ON(&task_checker_);
311 framedrop_percent_media_opt_.AddSample(100);
312 framedrop_percent_all_.AddSample(100);
313 }
314
ReportDroppedFrameByEncoder()315 void QualityScaler::ReportDroppedFrameByEncoder() {
316 RTC_DCHECK_RUN_ON(&task_checker_);
317 framedrop_percent_all_.AddSample(100);
318 }
319
ReportQp(int qp,int64_t time_sent_us)320 void QualityScaler::ReportQp(int qp, int64_t time_sent_us) {
321 RTC_DCHECK_RUN_ON(&task_checker_);
322 framedrop_percent_media_opt_.AddSample(0);
323 framedrop_percent_all_.AddSample(0);
324 average_qp_.AddSample(qp);
325 if (qp_smoother_high_)
326 qp_smoother_high_->Add(qp, time_sent_us);
327 if (qp_smoother_low_)
328 qp_smoother_low_->Add(qp, time_sent_us);
329 }
330
QpFastFilterLow() const331 bool QualityScaler::QpFastFilterLow() const {
332 RTC_DCHECK_RUN_ON(&task_checker_);
333 size_t num_frames = config_.use_all_drop_reasons
334 ? framedrop_percent_all_.Size()
335 : framedrop_percent_media_opt_.Size();
336 const size_t kMinNumFrames = 10;
337 if (num_frames < kMinNumFrames) {
338 return false; // Wait for more frames before making a decision.
339 }
340 absl::optional<int> avg_qp_high = qp_smoother_high_
341 ? qp_smoother_high_->GetAvg()
342 : average_qp_.GetAverageRoundedDown();
343 return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false;
344 }
345
CheckQp() const346 QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
347 RTC_DCHECK_RUN_ON(&task_checker_);
348 // Should be set through InitEncode -> Should be set by now.
349 RTC_DCHECK_GE(thresholds_.low, 0);
350
351 // If we have not observed at least this many frames we can't make a good
352 // scaling decision.
353 const size_t frames = config_.use_all_drop_reasons
354 ? framedrop_percent_all_.Size()
355 : framedrop_percent_media_opt_.Size();
356 if (frames < min_frames_needed_) {
357 return CheckQpResult::kInsufficientSamples;
358 }
359
360 // Check if we should scale down due to high frame drop.
361 const absl::optional<int> drop_rate =
362 config_.use_all_drop_reasons
363 ? framedrop_percent_all_.GetAverageRoundedDown()
364 : framedrop_percent_media_opt_.GetAverageRoundedDown();
365 if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
366 RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
367 return CheckQpResult::kHighQp;
368 }
369
370 // Check if we should scale up or down based on QP.
371 const absl::optional<int> avg_qp_high =
372 qp_smoother_high_ ? qp_smoother_high_->GetAvg()
373 : average_qp_.GetAverageRoundedDown();
374 const absl::optional<int> avg_qp_low =
375 qp_smoother_low_ ? qp_smoother_low_->GetAvg()
376 : average_qp_.GetAverageRoundedDown();
377 if (avg_qp_high && avg_qp_low) {
378 RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
379 << *avg_qp_low << ").";
380 if (*avg_qp_high > thresholds_.high) {
381 return CheckQpResult::kHighQp;
382 }
383 if (*avg_qp_low <= thresholds_.low) {
384 // QP has been low. We want to try a higher resolution.
385 return CheckQpResult::kLowQp;
386 }
387 }
388 return CheckQpResult::kNormalQp;
389 }
390
ClearSamples()391 void QualityScaler::ClearSamples() {
392 RTC_DCHECK_RUN_ON(&task_checker_);
393 framedrop_percent_media_opt_.Reset();
394 framedrop_percent_all_.Reset();
395 average_qp_.Reset();
396 if (qp_smoother_high_)
397 qp_smoother_high_->Reset();
398 if (qp_smoother_low_)
399 qp_smoother_low_->Reset();
400 }
401
~QualityScalerQpUsageHandlerInterface()402 QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {}
403
404 QualityScalerQpUsageHandlerCallbackInterface::
QualityScalerQpUsageHandlerCallbackInterface()405 QualityScalerQpUsageHandlerCallbackInterface() {}
406
407 QualityScalerQpUsageHandlerCallbackInterface::
~QualityScalerQpUsageHandlerCallbackInterface()408 ~QualityScalerQpUsageHandlerCallbackInterface() {}
409
410 } // namespace webrtc
411