1 /*
2 * Copyright 2020 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/adaptation/quality_scaler_resource.h"
12
13 #include <utility>
14
15 #include "rtc_base/experiments/balanced_degradation_settings.h"
16 #include "rtc_base/ref_counted_object.h"
17 #include "rtc_base/task_utils/to_queued_task.h"
18 #include "rtc_base/time_utils.h"
19
20 namespace webrtc {
21
22 namespace {
23
24 const int64_t kUnderuseDueToDisabledCooldownMs = 1000;
25
26 } // namespace
27
28 // static
Create(DegradationPreferenceProvider * degradation_preference_provider)29 rtc::scoped_refptr<QualityScalerResource> QualityScalerResource::Create(
30 DegradationPreferenceProvider* degradation_preference_provider) {
31 return new rtc::RefCountedObject<QualityScalerResource>(
32 degradation_preference_provider);
33 }
34
QualityScalerResource(DegradationPreferenceProvider * degradation_preference_provider)35 QualityScalerResource::QualityScalerResource(
36 DegradationPreferenceProvider* degradation_preference_provider)
37 : VideoStreamEncoderResource("QualityScalerResource"),
38 quality_scaler_(nullptr),
39 last_underuse_due_to_disabled_timestamp_ms_(absl::nullopt),
40 num_handled_callbacks_(0),
41 pending_callbacks_(),
42 degradation_preference_provider_(degradation_preference_provider),
43 clear_qp_samples_(false) {
44 RTC_CHECK(degradation_preference_provider_);
45 }
46
~QualityScalerResource()47 QualityScalerResource::~QualityScalerResource() {
48 RTC_DCHECK(!quality_scaler_);
49 RTC_DCHECK(pending_callbacks_.empty());
50 }
51
is_started() const52 bool QualityScalerResource::is_started() const {
53 RTC_DCHECK_RUN_ON(encoder_queue());
54 return quality_scaler_.get();
55 }
56
StartCheckForOveruse(VideoEncoder::QpThresholds qp_thresholds)57 void QualityScalerResource::StartCheckForOveruse(
58 VideoEncoder::QpThresholds qp_thresholds) {
59 RTC_DCHECK_RUN_ON(encoder_queue());
60 RTC_DCHECK(!is_started());
61 quality_scaler_ =
62 std::make_unique<QualityScaler>(this, std::move(qp_thresholds));
63 }
64
StopCheckForOveruse()65 void QualityScalerResource::StopCheckForOveruse() {
66 RTC_DCHECK_RUN_ON(encoder_queue());
67 // Ensure we have no pending callbacks. This makes it safe to destroy the
68 // QualityScaler and even task queues with tasks in-flight.
69 AbortPendingCallbacks();
70 quality_scaler_.reset();
71 }
72
SetQpThresholds(VideoEncoder::QpThresholds qp_thresholds)73 void QualityScalerResource::SetQpThresholds(
74 VideoEncoder::QpThresholds qp_thresholds) {
75 RTC_DCHECK_RUN_ON(encoder_queue());
76 RTC_DCHECK(is_started());
77 quality_scaler_->SetQpThresholds(std::move(qp_thresholds));
78 }
79
QpFastFilterLow()80 bool QualityScalerResource::QpFastFilterLow() {
81 RTC_DCHECK_RUN_ON(encoder_queue());
82 RTC_DCHECK(is_started());
83 return quality_scaler_->QpFastFilterLow();
84 }
85
OnEncodeCompleted(const EncodedImage & encoded_image,int64_t time_sent_in_us)86 void QualityScalerResource::OnEncodeCompleted(const EncodedImage& encoded_image,
87 int64_t time_sent_in_us) {
88 RTC_DCHECK_RUN_ON(encoder_queue());
89 if (quality_scaler_ && encoded_image.qp_ >= 0) {
90 quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us);
91 } else if (!quality_scaler_) {
92 // Reference counting guarantees that this object is still alive by the time
93 // the task is executed.
94 // TODO(webrtc:11553): this is a workaround to ensure that all quality
95 // scaler imposed limitations are removed once qualty scaler is disabled
96 // mid call.
97 // Instead it should be done at a higher layer in the same way for all
98 // resources.
99 int64_t timestamp_ms = rtc::TimeMillis();
100 if (!last_underuse_due_to_disabled_timestamp_ms_.has_value() ||
101 timestamp_ms - last_underuse_due_to_disabled_timestamp_ms_.value() >=
102 kUnderuseDueToDisabledCooldownMs) {
103 last_underuse_due_to_disabled_timestamp_ms_ = timestamp_ms;
104 MaybePostTaskToResourceAdaptationQueue(
105 [this_ref = rtc::scoped_refptr<QualityScalerResource>(this)] {
106 RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue());
107 this_ref->OnResourceUsageStateMeasured(
108 ResourceUsageState::kUnderuse);
109 });
110 }
111 }
112 }
113
OnFrameDropped(EncodedImageCallback::DropReason reason)114 void QualityScalerResource::OnFrameDropped(
115 EncodedImageCallback::DropReason reason) {
116 RTC_DCHECK_RUN_ON(encoder_queue());
117 if (!quality_scaler_)
118 return;
119 switch (reason) {
120 case EncodedImageCallback::DropReason::kDroppedByMediaOptimizations:
121 quality_scaler_->ReportDroppedFrameByMediaOpt();
122 break;
123 case EncodedImageCallback::DropReason::kDroppedByEncoder:
124 quality_scaler_->ReportDroppedFrameByEncoder();
125 break;
126 }
127 }
128
OnReportQpUsageHigh(rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback)129 void QualityScalerResource::OnReportQpUsageHigh(
130 rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback) {
131 RTC_DCHECK_RUN_ON(encoder_queue());
132 size_t callback_id = QueuePendingCallback(callback);
133 // Reference counting guarantees that this object is still alive by the time
134 // the task is executed.
135 MaybePostTaskToResourceAdaptationQueue(
136 [this_ref = rtc::scoped_refptr<QualityScalerResource>(this),
137 callback_id] {
138 RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue());
139 this_ref->clear_qp_samples_ = false;
140 // If this OnResourceUsageStateMeasured() triggers an adaptation,
141 // OnAdaptationApplied() will occur between this line and the next. This
142 // allows modifying |clear_qp_samples_| based on the adaptation.
143 this_ref->OnResourceUsageStateMeasured(ResourceUsageState::kOveruse);
144 this_ref->HandlePendingCallback(callback_id,
145 this_ref->clear_qp_samples_);
146 });
147 }
148
OnReportQpUsageLow(rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback)149 void QualityScalerResource::OnReportQpUsageLow(
150 rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback) {
151 RTC_DCHECK_RUN_ON(encoder_queue());
152 size_t callback_id = QueuePendingCallback(callback);
153 // Reference counting guarantees that this object is still alive by the time
154 // the task is executed.
155 MaybePostTaskToResourceAdaptationQueue(
156 [this_ref = rtc::scoped_refptr<QualityScalerResource>(this),
157 callback_id] {
158 RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue());
159 this_ref->OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse);
160 this_ref->HandlePendingCallback(callback_id, true);
161 });
162 }
163
OnAdaptationApplied(const VideoStreamInputState & input_state,const VideoSourceRestrictions & restrictions_before,const VideoSourceRestrictions & restrictions_after,rtc::scoped_refptr<Resource> reason_resource)164 void QualityScalerResource::OnAdaptationApplied(
165 const VideoStreamInputState& input_state,
166 const VideoSourceRestrictions& restrictions_before,
167 const VideoSourceRestrictions& restrictions_after,
168 rtc::scoped_refptr<Resource> reason_resource) {
169 RTC_DCHECK_RUN_ON(resource_adaptation_queue());
170 // We only clear QP samples on adaptations triggered by the QualityScaler.
171 if (reason_resource != this)
172 return;
173 clear_qp_samples_ = true;
174 // If we're in "balanced" and the frame rate before and after adaptation did
175 // not differ that much, don't clear the QP samples and instead check for QP
176 // again in a short amount of time. This may trigger adapting down again soon.
177 // TODO(hbos): Can this be simplified by getting rid of special casing logic?
178 // For example, we could decide whether or not to clear QP samples based on
179 // how big the adaptation step was alone (regardless of degradation preference
180 // or what resource triggered the adaptation) and the QualityScaler could
181 // check for QP when it had enough QP samples rather than at a variable
182 // interval whose delay is calculated based on events such as these. Now there
183 // is much dependency on a specific OnReportQpUsageHigh() event and "balanced"
184 // but adaptations happening might not align with QualityScaler's CheckQpTask.
185 if (degradation_preference_provider_->degradation_preference() ==
186 DegradationPreference::BALANCED &&
187 DidDecreaseFrameRate(restrictions_before, restrictions_after)) {
188 absl::optional<int> min_diff = BalancedDegradationSettings().MinFpsDiff(
189 input_state.frame_size_pixels().value());
190 if (min_diff && input_state.frames_per_second() > 0) {
191 int fps_diff = input_state.frames_per_second() -
192 restrictions_after.max_frame_rate().value();
193 if (fps_diff < min_diff.value()) {
194 clear_qp_samples_ = false;
195 }
196 }
197 }
198 }
199
QueuePendingCallback(rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback)200 size_t QualityScalerResource::QueuePendingCallback(
201 rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback) {
202 RTC_DCHECK_RUN_ON(encoder_queue());
203 pending_callbacks_.push(callback);
204 // The ID of a callback is its sequence number (1, 2, 3...).
205 return num_handled_callbacks_ + pending_callbacks_.size();
206 }
207
HandlePendingCallback(size_t callback_id,bool clear_qp_samples)208 void QualityScalerResource::HandlePendingCallback(size_t callback_id,
209 bool clear_qp_samples) {
210 RTC_DCHECK_RUN_ON(resource_adaptation_queue());
211 // Reference counting guarantees that this object is still alive by the time
212 // the task is executed.
213 encoder_queue()->PostTask(
214 ToQueuedTask([this_ref = rtc::scoped_refptr<QualityScalerResource>(this),
215 callback_id, clear_qp_samples] {
216 RTC_DCHECK_RUN_ON(this_ref->encoder_queue());
217 if (this_ref->num_handled_callbacks_ >= callback_id) {
218 // The callback with this ID has already been handled.
219 // This happens if AbortPendingCallbacks() is called while the task is
220 // in flight.
221 return;
222 }
223 RTC_DCHECK(!this_ref->pending_callbacks_.empty());
224 this_ref->pending_callbacks_.front()->OnQpUsageHandled(
225 clear_qp_samples);
226 ++this_ref->num_handled_callbacks_;
227 this_ref->pending_callbacks_.pop();
228 }));
229 }
230
AbortPendingCallbacks()231 void QualityScalerResource::AbortPendingCallbacks() {
232 RTC_DCHECK_RUN_ON(encoder_queue());
233 while (!pending_callbacks_.empty()) {
234 pending_callbacks_.front()->OnQpUsageHandled(false);
235 ++num_handled_callbacks_;
236 pending_callbacks_.pop();
237 }
238 }
239
240 } // namespace webrtc
241