1 /*
2 * Copyright 2020 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/adaptation/video_stream_encoder_resource_manager.h"
12
13 #include <cmath>
14 #include <limits>
15 #include <memory>
16 #include <utility>
17
18 #include "absl/algorithm/container.h"
19 #include "absl/base/macros.h"
20 #include "api/adaptation/resource.h"
21 #include "api/task_queue/task_queue_base.h"
22 #include "api/video/video_adaptation_reason.h"
23 #include "api/video/video_source_interface.h"
24 #include "call/adaptation/video_source_restrictions.h"
25 #include "rtc_base/logging.h"
26 #include "rtc_base/numerics/safe_conversions.h"
27 #include "rtc_base/ref_counted_object.h"
28 #include "rtc_base/strings/string_builder.h"
29 #include "rtc_base/time_utils.h"
30
31 namespace webrtc {
32
33 const int kDefaultInputPixelsWidth = 176;
34 const int kDefaultInputPixelsHeight = 144;
35
36 namespace {
37
IsResolutionScalingEnabled(DegradationPreference degradation_preference)38 bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
39 return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE ||
40 degradation_preference == DegradationPreference::BALANCED;
41 }
42
IsFramerateScalingEnabled(DegradationPreference degradation_preference)43 bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) {
44 return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION ||
45 degradation_preference == DegradationPreference::BALANCED;
46 }
47
ToString(VideoAdaptationReason reason)48 std::string ToString(VideoAdaptationReason reason) {
49 switch (reason) {
50 case VideoAdaptationReason::kQuality:
51 return "quality";
52 case VideoAdaptationReason::kCpu:
53 return "cpu";
54 }
55 }
56
57 } // namespace
58
59 class VideoStreamEncoderResourceManager::InitialFrameDropper {
60 public:
InitialFrameDropper(rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource)61 explicit InitialFrameDropper(
62 rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource)
63 : quality_scaler_resource_(quality_scaler_resource),
64 quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()),
65 has_seen_first_bwe_drop_(false),
66 set_start_bitrate_(DataRate::Zero()),
67 set_start_bitrate_time_ms_(0),
68 initial_framedrop_(0) {
69 RTC_DCHECK(quality_scaler_resource_);
70 }
71
72 // Output signal.
DropInitialFrames() const73 bool DropInitialFrames() const {
74 return initial_framedrop_ < kMaxInitialFramedrop;
75 }
76
77 // Input signals.
SetStartBitrate(DataRate start_bitrate,int64_t now_ms)78 void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) {
79 set_start_bitrate_ = start_bitrate;
80 set_start_bitrate_time_ms_ = now_ms;
81 }
82
SetTargetBitrate(DataRate target_bitrate,int64_t now_ms)83 void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) {
84 if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ &&
85 quality_scaler_resource_->is_started() &&
86 quality_scaler_settings_.InitialBitrateIntervalMs() &&
87 quality_scaler_settings_.InitialBitrateFactor()) {
88 int64_t diff_ms = now_ms - set_start_bitrate_time_ms_;
89 if (diff_ms <
90 quality_scaler_settings_.InitialBitrateIntervalMs().value() &&
91 (target_bitrate <
92 (set_start_bitrate_ *
93 quality_scaler_settings_.InitialBitrateFactor().value()))) {
94 RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: "
95 << set_start_bitrate_.bps()
96 << ", target bitrate: " << target_bitrate.bps();
97 initial_framedrop_ = 0;
98 has_seen_first_bwe_drop_ = true;
99 }
100 }
101 }
102
OnFrameDroppedDueToSize()103 void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
104
OnMaybeEncodeFrame()105 void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; }
106
OnQualityScalerSettingsUpdated()107 void OnQualityScalerSettingsUpdated() {
108 if (quality_scaler_resource_->is_started()) {
109 // Restart frame drops due to size.
110 initial_framedrop_ = 0;
111 } else {
112 // Quality scaling disabled so we shouldn't drop initial frames.
113 initial_framedrop_ = kMaxInitialFramedrop;
114 }
115 }
116
117 private:
118 // The maximum number of frames to drop at beginning of stream to try and
119 // achieve desired bitrate.
120 static const int kMaxInitialFramedrop = 4;
121
122 const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
123 const QualityScalerSettings quality_scaler_settings_;
124 bool has_seen_first_bwe_drop_;
125 DataRate set_start_bitrate_;
126 int64_t set_start_bitrate_time_ms_;
127 // Counts how many frames we've dropped in the initial framedrop phase.
128 int initial_framedrop_;
129 };
130
BitrateConstraint(VideoStreamEncoderResourceManager * manager)131 VideoStreamEncoderResourceManager::BitrateConstraint::BitrateConstraint(
132 VideoStreamEncoderResourceManager* manager)
133 : manager_(manager),
134 resource_adaptation_queue_(nullptr),
135 encoder_settings_(absl::nullopt),
136 encoder_target_bitrate_bps_(absl::nullopt) {}
137
SetAdaptationQueue(TaskQueueBase * resource_adaptation_queue)138 void VideoStreamEncoderResourceManager::BitrateConstraint::SetAdaptationQueue(
139 TaskQueueBase* resource_adaptation_queue) {
140 resource_adaptation_queue_ = resource_adaptation_queue;
141 }
142
143 void VideoStreamEncoderResourceManager::BitrateConstraint::
OnEncoderSettingsUpdated(absl::optional<EncoderSettings> encoder_settings)144 OnEncoderSettingsUpdated(absl::optional<EncoderSettings> encoder_settings) {
145 RTC_DCHECK_RUN_ON(manager_->encoder_queue_);
146 resource_adaptation_queue_->PostTask(
147 ToQueuedTask([this_ref = rtc::scoped_refptr<BitrateConstraint>(this),
148 encoder_settings] {
149 RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_);
150 this_ref->encoder_settings_ = std::move(encoder_settings);
151 }));
152 }
153
154 void VideoStreamEncoderResourceManager::BitrateConstraint::
OnEncoderTargetBitrateUpdated(absl::optional<uint32_t> encoder_target_bitrate_bps)155 OnEncoderTargetBitrateUpdated(
156 absl::optional<uint32_t> encoder_target_bitrate_bps) {
157 RTC_DCHECK_RUN_ON(manager_->encoder_queue_);
158 resource_adaptation_queue_->PostTask(
159 ToQueuedTask([this_ref = rtc::scoped_refptr<BitrateConstraint>(this),
160 encoder_target_bitrate_bps] {
161 RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_);
162 this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps;
163 }));
164 }
165
166 bool VideoStreamEncoderResourceManager::BitrateConstraint::
IsAdaptationUpAllowed(const VideoStreamInputState & input_state,const VideoSourceRestrictions & restrictions_before,const VideoSourceRestrictions & restrictions_after,rtc::scoped_refptr<Resource> reason_resource) const167 IsAdaptationUpAllowed(const VideoStreamInputState& input_state,
168 const VideoSourceRestrictions& restrictions_before,
169 const VideoSourceRestrictions& restrictions_after,
170 rtc::scoped_refptr<Resource> reason_resource) const {
171 RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
172 VideoAdaptationReason reason =
173 manager_->GetReasonFromResource(reason_resource);
174 // If increasing resolution due to kQuality, make sure bitrate limits are not
175 // violated.
176 // TODO(https://crbug.com/webrtc/11771): Why are we allowing violating bitrate
177 // constraints if adapting due to CPU? Shouldn't this condition be checked
178 // regardless of reason?
179 if (reason == VideoAdaptationReason::kQuality &&
180 DidIncreaseResolution(restrictions_before, restrictions_after)) {
181 uint32_t bitrate_bps = encoder_target_bitrate_bps_.value_or(0);
182 absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
183 encoder_settings_.has_value()
184 ? encoder_settings_->encoder_info()
185 .GetEncoderBitrateLimitsForResolution(
186 // Need some sort of expected resulting pixels to be used
187 // instead of unrestricted.
188 GetHigherResolutionThan(
189 input_state.frame_size_pixels().value()))
190 : absl::nullopt;
191 if (bitrate_limits.has_value() && bitrate_bps != 0) {
192 RTC_DCHECK_GE(bitrate_limits->frame_size_pixels,
193 input_state.frame_size_pixels().value());
194 return bitrate_bps >=
195 static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
196 }
197 }
198 return true;
199 }
200
BalancedConstraint(VideoStreamEncoderResourceManager * manager,DegradationPreferenceProvider * degradation_preference_provider)201 VideoStreamEncoderResourceManager::BalancedConstraint::BalancedConstraint(
202 VideoStreamEncoderResourceManager* manager,
203 DegradationPreferenceProvider* degradation_preference_provider)
204 : manager_(manager),
205 resource_adaptation_queue_(nullptr),
206 encoder_target_bitrate_bps_(absl::nullopt),
207 degradation_preference_provider_(degradation_preference_provider) {
208 RTC_DCHECK(manager_);
209 RTC_DCHECK(degradation_preference_provider_);
210 }
211
SetAdaptationQueue(TaskQueueBase * resource_adaptation_queue)212 void VideoStreamEncoderResourceManager::BalancedConstraint::SetAdaptationQueue(
213 TaskQueueBase* resource_adaptation_queue) {
214 resource_adaptation_queue_ = resource_adaptation_queue;
215 }
216
217 void VideoStreamEncoderResourceManager::BalancedConstraint::
OnEncoderTargetBitrateUpdated(absl::optional<uint32_t> encoder_target_bitrate_bps)218 OnEncoderTargetBitrateUpdated(
219 absl::optional<uint32_t> encoder_target_bitrate_bps) {
220 RTC_DCHECK_RUN_ON(manager_->encoder_queue_);
221 resource_adaptation_queue_->PostTask(
222 ToQueuedTask([this_ref = rtc::scoped_refptr<BalancedConstraint>(this),
223 encoder_target_bitrate_bps] {
224 RTC_DCHECK_RUN_ON(this_ref->resource_adaptation_queue_);
225 this_ref->encoder_target_bitrate_bps_ = encoder_target_bitrate_bps;
226 }));
227 }
228
229 bool VideoStreamEncoderResourceManager::BalancedConstraint::
IsAdaptationUpAllowed(const VideoStreamInputState & input_state,const VideoSourceRestrictions & restrictions_before,const VideoSourceRestrictions & restrictions_after,rtc::scoped_refptr<Resource> reason_resource) const230 IsAdaptationUpAllowed(const VideoStreamInputState& input_state,
231 const VideoSourceRestrictions& restrictions_before,
232 const VideoSourceRestrictions& restrictions_after,
233 rtc::scoped_refptr<Resource> reason_resource) const {
234 RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
235 VideoAdaptationReason reason =
236 manager_->GetReasonFromResource(reason_resource);
237 // Don't adapt if BalancedDegradationSettings applies and determines this will
238 // exceed bitrate constraints.
239 // TODO(https://crbug.com/webrtc/11771): Why are we allowing violating
240 // balanced settings if adapting due CPU? Shouldn't this condition be checked
241 // regardless of reason?
242 if (reason == VideoAdaptationReason::kQuality &&
243 degradation_preference_provider_->degradation_preference() ==
244 DegradationPreference::BALANCED &&
245 !manager_->balanced_settings_.CanAdaptUp(
246 input_state.video_codec_type(),
247 input_state.frame_size_pixels().value(),
248 encoder_target_bitrate_bps_.value_or(0))) {
249 return false;
250 }
251 if (reason == VideoAdaptationReason::kQuality &&
252 DidIncreaseResolution(restrictions_before, restrictions_after) &&
253 !manager_->balanced_settings_.CanAdaptUpResolution(
254 input_state.video_codec_type(),
255 input_state.frame_size_pixels().value(),
256 encoder_target_bitrate_bps_.value_or(0))) {
257 return false;
258 }
259 return true;
260 }
261
VideoStreamEncoderResourceManager(VideoStreamInputStateProvider * input_state_provider,VideoStreamEncoderObserver * encoder_stats_observer,Clock * clock,bool experiment_cpu_load_estimator,std::unique_ptr<OveruseFrameDetector> overuse_detector,DegradationPreferenceProvider * degradation_preference_provider)262 VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
263 VideoStreamInputStateProvider* input_state_provider,
264 VideoStreamEncoderObserver* encoder_stats_observer,
265 Clock* clock,
266 bool experiment_cpu_load_estimator,
267 std::unique_ptr<OveruseFrameDetector> overuse_detector,
268 DegradationPreferenceProvider* degradation_preference_provider)
269 : degradation_preference_provider_(degradation_preference_provider),
270 bitrate_constraint_(new rtc::RefCountedObject<BitrateConstraint>(this)),
271 balanced_constraint_(new rtc::RefCountedObject<BalancedConstraint>(
272 this,
273 degradation_preference_provider_)),
274 encode_usage_resource_(
275 EncodeUsageResource::Create(std::move(overuse_detector))),
276 quality_scaler_resource_(
277 QualityScalerResource::Create(degradation_preference_provider_)),
278 encoder_queue_(nullptr),
279 resource_adaptation_queue_(nullptr),
280 input_state_provider_(input_state_provider),
281 adaptation_processor_(nullptr),
282 encoder_stats_observer_(encoder_stats_observer),
283 degradation_preference_(DegradationPreference::DISABLED),
284 video_source_restrictions_(),
285 clock_(clock),
286 experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
287 initial_frame_dropper_(
288 std::make_unique<InitialFrameDropper>(quality_scaler_resource_)),
289 quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
290 encoder_target_bitrate_bps_(absl::nullopt),
291 quality_rampup_experiment_(
292 QualityRampUpExperimentHelper::CreateIfEnabled(this, clock_)),
293 encoder_settings_(absl::nullopt) {
294 RTC_CHECK(degradation_preference_provider_);
295 RTC_CHECK(encoder_stats_observer_);
296 MapResourceToReason(encode_usage_resource_, VideoAdaptationReason::kCpu);
297 MapResourceToReason(quality_scaler_resource_,
298 VideoAdaptationReason::kQuality);
299 }
300
~VideoStreamEncoderResourceManager()301 VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() {}
302
Initialize(rtc::TaskQueue * encoder_queue,rtc::TaskQueue * resource_adaptation_queue)303 void VideoStreamEncoderResourceManager::Initialize(
304 rtc::TaskQueue* encoder_queue,
305 rtc::TaskQueue* resource_adaptation_queue) {
306 RTC_DCHECK(!encoder_queue_);
307 RTC_DCHECK(encoder_queue);
308 RTC_DCHECK(!resource_adaptation_queue_);
309 RTC_DCHECK(resource_adaptation_queue);
310 encoder_queue_ = encoder_queue;
311 resource_adaptation_queue_ = resource_adaptation_queue;
312 bitrate_constraint_->SetAdaptationQueue(resource_adaptation_queue_->Get());
313 balanced_constraint_->SetAdaptationQueue(resource_adaptation_queue_->Get());
314 encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
315 encode_usage_resource_->RegisterAdaptationTaskQueue(
316 resource_adaptation_queue_->Get());
317 quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
318 quality_scaler_resource_->RegisterAdaptationTaskQueue(
319 resource_adaptation_queue_->Get());
320 }
321
SetAdaptationProcessor(ResourceAdaptationProcessorInterface * adaptation_processor,VideoStreamAdapter * stream_adapter)322 void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
323 ResourceAdaptationProcessorInterface* adaptation_processor,
324 VideoStreamAdapter* stream_adapter) {
325 RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
326 adaptation_processor_ = adaptation_processor;
327 stream_adapter_ = stream_adapter;
328 }
329
SetDegradationPreferences(DegradationPreference degradation_preference)330 void VideoStreamEncoderResourceManager::SetDegradationPreferences(
331 DegradationPreference degradation_preference) {
332 RTC_DCHECK_RUN_ON(encoder_queue_);
333 degradation_preference_ = degradation_preference;
334 UpdateStatsAdaptationSettings();
335 }
336
337 DegradationPreference
degradation_preference() const338 VideoStreamEncoderResourceManager::degradation_preference() const {
339 RTC_DCHECK_RUN_ON(encoder_queue_);
340 return degradation_preference_;
341 }
342
StartEncodeUsageResource()343 void VideoStreamEncoderResourceManager::StartEncodeUsageResource() {
344 RTC_DCHECK_RUN_ON(encoder_queue_);
345 RTC_DCHECK(!encode_usage_resource_->is_started());
346 RTC_DCHECK(encoder_settings_.has_value());
347 encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions());
348 }
349
StopManagedResources()350 void VideoStreamEncoderResourceManager::StopManagedResources() {
351 RTC_DCHECK_RUN_ON(encoder_queue_);
352 encode_usage_resource_->StopCheckForOveruse();
353 quality_scaler_resource_->StopCheckForOveruse();
354 }
355
MapResourceToReason(rtc::scoped_refptr<Resource> resource,VideoAdaptationReason reason)356 void VideoStreamEncoderResourceManager::MapResourceToReason(
357 rtc::scoped_refptr<Resource> resource,
358 VideoAdaptationReason reason) {
359 MutexLock lock(&resource_lock_);
360 RTC_DCHECK(resource);
361 RTC_DCHECK(absl::c_find_if(resources_,
362 [resource](const ResourceAndReason& r) {
363 return r.resource == resource;
364 }) == resources_.end())
365 << "Resource " << resource->Name() << " already was inserted";
366 resources_.emplace_back(resource, reason);
367 }
368
369 std::vector<rtc::scoped_refptr<Resource>>
MappedResources() const370 VideoStreamEncoderResourceManager::MappedResources() const {
371 MutexLock lock(&resource_lock_);
372 std::vector<rtc::scoped_refptr<Resource>> resources;
373 for (auto const& resource_and_reason : resources_) {
374 resources.push_back(resource_and_reason.resource);
375 }
376 return resources;
377 }
378
379 std::vector<AdaptationConstraint*>
AdaptationConstraints() const380 VideoStreamEncoderResourceManager::AdaptationConstraints() const {
381 return {bitrate_constraint_, balanced_constraint_};
382 }
383
384 std::vector<AdaptationListener*>
AdaptationListeners() const385 VideoStreamEncoderResourceManager::AdaptationListeners() const {
386 return {quality_scaler_resource_};
387 }
388
389 rtc::scoped_refptr<QualityScalerResource>
quality_scaler_resource_for_testing()390 VideoStreamEncoderResourceManager::quality_scaler_resource_for_testing() {
391 MutexLock lock(&resource_lock_);
392 return quality_scaler_resource_;
393 }
394
SetEncoderSettings(EncoderSettings encoder_settings)395 void VideoStreamEncoderResourceManager::SetEncoderSettings(
396 EncoderSettings encoder_settings) {
397 RTC_DCHECK_RUN_ON(encoder_queue_);
398 encoder_settings_ = std::move(encoder_settings);
399 bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_);
400 MaybeUpdateTargetFrameRate();
401 }
402
SetStartBitrate(DataRate start_bitrate)403 void VideoStreamEncoderResourceManager::SetStartBitrate(
404 DataRate start_bitrate) {
405 RTC_DCHECK_RUN_ON(encoder_queue_);
406 if (!start_bitrate.IsZero()) {
407 encoder_target_bitrate_bps_ = start_bitrate.bps();
408 bitrate_constraint_->OnEncoderTargetBitrateUpdated(
409 encoder_target_bitrate_bps_);
410 balanced_constraint_->OnEncoderTargetBitrateUpdated(
411 encoder_target_bitrate_bps_);
412 }
413 initial_frame_dropper_->SetStartBitrate(start_bitrate,
414 clock_->TimeInMicroseconds());
415 }
416
SetTargetBitrate(DataRate target_bitrate)417 void VideoStreamEncoderResourceManager::SetTargetBitrate(
418 DataRate target_bitrate) {
419 RTC_DCHECK_RUN_ON(encoder_queue_);
420 if (!target_bitrate.IsZero()) {
421 encoder_target_bitrate_bps_ = target_bitrate.bps();
422 bitrate_constraint_->OnEncoderTargetBitrateUpdated(
423 encoder_target_bitrate_bps_);
424 balanced_constraint_->OnEncoderTargetBitrateUpdated(
425 encoder_target_bitrate_bps_);
426 }
427 initial_frame_dropper_->SetTargetBitrate(target_bitrate,
428 clock_->TimeInMilliseconds());
429 }
430
SetEncoderRates(const VideoEncoder::RateControlParameters & encoder_rates)431 void VideoStreamEncoderResourceManager::SetEncoderRates(
432 const VideoEncoder::RateControlParameters& encoder_rates) {
433 RTC_DCHECK_RUN_ON(encoder_queue_);
434 encoder_rates_ = encoder_rates;
435 }
436
OnFrameDroppedDueToSize()437 void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() {
438 RTC_DCHECK_RUN_ON(encoder_queue_);
439 // The VideoStreamEncoder makes the manager outlive the adaptation queue. This
440 // means that if the task gets executed, |this| has not been freed yet.
441 // TODO(https://crbug.com/webrtc/11565): When the manager no longer outlives
442 // the adaptation queue, add logic to prevent use-after-free on |this|.
443 resource_adaptation_queue_->PostTask([this] {
444 RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
445 if (!adaptation_processor_) {
446 // The processor nulled before this task had a chance to execute. This
447 // happens if the processor is destroyed. No action needed.
448 return;
449 }
450 Adaptation reduce_resolution = stream_adapter_->GetAdaptDownResolution();
451 if (reduce_resolution.status() == Adaptation::Status::kValid) {
452 stream_adapter_->ApplyAdaptation(reduce_resolution,
453 quality_scaler_resource_);
454 }
455 });
456 initial_frame_dropper_->OnFrameDroppedDueToSize();
457 }
458
OnEncodeStarted(const VideoFrame & cropped_frame,int64_t time_when_first_seen_us)459 void VideoStreamEncoderResourceManager::OnEncodeStarted(
460 const VideoFrame& cropped_frame,
461 int64_t time_when_first_seen_us) {
462 RTC_DCHECK_RUN_ON(encoder_queue_);
463 encode_usage_resource_->OnEncodeStarted(cropped_frame,
464 time_when_first_seen_us);
465 }
466
OnEncodeCompleted(const EncodedImage & encoded_image,int64_t time_sent_in_us,absl::optional<int> encode_duration_us)467 void VideoStreamEncoderResourceManager::OnEncodeCompleted(
468 const EncodedImage& encoded_image,
469 int64_t time_sent_in_us,
470 absl::optional<int> encode_duration_us) {
471 RTC_DCHECK_RUN_ON(encoder_queue_);
472 // Inform |encode_usage_resource_| of the encode completed event.
473 uint32_t timestamp = encoded_image.Timestamp();
474 int64_t capture_time_us =
475 encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
476 encode_usage_resource_->OnEncodeCompleted(
477 timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
478 // Inform |quality_scaler_resource_| of the encode completed event.
479 quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
480 }
481
OnFrameDropped(EncodedImageCallback::DropReason reason)482 void VideoStreamEncoderResourceManager::OnFrameDropped(
483 EncodedImageCallback::DropReason reason) {
484 RTC_DCHECK_RUN_ON(encoder_queue_);
485 quality_scaler_resource_->OnFrameDropped(reason);
486 }
487
DropInitialFrames() const488 bool VideoStreamEncoderResourceManager::DropInitialFrames() const {
489 RTC_DCHECK_RUN_ON(encoder_queue_);
490 return initial_frame_dropper_->DropInitialFrames();
491 }
492
OnMaybeEncodeFrame()493 void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
494 RTC_DCHECK_RUN_ON(encoder_queue_);
495 initial_frame_dropper_->OnMaybeEncodeFrame();
496 if (quality_rampup_experiment_) {
497 DataRate bandwidth = encoder_rates_.has_value()
498 ? encoder_rates_->bandwidth_allocation
499 : DataRate::Zero();
500 quality_rampup_experiment_->PerformQualityRampupExperiment(
501 quality_scaler_resource_, bandwidth,
502 DataRate::BitsPerSec(encoder_target_bitrate_bps_.value_or(0)),
503 DataRate::KilobitsPerSec(encoder_settings_->video_codec().maxBitrate),
504 LastInputFrameSizeOrDefault());
505 }
506 }
507
UpdateQualityScalerSettings(absl::optional<VideoEncoder::QpThresholds> qp_thresholds)508 void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings(
509 absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
510 RTC_DCHECK_RUN_ON(encoder_queue_);
511 if (qp_thresholds.has_value()) {
512 quality_scaler_resource_->StopCheckForOveruse();
513 quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value());
514 } else {
515 quality_scaler_resource_->StopCheckForOveruse();
516 }
517 initial_frame_dropper_->OnQualityScalerSettingsUpdated();
518 }
519
ConfigureQualityScaler(const VideoEncoder::EncoderInfo & encoder_info)520 void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
521 const VideoEncoder::EncoderInfo& encoder_info) {
522 RTC_DCHECK_RUN_ON(encoder_queue_);
523 const auto scaling_settings = encoder_info.scaling_settings;
524 const bool quality_scaling_allowed =
525 IsResolutionScalingEnabled(degradation_preference_) &&
526 scaling_settings.thresholds;
527
528 // TODO(https://crbug.com/webrtc/11222): Should this move to
529 // QualityScalerResource?
530 if (quality_scaling_allowed) {
531 if (!quality_scaler_resource_->is_started()) {
532 // Quality scaler has not already been configured.
533
534 // Use experimental thresholds if available.
535 absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
536 if (quality_scaling_experiment_enabled_) {
537 experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
538 GetVideoCodecTypeOrGeneric(encoder_settings_));
539 }
540 UpdateQualityScalerSettings(experimental_thresholds
541 ? *experimental_thresholds
542 : *(scaling_settings.thresholds));
543 }
544 } else {
545 UpdateQualityScalerSettings(absl::nullopt);
546 }
547
548 // Set the qp-thresholds to the balanced settings if balanced mode.
549 if (degradation_preference_ == DegradationPreference::BALANCED &&
550 quality_scaler_resource_->is_started()) {
551 absl::optional<VideoEncoder::QpThresholds> thresholds =
552 balanced_settings_.GetQpThresholds(
553 GetVideoCodecTypeOrGeneric(encoder_settings_),
554 LastInputFrameSizeOrDefault());
555 if (thresholds) {
556 quality_scaler_resource_->SetQpThresholds(*thresholds);
557 }
558 }
559 UpdateStatsAdaptationSettings();
560 }
561
GetReasonFromResource(rtc::scoped_refptr<Resource> resource) const562 VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource(
563 rtc::scoped_refptr<Resource> resource) const {
564 MutexLock lock(&resource_lock_);
565 const auto& registered_resource =
566 absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) {
567 return r.resource == resource;
568 });
569 RTC_DCHECK(registered_resource != resources_.end())
570 << resource->Name() << " not found.";
571 return registered_resource->reason;
572 }
573
574 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
575 // pipelining encoders better (multiple input frames before something comes
576 // out). This should effectively turn off CPU adaptations for systems that
577 // remotely cope with the load right now.
GetCpuOveruseOptions() const578 CpuOveruseOptions VideoStreamEncoderResourceManager::GetCpuOveruseOptions()
579 const {
580 RTC_DCHECK_RUN_ON(encoder_queue_);
581 // This is already ensured by the only caller of this method:
582 // StartResourceAdaptation().
583 RTC_DCHECK(encoder_settings_.has_value());
584 CpuOveruseOptions options;
585 // Hardware accelerated encoders are assumed to be pipelined; give them
586 // additional overuse time.
587 if (encoder_settings_->encoder_info().is_hardware_accelerated) {
588 options.low_encode_usage_threshold_percent = 150;
589 options.high_encode_usage_threshold_percent = 200;
590 }
591 if (experiment_cpu_load_estimator_) {
592 options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec;
593 }
594 return options;
595 }
596
LastInputFrameSizeOrDefault() const597 int VideoStreamEncoderResourceManager::LastInputFrameSizeOrDefault() const {
598 RTC_DCHECK_RUN_ON(encoder_queue_);
599 return input_state_provider_->InputState().frame_size_pixels().value_or(
600 kDefaultInputPixelsWidth * kDefaultInputPixelsHeight);
601 }
602
OnVideoSourceRestrictionsUpdated(VideoSourceRestrictions restrictions,const VideoAdaptationCounters & adaptation_counters,rtc::scoped_refptr<Resource> reason,const VideoSourceRestrictions & unfiltered_restrictions)603 void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
604 VideoSourceRestrictions restrictions,
605 const VideoAdaptationCounters& adaptation_counters,
606 rtc::scoped_refptr<Resource> reason,
607 const VideoSourceRestrictions& unfiltered_restrictions) {
608 RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
609 // TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback.
610 if (!reason && adaptation_counters.Total() == 0) {
611 // Adaptation was manually reset - clear the per-reason counters too.
612 encoder_stats_observer_->ClearAdaptationStats();
613 }
614
615 // The VideoStreamEncoder makes the manager outlive the encoder queue. This
616 // means that if the task gets executed, |this| has not been freed yet.
617 encoder_queue_->PostTask([this, restrictions] {
618 RTC_DCHECK_RUN_ON(encoder_queue_);
619 video_source_restrictions_ = FilterRestrictionsByDegradationPreference(
620 restrictions, degradation_preference_);
621 MaybeUpdateTargetFrameRate();
622 });
623 }
624
OnResourceLimitationChanged(rtc::scoped_refptr<Resource> resource,const std::map<rtc::scoped_refptr<Resource>,VideoAdaptationCounters> & resource_limitations)625 void VideoStreamEncoderResourceManager::OnResourceLimitationChanged(
626 rtc::scoped_refptr<Resource> resource,
627 const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
628 resource_limitations) {
629 RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
630 if (!resource) {
631 encoder_stats_observer_->ClearAdaptationStats();
632 return;
633 }
634
635 std::map<VideoAdaptationReason, VideoAdaptationCounters> limitations;
636 for (auto& resource_counter : resource_limitations) {
637 std::map<VideoAdaptationReason, VideoAdaptationCounters>::iterator it;
638 bool inserted;
639 std::tie(it, inserted) = limitations.emplace(
640 GetReasonFromResource(resource_counter.first), resource_counter.second);
641 if (!inserted && it->second.Total() < resource_counter.second.Total()) {
642 it->second = resource_counter.second;
643 }
644 }
645
646 VideoAdaptationReason adaptation_reason = GetReasonFromResource(resource);
647 encoder_stats_observer_->OnAdaptationChanged(
648 adaptation_reason, limitations[VideoAdaptationReason::kCpu],
649 limitations[VideoAdaptationReason::kQuality]);
650
651 encoder_queue_->PostTask(ToQueuedTask(
652 [cpu_limited = limitations.at(VideoAdaptationReason::kCpu).Total() > 0,
653 qp_resolution_adaptations =
654 limitations.at(VideoAdaptationReason::kQuality)
655 .resolution_adaptations,
656 this]() {
657 RTC_DCHECK_RUN_ON(encoder_queue_);
658 if (quality_rampup_experiment_) {
659 quality_rampup_experiment_->cpu_adapted(cpu_limited);
660 quality_rampup_experiment_->qp_resolution_adaptations(
661 qp_resolution_adaptations);
662 }
663 }));
664
665 RTC_LOG(LS_INFO) << ActiveCountsToString(limitations);
666 }
667
MaybeUpdateTargetFrameRate()668 void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() {
669 RTC_DCHECK_RUN_ON(encoder_queue_);
670 absl::optional<double> codec_max_frame_rate =
671 encoder_settings_.has_value()
672 ? absl::optional<double>(
673 encoder_settings_->video_codec().maxFramerate)
674 : absl::nullopt;
675 // The current target framerate is the maximum frame rate as specified by
676 // the current codec configuration or any limit imposed by the adaptation
677 // module. This is used to make sure overuse detection doesn't needlessly
678 // trigger in low and/or variable framerate scenarios.
679 absl::optional<double> target_frame_rate =
680 video_source_restrictions_.max_frame_rate();
681 if (!target_frame_rate.has_value() ||
682 (codec_max_frame_rate.has_value() &&
683 codec_max_frame_rate.value() < target_frame_rate.value())) {
684 target_frame_rate = codec_max_frame_rate;
685 }
686 encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
687 }
688
UpdateStatsAdaptationSettings() const689 void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const {
690 RTC_DCHECK_RUN_ON(encoder_queue_);
691 VideoStreamEncoderObserver::AdaptationSettings cpu_settings(
692 IsResolutionScalingEnabled(degradation_preference_),
693 IsFramerateScalingEnabled(degradation_preference_));
694
695 VideoStreamEncoderObserver::AdaptationSettings quality_settings =
696 quality_scaler_resource_->is_started()
697 ? cpu_settings
698 : VideoStreamEncoderObserver::AdaptationSettings();
699 encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings,
700 quality_settings);
701 }
702
703 // static
ActiveCountsToString(const std::map<VideoAdaptationReason,VideoAdaptationCounters> & active_counts)704 std::string VideoStreamEncoderResourceManager::ActiveCountsToString(
705 const std::map<VideoAdaptationReason, VideoAdaptationCounters>&
706 active_counts) {
707 rtc::StringBuilder ss;
708
709 ss << "Downgrade counts: fps: {";
710 for (auto& reason_count : active_counts) {
711 ss << ToString(reason_count.first) << ":";
712 ss << reason_count.second.fps_adaptations;
713 }
714 ss << "}, resolution {";
715 for (auto& reason_count : active_counts) {
716 ss << ToString(reason_count.first) << ":";
717 ss << reason_count.second.resolution_adaptations;
718 }
719 ss << "}";
720
721 return ss.Release();
722 }
723
OnQualityRampUp()724 void VideoStreamEncoderResourceManager::OnQualityRampUp() {
725 RTC_DCHECK_RUN_ON(encoder_queue_);
726 // The VideoStreamEncoder makes the manager outlive the adaptation queue.
727 // This means that if the task gets executed, |this| has not been freed yet.
728 // TODO(https://crbug.com/webrtc/11565): When the manager no longer outlives
729 // the adaptation queue, add logic to prevent use-after-free on |this|.
730 resource_adaptation_queue_->PostTask([this] {
731 RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
732 if (!stream_adapter_) {
733 // The processor nulled before this task had a chance to execute. This
734 // happens if the processor is destroyed. No action needed.
735 return;
736 }
737 stream_adapter_->ClearRestrictions();
738 });
739 quality_rampup_experiment_.reset();
740 }
741 } // namespace webrtc
742