1 /*
2 * Copyright 2020 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/adaptation/video_stream_encoder_resource_manager.h"
12
13 #include <stdio.h>
14
15 #include <algorithm>
16 #include <cmath>
17 #include <limits>
18 #include <memory>
19 #include <utility>
20
21 #include "absl/algorithm/container.h"
22 #include "absl/base/macros.h"
23 #include "api/adaptation/resource.h"
24 #include "api/sequence_checker.h"
25 #include "api/task_queue/task_queue_base.h"
26 #include "api/video/video_adaptation_reason.h"
27 #include "api/video/video_source_interface.h"
28 #include "call/adaptation/video_source_restrictions.h"
29 #include "modules/video_coding/svc/scalability_mode_util.h"
30 #include "rtc_base/checks.h"
31 #include "rtc_base/logging.h"
32 #include "rtc_base/numerics/safe_conversions.h"
33 #include "rtc_base/strings/string_builder.h"
34 #include "rtc_base/time_utils.h"
35 #include "rtc_base/trace_event.h"
36 #include "video/adaptation/quality_scaler_resource.h"
37
38 namespace webrtc {
39
40 const int kDefaultInputPixelsWidth = 176;
41 const int kDefaultInputPixelsHeight = 144;
42
43 namespace {
44
45 constexpr const char* kPixelLimitResourceFieldTrialName =
46 "WebRTC-PixelLimitResource";
47
IsResolutionScalingEnabled(DegradationPreference degradation_preference)48 bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
49 return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE ||
50 degradation_preference == DegradationPreference::BALANCED;
51 }
52
IsFramerateScalingEnabled(DegradationPreference degradation_preference)53 bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) {
54 return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION ||
55 degradation_preference == DegradationPreference::BALANCED;
56 }
57
ToString(VideoAdaptationReason reason)58 std::string ToString(VideoAdaptationReason reason) {
59 switch (reason) {
60 case VideoAdaptationReason::kQuality:
61 return "quality";
62 case VideoAdaptationReason::kCpu:
63 return "cpu";
64 }
65 RTC_CHECK_NOTREACHED();
66 }
67
GetActiveLayersFlags(const VideoCodec & codec)68 std::vector<bool> GetActiveLayersFlags(const VideoCodec& codec) {
69 std::vector<bool> flags;
70 if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
71 flags.resize(codec.VP9().numberOfSpatialLayers);
72 for (size_t i = 0; i < flags.size(); ++i) {
73 flags[i] = codec.spatialLayers[i].active;
74 }
75 } else {
76 flags.resize(codec.numberOfSimulcastStreams);
77 for (size_t i = 0; i < flags.size(); ++i) {
78 flags[i] = codec.simulcastStream[i].active;
79 }
80 }
81 return flags;
82 }
83
EqualFlags(const std::vector<bool> & a,const std::vector<bool> & b)84 bool EqualFlags(const std::vector<bool>& a, const std::vector<bool>& b) {
85 if (a.size() != b.size())
86 return false;
87 return std::equal(a.begin(), a.end(), b.begin());
88 }
89
GetSingleActiveLayerMaxBitrate(const VideoCodec & codec)90 absl::optional<DataRate> GetSingleActiveLayerMaxBitrate(
91 const VideoCodec& codec) {
92 int num_active = 0;
93 absl::optional<DataRate> max_bitrate;
94 if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
95 for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
96 if (codec.spatialLayers[i].active) {
97 ++num_active;
98 max_bitrate =
99 DataRate::KilobitsPerSec(codec.spatialLayers[i].maxBitrate);
100 }
101 }
102 } else {
103 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
104 if (codec.simulcastStream[i].active) {
105 ++num_active;
106 max_bitrate =
107 DataRate::KilobitsPerSec(codec.simulcastStream[i].maxBitrate);
108 }
109 }
110 }
111 return (num_active > 1) ? absl::nullopt : max_bitrate;
112 }
113
114 } // namespace
115
116 class VideoStreamEncoderResourceManager::InitialFrameDropper {
117 public:
InitialFrameDropper(rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource)118 explicit InitialFrameDropper(
119 rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource)
120 : quality_scaler_resource_(quality_scaler_resource),
121 quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()),
122 has_seen_first_bwe_drop_(false),
123 set_start_bitrate_(DataRate::Zero()),
124 set_start_bitrate_time_ms_(0),
125 initial_framedrop_(0),
126 use_bandwidth_allocation_(false),
127 bandwidth_allocation_(DataRate::Zero()),
128 last_input_width_(0),
129 last_input_height_(0),
130 last_stream_configuration_changed_(false) {
131 RTC_DCHECK(quality_scaler_resource_);
132 }
133
134 // Output signal.
DropInitialFrames() const135 bool DropInitialFrames() const {
136 return initial_framedrop_ < kMaxInitialFramedrop;
137 }
138
single_active_stream_pixels() const139 absl::optional<uint32_t> single_active_stream_pixels() const {
140 return single_active_stream_pixels_;
141 }
142
UseBandwidthAllocationBps() const143 absl::optional<uint32_t> UseBandwidthAllocationBps() const {
144 return (use_bandwidth_allocation_ &&
145 bandwidth_allocation_ > DataRate::Zero())
146 ? absl::optional<uint32_t>(bandwidth_allocation_.bps())
147 : absl::nullopt;
148 }
149
last_stream_configuration_changed() const150 bool last_stream_configuration_changed() const {
151 return last_stream_configuration_changed_;
152 }
153
154 // Input signals.
SetStartBitrate(DataRate start_bitrate,int64_t now_ms)155 void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) {
156 set_start_bitrate_ = start_bitrate;
157 set_start_bitrate_time_ms_ = now_ms;
158 }
159
SetBandwidthAllocation(DataRate bandwidth_allocation)160 void SetBandwidthAllocation(DataRate bandwidth_allocation) {
161 bandwidth_allocation_ = bandwidth_allocation;
162 }
163
SetTargetBitrate(DataRate target_bitrate,int64_t now_ms)164 void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) {
165 if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ &&
166 quality_scaler_resource_->is_started() &&
167 quality_scaler_settings_.InitialBitrateIntervalMs() &&
168 quality_scaler_settings_.InitialBitrateFactor()) {
169 int64_t diff_ms = now_ms - set_start_bitrate_time_ms_;
170 if (diff_ms <
171 quality_scaler_settings_.InitialBitrateIntervalMs().value() &&
172 (target_bitrate <
173 (set_start_bitrate_ *
174 quality_scaler_settings_.InitialBitrateFactor().value()))) {
175 RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: "
176 << set_start_bitrate_.bps()
177 << ", target bitrate: " << target_bitrate.bps();
178 initial_framedrop_ = 0;
179 has_seen_first_bwe_drop_ = true;
180 }
181 }
182 }
183
OnEncoderSettingsUpdated(const VideoCodec & codec,const VideoAdaptationCounters & adaptation_counters)184 void OnEncoderSettingsUpdated(
185 const VideoCodec& codec,
186 const VideoAdaptationCounters& adaptation_counters) {
187 last_stream_configuration_changed_ = false;
188 std::vector<bool> active_flags = GetActiveLayersFlags(codec);
189 // Check if the source resolution has changed for the external reasons,
190 // i.e. without any adaptation from WebRTC.
191 const bool source_resolution_changed =
192 (last_input_width_ != codec.width ||
193 last_input_height_ != codec.height) &&
194 adaptation_counters.resolution_adaptations ==
195 last_adaptation_counters_.resolution_adaptations;
196 if (!EqualFlags(active_flags, last_active_flags_) ||
197 source_resolution_changed) {
198 // Streams configuration has changed.
199 last_stream_configuration_changed_ = true;
200 // Initial frame drop must be enabled because BWE might be way too low
201 // for the selected resolution.
202 if (quality_scaler_resource_->is_started()) {
203 RTC_LOG(LS_INFO) << "Resetting initial_framedrop_ due to changed "
204 "stream parameters";
205 initial_framedrop_ = 0;
206 if (single_active_stream_pixels_ &&
207 VideoStreamAdapter::GetSingleActiveLayerPixels(codec) >
208 *single_active_stream_pixels_) {
209 // Resolution increased.
210 use_bandwidth_allocation_ = true;
211 }
212 }
213 }
214 last_adaptation_counters_ = adaptation_counters;
215 last_active_flags_ = active_flags;
216 last_input_width_ = codec.width;
217 last_input_height_ = codec.height;
218 single_active_stream_pixels_ =
219 VideoStreamAdapter::GetSingleActiveLayerPixels(codec);
220 }
221
OnFrameDroppedDueToSize()222 void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
223
Disable()224 void Disable() {
225 initial_framedrop_ = kMaxInitialFramedrop;
226 use_bandwidth_allocation_ = false;
227 }
228
OnQualityScalerSettingsUpdated()229 void OnQualityScalerSettingsUpdated() {
230 if (quality_scaler_resource_->is_started()) {
231 // Restart frame drops due to size.
232 initial_framedrop_ = 0;
233 } else {
234 // Quality scaling disabled so we shouldn't drop initial frames.
235 Disable();
236 }
237 }
238
239 private:
240 // The maximum number of frames to drop at beginning of stream to try and
241 // achieve desired bitrate.
242 static const int kMaxInitialFramedrop = 4;
243
244 const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
245 const QualityScalerSettings quality_scaler_settings_;
246 bool has_seen_first_bwe_drop_;
247 DataRate set_start_bitrate_;
248 int64_t set_start_bitrate_time_ms_;
249 // Counts how many frames we've dropped in the initial framedrop phase.
250 int initial_framedrop_;
251 absl::optional<uint32_t> single_active_stream_pixels_;
252 bool use_bandwidth_allocation_;
253 DataRate bandwidth_allocation_;
254
255 std::vector<bool> last_active_flags_;
256 VideoAdaptationCounters last_adaptation_counters_;
257 int last_input_width_;
258 int last_input_height_;
259 bool last_stream_configuration_changed_;
260 };
261
VideoStreamEncoderResourceManager(VideoStreamInputStateProvider * input_state_provider,VideoStreamEncoderObserver * encoder_stats_observer,Clock * clock,bool experiment_cpu_load_estimator,std::unique_ptr<OveruseFrameDetector> overuse_detector,DegradationPreferenceProvider * degradation_preference_provider,const FieldTrialsView & field_trials)262 VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
263 VideoStreamInputStateProvider* input_state_provider,
264 VideoStreamEncoderObserver* encoder_stats_observer,
265 Clock* clock,
266 bool experiment_cpu_load_estimator,
267 std::unique_ptr<OveruseFrameDetector> overuse_detector,
268 DegradationPreferenceProvider* degradation_preference_provider,
269 const FieldTrialsView& field_trials)
270 : field_trials_(field_trials),
271 degradation_preference_provider_(degradation_preference_provider),
272 bitrate_constraint_(std::make_unique<BitrateConstraint>()),
273 balanced_constraint_(
274 std::make_unique<BalancedConstraint>(degradation_preference_provider_,
275 field_trials)),
276 encode_usage_resource_(
277 EncodeUsageResource::Create(std::move(overuse_detector))),
278 quality_scaler_resource_(QualityScalerResource::Create()),
279 pixel_limit_resource_(nullptr),
280 bandwidth_quality_scaler_resource_(
281 BandwidthQualityScalerResource::Create()),
282 encoder_queue_(nullptr),
283 input_state_provider_(input_state_provider),
284 adaptation_processor_(nullptr),
285 encoder_stats_observer_(encoder_stats_observer),
286 degradation_preference_(DegradationPreference::DISABLED),
287 video_source_restrictions_(),
288 balanced_settings_(field_trials),
289 clock_(clock),
290 experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
291 initial_frame_dropper_(
292 std::make_unique<InitialFrameDropper>(quality_scaler_resource_)),
293 quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
294 pixel_limit_resource_experiment_enabled_(
295 field_trials.IsEnabled(kPixelLimitResourceFieldTrialName)),
296 encoder_target_bitrate_bps_(absl::nullopt),
297 quality_rampup_experiment_(
298 QualityRampUpExperimentHelper::CreateIfEnabled(this, clock_)),
299 encoder_settings_(absl::nullopt) {
300 TRACE_EVENT0(
301 "webrtc",
302 "VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager");
303 RTC_CHECK(degradation_preference_provider_);
304 RTC_CHECK(encoder_stats_observer_);
305 }
306
307 VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() =
308 default;
309
Initialize(TaskQueueBase * encoder_queue)310 void VideoStreamEncoderResourceManager::Initialize(
311 TaskQueueBase* encoder_queue) {
312 RTC_DCHECK(!encoder_queue_);
313 RTC_DCHECK(encoder_queue);
314 encoder_queue_ = encoder_queue;
315 encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_);
316 quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_);
317 bandwidth_quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_);
318 }
319
SetAdaptationProcessor(ResourceAdaptationProcessorInterface * adaptation_processor,VideoStreamAdapter * stream_adapter)320 void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
321 ResourceAdaptationProcessorInterface* adaptation_processor,
322 VideoStreamAdapter* stream_adapter) {
323 RTC_DCHECK_RUN_ON(encoder_queue_);
324 adaptation_processor_ = adaptation_processor;
325 stream_adapter_ = stream_adapter;
326 }
327
SetDegradationPreferences(DegradationPreference degradation_preference)328 void VideoStreamEncoderResourceManager::SetDegradationPreferences(
329 DegradationPreference degradation_preference) {
330 RTC_DCHECK_RUN_ON(encoder_queue_);
331 degradation_preference_ = degradation_preference;
332 UpdateStatsAdaptationSettings();
333 }
334
335 DegradationPreference
degradation_preference() const336 VideoStreamEncoderResourceManager::degradation_preference() const {
337 RTC_DCHECK_RUN_ON(encoder_queue_);
338 return degradation_preference_;
339 }
340
ConfigureEncodeUsageResource()341 void VideoStreamEncoderResourceManager::ConfigureEncodeUsageResource() {
342 RTC_DCHECK_RUN_ON(encoder_queue_);
343 RTC_DCHECK(encoder_settings_.has_value());
344 if (encode_usage_resource_->is_started()) {
345 encode_usage_resource_->StopCheckForOveruse();
346 } else {
347 // If the resource has not yet started then it needs to be added.
348 AddResource(encode_usage_resource_, VideoAdaptationReason::kCpu);
349 }
350 encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions());
351 }
352
MaybeInitializePixelLimitResource()353 void VideoStreamEncoderResourceManager::MaybeInitializePixelLimitResource() {
354 RTC_DCHECK_RUN_ON(encoder_queue_);
355 RTC_DCHECK(adaptation_processor_);
356 RTC_DCHECK(!pixel_limit_resource_);
357 if (!pixel_limit_resource_experiment_enabled_) {
358 // The field trial is not running.
359 return;
360 }
361 int max_pixels = 0;
362 std::string pixel_limit_field_trial =
363 field_trials_.Lookup(kPixelLimitResourceFieldTrialName);
364 if (sscanf(pixel_limit_field_trial.c_str(), "Enabled-%d", &max_pixels) != 1) {
365 RTC_LOG(LS_ERROR) << "Couldn't parse " << kPixelLimitResourceFieldTrialName
366 << " trial config: " << pixel_limit_field_trial;
367 return;
368 }
369 RTC_LOG(LS_INFO) << "Running field trial "
370 << kPixelLimitResourceFieldTrialName << " configured to "
371 << max_pixels << " max pixels";
372 // Configure the specified max pixels from the field trial. The pixel limit
373 // resource is active for the lifetme of the stream (until
374 // StopManagedResources() is called).
375 pixel_limit_resource_ =
376 PixelLimitResource::Create(encoder_queue_, input_state_provider_);
377 pixel_limit_resource_->SetMaxPixels(max_pixels);
378 AddResource(pixel_limit_resource_, VideoAdaptationReason::kCpu);
379 }
380
StopManagedResources()381 void VideoStreamEncoderResourceManager::StopManagedResources() {
382 RTC_DCHECK_RUN_ON(encoder_queue_);
383 RTC_DCHECK(adaptation_processor_);
384 if (encode_usage_resource_->is_started()) {
385 encode_usage_resource_->StopCheckForOveruse();
386 RemoveResource(encode_usage_resource_);
387 }
388 if (quality_scaler_resource_->is_started()) {
389 quality_scaler_resource_->StopCheckForOveruse();
390 RemoveResource(quality_scaler_resource_);
391 }
392 if (pixel_limit_resource_) {
393 RemoveResource(pixel_limit_resource_);
394 pixel_limit_resource_ = nullptr;
395 }
396 if (bandwidth_quality_scaler_resource_->is_started()) {
397 bandwidth_quality_scaler_resource_->StopCheckForOveruse();
398 RemoveResource(bandwidth_quality_scaler_resource_);
399 }
400 }
401
AddResource(rtc::scoped_refptr<Resource> resource,VideoAdaptationReason reason)402 void VideoStreamEncoderResourceManager::AddResource(
403 rtc::scoped_refptr<Resource> resource,
404 VideoAdaptationReason reason) {
405 RTC_DCHECK_RUN_ON(encoder_queue_);
406 RTC_DCHECK(resource);
407 bool inserted;
408 std::tie(std::ignore, inserted) = resources_.emplace(resource, reason);
409 RTC_DCHECK(inserted) << "Resource " << resource->Name()
410 << " already was inserted";
411 adaptation_processor_->AddResource(resource);
412 }
413
RemoveResource(rtc::scoped_refptr<Resource> resource)414 void VideoStreamEncoderResourceManager::RemoveResource(
415 rtc::scoped_refptr<Resource> resource) {
416 {
417 RTC_DCHECK_RUN_ON(encoder_queue_);
418 RTC_DCHECK(resource);
419 const auto& it = resources_.find(resource);
420 RTC_DCHECK(it != resources_.end())
421 << "Resource \"" << resource->Name() << "\" not found.";
422 resources_.erase(it);
423 }
424 adaptation_processor_->RemoveResource(resource);
425 }
426
427 std::vector<AdaptationConstraint*>
AdaptationConstraints() const428 VideoStreamEncoderResourceManager::AdaptationConstraints() const {
429 RTC_DCHECK_RUN_ON(encoder_queue_);
430 return {bitrate_constraint_.get(), balanced_constraint_.get()};
431 }
432
SetEncoderSettings(EncoderSettings encoder_settings)433 void VideoStreamEncoderResourceManager::SetEncoderSettings(
434 EncoderSettings encoder_settings) {
435 RTC_DCHECK_RUN_ON(encoder_queue_);
436 encoder_settings_ = std::move(encoder_settings);
437 bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_);
438 initial_frame_dropper_->OnEncoderSettingsUpdated(
439 encoder_settings_->video_codec(), current_adaptation_counters_);
440 MaybeUpdateTargetFrameRate();
441 if (quality_rampup_experiment_) {
442 quality_rampup_experiment_->ConfigureQualityRampupExperiment(
443 initial_frame_dropper_->last_stream_configuration_changed(),
444 initial_frame_dropper_->single_active_stream_pixels(),
445 GetSingleActiveLayerMaxBitrate(encoder_settings_->video_codec()));
446 }
447 }
448
SetStartBitrate(DataRate start_bitrate)449 void VideoStreamEncoderResourceManager::SetStartBitrate(
450 DataRate start_bitrate) {
451 RTC_DCHECK_RUN_ON(encoder_queue_);
452 if (!start_bitrate.IsZero()) {
453 encoder_target_bitrate_bps_ = start_bitrate.bps();
454 bitrate_constraint_->OnEncoderTargetBitrateUpdated(
455 encoder_target_bitrate_bps_);
456 balanced_constraint_->OnEncoderTargetBitrateUpdated(
457 encoder_target_bitrate_bps_);
458 }
459 initial_frame_dropper_->SetStartBitrate(start_bitrate,
460 clock_->TimeInMicroseconds());
461 }
462
SetTargetBitrate(DataRate target_bitrate)463 void VideoStreamEncoderResourceManager::SetTargetBitrate(
464 DataRate target_bitrate) {
465 RTC_DCHECK_RUN_ON(encoder_queue_);
466 if (!target_bitrate.IsZero()) {
467 encoder_target_bitrate_bps_ = target_bitrate.bps();
468 bitrate_constraint_->OnEncoderTargetBitrateUpdated(
469 encoder_target_bitrate_bps_);
470 balanced_constraint_->OnEncoderTargetBitrateUpdated(
471 encoder_target_bitrate_bps_);
472 }
473 initial_frame_dropper_->SetTargetBitrate(target_bitrate,
474 clock_->TimeInMilliseconds());
475 }
476
SetEncoderRates(const VideoEncoder::RateControlParameters & encoder_rates)477 void VideoStreamEncoderResourceManager::SetEncoderRates(
478 const VideoEncoder::RateControlParameters& encoder_rates) {
479 RTC_DCHECK_RUN_ON(encoder_queue_);
480 encoder_rates_ = encoder_rates;
481 initial_frame_dropper_->SetBandwidthAllocation(
482 encoder_rates.bandwidth_allocation);
483 }
484
OnFrameDroppedDueToSize()485 void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() {
486 RTC_DCHECK_RUN_ON(encoder_queue_);
487 initial_frame_dropper_->OnFrameDroppedDueToSize();
488 Adaptation reduce_resolution = stream_adapter_->GetAdaptDownResolution();
489 if (reduce_resolution.status() == Adaptation::Status::kValid) {
490 stream_adapter_->ApplyAdaptation(reduce_resolution,
491 quality_scaler_resource_);
492 }
493 }
494
OnEncodeStarted(const VideoFrame & cropped_frame,int64_t time_when_first_seen_us)495 void VideoStreamEncoderResourceManager::OnEncodeStarted(
496 const VideoFrame& cropped_frame,
497 int64_t time_when_first_seen_us) {
498 RTC_DCHECK_RUN_ON(encoder_queue_);
499 encode_usage_resource_->OnEncodeStarted(cropped_frame,
500 time_when_first_seen_us);
501 }
502
OnEncodeCompleted(const EncodedImage & encoded_image,int64_t time_sent_in_us,absl::optional<int> encode_duration_us,DataSize frame_size)503 void VideoStreamEncoderResourceManager::OnEncodeCompleted(
504 const EncodedImage& encoded_image,
505 int64_t time_sent_in_us,
506 absl::optional<int> encode_duration_us,
507 DataSize frame_size) {
508 RTC_DCHECK_RUN_ON(encoder_queue_);
509 // Inform `encode_usage_resource_` of the encode completed event.
510 uint32_t timestamp = encoded_image.Timestamp();
511 int64_t capture_time_us =
512 encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
513 encode_usage_resource_->OnEncodeCompleted(
514 timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
515 quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
516 bandwidth_quality_scaler_resource_->OnEncodeCompleted(
517 encoded_image, time_sent_in_us, frame_size.bytes());
518 }
519
OnFrameDropped(EncodedImageCallback::DropReason reason)520 void VideoStreamEncoderResourceManager::OnFrameDropped(
521 EncodedImageCallback::DropReason reason) {
522 RTC_DCHECK_RUN_ON(encoder_queue_);
523 quality_scaler_resource_->OnFrameDropped(reason);
524 }
525
DropInitialFrames() const526 bool VideoStreamEncoderResourceManager::DropInitialFrames() const {
527 RTC_DCHECK_RUN_ON(encoder_queue_);
528 return initial_frame_dropper_->DropInitialFrames();
529 }
530
531 absl::optional<uint32_t>
SingleActiveStreamPixels() const532 VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const {
533 RTC_DCHECK_RUN_ON(encoder_queue_);
534 return initial_frame_dropper_->single_active_stream_pixels();
535 }
536
537 absl::optional<uint32_t>
UseBandwidthAllocationBps() const538 VideoStreamEncoderResourceManager::UseBandwidthAllocationBps() const {
539 RTC_DCHECK_RUN_ON(encoder_queue_);
540 return initial_frame_dropper_->UseBandwidthAllocationBps();
541 }
542
OnMaybeEncodeFrame()543 void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
544 RTC_DCHECK_RUN_ON(encoder_queue_);
545 initial_frame_dropper_->Disable();
546 if (quality_rampup_experiment_ && quality_scaler_resource_->is_started()) {
547 DataRate bandwidth = encoder_rates_.has_value()
548 ? encoder_rates_->bandwidth_allocation
549 : DataRate::Zero();
550 quality_rampup_experiment_->PerformQualityRampupExperiment(
551 quality_scaler_resource_, bandwidth,
552 DataRate::BitsPerSec(encoder_target_bitrate_bps_.value_or(0)),
553 GetSingleActiveLayerMaxBitrate(encoder_settings_->video_codec()));
554 }
555 }
556
UpdateQualityScalerSettings(absl::optional<VideoEncoder::QpThresholds> qp_thresholds)557 void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings(
558 absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
559 RTC_DCHECK_RUN_ON(encoder_queue_);
560 if (qp_thresholds.has_value()) {
561 if (quality_scaler_resource_->is_started()) {
562 quality_scaler_resource_->SetQpThresholds(qp_thresholds.value());
563 } else {
564 quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value());
565 AddResource(quality_scaler_resource_, VideoAdaptationReason::kQuality);
566 }
567 } else if (quality_scaler_resource_->is_started()) {
568 quality_scaler_resource_->StopCheckForOveruse();
569 RemoveResource(quality_scaler_resource_);
570 }
571 initial_frame_dropper_->OnQualityScalerSettingsUpdated();
572 }
573
UpdateBandwidthQualityScalerSettings(bool bandwidth_quality_scaling_allowed,const std::vector<VideoEncoder::ResolutionBitrateLimits> & resolution_bitrate_limits)574 void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings(
575 bool bandwidth_quality_scaling_allowed,
576 const std::vector<VideoEncoder::ResolutionBitrateLimits>&
577 resolution_bitrate_limits) {
578 RTC_DCHECK_RUN_ON(encoder_queue_);
579
580 if (!bandwidth_quality_scaling_allowed) {
581 if (bandwidth_quality_scaler_resource_->is_started()) {
582 bandwidth_quality_scaler_resource_->StopCheckForOveruse();
583 RemoveResource(bandwidth_quality_scaler_resource_);
584 }
585 } else {
586 if (!bandwidth_quality_scaler_resource_->is_started()) {
587 // Before executing "StartCheckForOveruse",we must execute "AddResource"
588 // firstly,because it can make the listener valid.
589 AddResource(bandwidth_quality_scaler_resource_,
590 webrtc::VideoAdaptationReason::kQuality);
591 bandwidth_quality_scaler_resource_->StartCheckForOveruse(
592 resolution_bitrate_limits);
593 }
594 }
595 }
596
ConfigureQualityScaler(const VideoEncoder::EncoderInfo & encoder_info)597 void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
598 const VideoEncoder::EncoderInfo& encoder_info) {
599 RTC_DCHECK_RUN_ON(encoder_queue_);
600 const auto scaling_settings = encoder_info.scaling_settings;
601 const bool quality_scaling_allowed =
602 IsResolutionScalingEnabled(degradation_preference_) &&
603 (scaling_settings.thresholds.has_value() ||
604 (encoder_settings_.has_value() &&
605 encoder_settings_->encoder_config().is_quality_scaling_allowed)) &&
606 encoder_info.is_qp_trusted.value_or(true);
607
608 // TODO(https://crbug.com/webrtc/11222): Should this move to
609 // QualityScalerResource?
610 if (quality_scaling_allowed) {
611 if (!quality_scaler_resource_->is_started()) {
612 // Quality scaler has not already been configured.
613
614 // Use experimental thresholds if available.
615 absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
616 if (quality_scaling_experiment_enabled_) {
617 experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
618 GetVideoCodecTypeOrGeneric(encoder_settings_));
619 }
620 UpdateQualityScalerSettings(experimental_thresholds.has_value()
621 ? experimental_thresholds
622 : scaling_settings.thresholds);
623 }
624 } else {
625 UpdateQualityScalerSettings(absl::nullopt);
626 }
627
628 // Set the qp-thresholds to the balanced settings if balanced mode.
629 if (degradation_preference_ == DegradationPreference::BALANCED &&
630 quality_scaler_resource_->is_started()) {
631 absl::optional<VideoEncoder::QpThresholds> thresholds =
632 balanced_settings_.GetQpThresholds(
633 GetVideoCodecTypeOrGeneric(encoder_settings_),
634 LastFrameSizeOrDefault());
635 if (thresholds) {
636 quality_scaler_resource_->SetQpThresholds(*thresholds);
637 }
638 }
639 UpdateStatsAdaptationSettings();
640 }
641
ConfigureBandwidthQualityScaler(const VideoEncoder::EncoderInfo & encoder_info)642 void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler(
643 const VideoEncoder::EncoderInfo& encoder_info) {
644 RTC_DCHECK_RUN_ON(encoder_queue_);
645 const bool bandwidth_quality_scaling_allowed =
646 IsResolutionScalingEnabled(degradation_preference_) &&
647 (encoder_settings_.has_value() &&
648 encoder_settings_->encoder_config().is_quality_scaling_allowed) &&
649 !encoder_info.is_qp_trusted.value_or(true);
650
651 UpdateBandwidthQualityScalerSettings(bandwidth_quality_scaling_allowed,
652 encoder_info.resolution_bitrate_limits);
653 UpdateStatsAdaptationSettings();
654 }
655
GetReasonFromResource(rtc::scoped_refptr<Resource> resource) const656 VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource(
657 rtc::scoped_refptr<Resource> resource) const {
658 RTC_DCHECK_RUN_ON(encoder_queue_);
659 const auto& registered_resource = resources_.find(resource);
660 RTC_DCHECK(registered_resource != resources_.end())
661 << resource->Name() << " not found.";
662 return registered_resource->second;
663 }
664
665 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
666 // pipelining encoders better (multiple input frames before something comes
667 // out). This should effectively turn off CPU adaptations for systems that
668 // remotely cope with the load right now.
GetCpuOveruseOptions() const669 CpuOveruseOptions VideoStreamEncoderResourceManager::GetCpuOveruseOptions()
670 const {
671 RTC_DCHECK_RUN_ON(encoder_queue_);
672 // This is already ensured by the only caller of this method:
673 // StartResourceAdaptation().
674 RTC_DCHECK(encoder_settings_.has_value());
675 CpuOveruseOptions options(field_trials_);
676 // Hardware accelerated encoders are assumed to be pipelined; give them
677 // additional overuse time.
678 if (encoder_settings_->encoder_info().is_hardware_accelerated) {
679 options.low_encode_usage_threshold_percent = 150;
680 options.high_encode_usage_threshold_percent = 200;
681 }
682 if (experiment_cpu_load_estimator_) {
683 options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec;
684 }
685 return options;
686 }
687
LastFrameSizeOrDefault() const688 int VideoStreamEncoderResourceManager::LastFrameSizeOrDefault() const {
689 RTC_DCHECK_RUN_ON(encoder_queue_);
690 return input_state_provider_->InputState()
691 .single_active_stream_pixels()
692 .value_or(
693 input_state_provider_->InputState().frame_size_pixels().value_or(
694 kDefaultInputPixelsWidth * kDefaultInputPixelsHeight));
695 }
696
OnVideoSourceRestrictionsUpdated(VideoSourceRestrictions restrictions,const VideoAdaptationCounters & adaptation_counters,rtc::scoped_refptr<Resource> reason,const VideoSourceRestrictions & unfiltered_restrictions)697 void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
698 VideoSourceRestrictions restrictions,
699 const VideoAdaptationCounters& adaptation_counters,
700 rtc::scoped_refptr<Resource> reason,
701 const VideoSourceRestrictions& unfiltered_restrictions) {
702 RTC_DCHECK_RUN_ON(encoder_queue_);
703 current_adaptation_counters_ = adaptation_counters;
704
705 // TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback.
706 if (!reason && adaptation_counters.Total() == 0) {
707 // Adaptation was manually reset - clear the per-reason counters too.
708 encoder_stats_observer_->ClearAdaptationStats();
709 }
710
711 video_source_restrictions_ = FilterRestrictionsByDegradationPreference(
712 restrictions, degradation_preference_);
713 MaybeUpdateTargetFrameRate();
714 }
715
OnResourceLimitationChanged(rtc::scoped_refptr<Resource> resource,const std::map<rtc::scoped_refptr<Resource>,VideoAdaptationCounters> & resource_limitations)716 void VideoStreamEncoderResourceManager::OnResourceLimitationChanged(
717 rtc::scoped_refptr<Resource> resource,
718 const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
719 resource_limitations) {
720 RTC_DCHECK_RUN_ON(encoder_queue_);
721 if (!resource) {
722 encoder_stats_observer_->ClearAdaptationStats();
723 return;
724 }
725
726 std::map<VideoAdaptationReason, VideoAdaptationCounters> limitations;
727 for (auto& resource_counter : resource_limitations) {
728 std::map<VideoAdaptationReason, VideoAdaptationCounters>::iterator it;
729 bool inserted;
730 std::tie(it, inserted) = limitations.emplace(
731 GetReasonFromResource(resource_counter.first), resource_counter.second);
732 if (!inserted && it->second.Total() < resource_counter.second.Total()) {
733 it->second = resource_counter.second;
734 }
735 }
736
737 VideoAdaptationReason adaptation_reason = GetReasonFromResource(resource);
738 encoder_stats_observer_->OnAdaptationChanged(
739 adaptation_reason, limitations[VideoAdaptationReason::kCpu],
740 limitations[VideoAdaptationReason::kQuality]);
741
742 if (quality_rampup_experiment_) {
743 bool cpu_limited = limitations.at(VideoAdaptationReason::kCpu).Total() > 0;
744 auto qp_resolution_adaptations =
745 limitations.at(VideoAdaptationReason::kQuality).resolution_adaptations;
746 quality_rampup_experiment_->cpu_adapted(cpu_limited);
747 quality_rampup_experiment_->qp_resolution_adaptations(
748 qp_resolution_adaptations);
749 }
750
751 RTC_LOG(LS_INFO) << ActiveCountsToString(limitations);
752 }
753
MaybeUpdateTargetFrameRate()754 void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() {
755 RTC_DCHECK_RUN_ON(encoder_queue_);
756 absl::optional<double> codec_max_frame_rate =
757 encoder_settings_.has_value()
758 ? absl::optional<double>(
759 encoder_settings_->video_codec().maxFramerate)
760 : absl::nullopt;
761 // The current target framerate is the maximum frame rate as specified by
762 // the current codec configuration or any limit imposed by the adaptation
763 // module. This is used to make sure overuse detection doesn't needlessly
764 // trigger in low and/or variable framerate scenarios.
765 absl::optional<double> target_frame_rate =
766 video_source_restrictions_.max_frame_rate();
767 if (!target_frame_rate.has_value() ||
768 (codec_max_frame_rate.has_value() &&
769 codec_max_frame_rate.value() < target_frame_rate.value())) {
770 target_frame_rate = codec_max_frame_rate;
771 }
772 encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
773 }
774
UpdateStatsAdaptationSettings() const775 void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const {
776 RTC_DCHECK_RUN_ON(encoder_queue_);
777 VideoStreamEncoderObserver::AdaptationSettings cpu_settings(
778 IsResolutionScalingEnabled(degradation_preference_),
779 IsFramerateScalingEnabled(degradation_preference_));
780
781 VideoStreamEncoderObserver::AdaptationSettings quality_settings =
782 (quality_scaler_resource_->is_started() ||
783 bandwidth_quality_scaler_resource_->is_started())
784 ? cpu_settings
785 : VideoStreamEncoderObserver::AdaptationSettings();
786 encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings,
787 quality_settings);
788 }
789
790 // static
ActiveCountsToString(const std::map<VideoAdaptationReason,VideoAdaptationCounters> & active_counts)791 std::string VideoStreamEncoderResourceManager::ActiveCountsToString(
792 const std::map<VideoAdaptationReason, VideoAdaptationCounters>&
793 active_counts) {
794 rtc::StringBuilder ss;
795
796 ss << "Downgrade counts: fps: {";
797 for (auto& reason_count : active_counts) {
798 ss << ToString(reason_count.first) << ":";
799 ss << reason_count.second.fps_adaptations;
800 }
801 ss << "}, resolution {";
802 for (auto& reason_count : active_counts) {
803 ss << ToString(reason_count.first) << ":";
804 ss << reason_count.second.resolution_adaptations;
805 }
806 ss << "}";
807
808 return ss.Release();
809 }
810
OnQualityRampUp()811 void VideoStreamEncoderResourceManager::OnQualityRampUp() {
812 RTC_DCHECK_RUN_ON(encoder_queue_);
813 stream_adapter_->ClearRestrictions();
814 quality_rampup_experiment_.reset();
815 }
816
IsSimulcastOrMultipleSpatialLayers(const VideoEncoderConfig & encoder_config)817 bool VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers(
818 const VideoEncoderConfig& encoder_config) {
819 const std::vector<VideoStream>& simulcast_layers =
820 encoder_config.simulcast_layers;
821 if (simulcast_layers.empty()) {
822 return false;
823 }
824
825 absl::optional<int> num_spatial_layers;
826 if (simulcast_layers[0].scalability_mode.has_value() &&
827 encoder_config.number_of_streams == 1) {
828 num_spatial_layers = ScalabilityModeToNumSpatialLayers(
829 *simulcast_layers[0].scalability_mode);
830 }
831
832 if (simulcast_layers.size() == 1) {
833 // Check if multiple spatial layers are used.
834 return num_spatial_layers && *num_spatial_layers > 1;
835 }
836
837 bool svc_with_one_spatial_layer =
838 num_spatial_layers && *num_spatial_layers == 1;
839 if (simulcast_layers[0].active && !svc_with_one_spatial_layer) {
840 // We can't distinguish between simulcast and singlecast when only the
841 // lowest spatial layer is active. Treat this case as simulcast.
842 return true;
843 }
844
845 int num_active_layers =
846 std::count_if(simulcast_layers.begin(), simulcast_layers.end(),
847 [](const VideoStream& layer) { return layer.active; });
848 return num_active_layers > 1;
849 }
850
851 } // namespace webrtc
852