1 /* 2 * Copyright 2020 The WebRTC Project Authors. All rights reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ 12 #define VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ 13 14 #include <atomic> 15 #include <map> 16 #include <memory> 17 #include <string> 18 #include <unordered_map> 19 #include <utility> 20 #include <vector> 21 22 #include "absl/types/optional.h" 23 #include "api/adaptation/resource.h" 24 #include "api/rtp_parameters.h" 25 #include "api/scoped_refptr.h" 26 #include "api/task_queue/task_queue_base.h" 27 #include "api/video/video_adaptation_counters.h" 28 #include "api/video/video_adaptation_reason.h" 29 #include "api/video/video_frame.h" 30 #include "api/video/video_source_interface.h" 31 #include "api/video/video_stream_encoder_observer.h" 32 #include "api/video_codecs/video_codec.h" 33 #include "api/video_codecs/video_encoder.h" 34 #include "api/video_codecs/video_encoder_config.h" 35 #include "call/adaptation/resource_adaptation_processor_interface.h" 36 #include "call/adaptation/video_stream_adapter.h" 37 #include "call/adaptation/video_stream_input_state_provider.h" 38 #include "rtc_base/experiments/quality_scaler_settings.h" 39 #include "rtc_base/ref_count.h" 40 #include "rtc_base/strings/string_builder.h" 41 #include "rtc_base/synchronization/mutex.h" 42 #include "rtc_base/task_queue.h" 43 #include "system_wrappers/include/clock.h" 44 #include "video/adaptation/encode_usage_resource.h" 45 #include "video/adaptation/overuse_frame_detector.h" 46 #include "video/adaptation/quality_rampup_experiment_helper.h" 47 #include "video/adaptation/quality_scaler_resource.h" 48 #include "video/adaptation/video_stream_encoder_resource.h" 49 50 namespace webrtc { 51 52 // The assumed input frame size if we have not yet received a frame. 53 // TODO(hbos): This is 144p - why are we assuming super low quality? Seems like 54 // a bad heuristic. 55 extern const int kDefaultInputPixelsWidth; 56 extern const int kDefaultInputPixelsHeight; 57 58 // Owns adaptation-related Resources pertaining to a single VideoStreamEncoder 59 // and passes on the relevant input from the encoder to the resources. The 60 // resources provide resource usage states to the ResourceAdaptationProcessor 61 // which is responsible for reconfiguring streams in order not to overuse 62 // resources. 63 // 64 // The manager is also involved with various mitigations not part of the 65 // ResourceAdaptationProcessor code such as the inital frame dropping. 66 class VideoStreamEncoderResourceManager 67 : public VideoSourceRestrictionsListener, 68 public ResourceLimitationsListener, 69 public QualityRampUpExperimentListener { 70 public: 71 VideoStreamEncoderResourceManager( 72 VideoStreamInputStateProvider* input_state_provider, 73 VideoStreamEncoderObserver* encoder_stats_observer, 74 Clock* clock, 75 bool experiment_cpu_load_estimator, 76 std::unique_ptr<OveruseFrameDetector> overuse_detector, 77 DegradationPreferenceProvider* degradation_preference_provider); 78 ~VideoStreamEncoderResourceManager() override; 79 80 void Initialize(rtc::TaskQueue* encoder_queue, 81 rtc::TaskQueue* resource_adaptation_queue); 82 void SetAdaptationProcessor( 83 ResourceAdaptationProcessorInterface* adaptation_processor, 84 VideoStreamAdapter* stream_adapter); 85 86 // TODO(https://crbug.com/webrtc/11563): The degradation preference is a 87 // setting of the Processor, it does not belong to the Manager - can we get 88 // rid of this? 89 void SetDegradationPreferences(DegradationPreference degradation_preference); 90 DegradationPreference degradation_preference() const; 91 92 // Starts the encode usage resource. The quality scaler resource is 93 // automatically started on being configured. 94 void StartEncodeUsageResource(); 95 // Stops the encode usage and quality scaler resources if not already stopped. 96 void StopManagedResources(); 97 98 // Settings that affect the VideoStreamEncoder-specific resources. 99 void SetEncoderSettings(EncoderSettings encoder_settings); 100 void SetStartBitrate(DataRate start_bitrate); 101 void SetTargetBitrate(DataRate target_bitrate); 102 void SetEncoderRates( 103 const VideoEncoder::RateControlParameters& encoder_rates); 104 // TODO(https://crbug.com/webrtc/11338): This can be made private if we 105 // configure on SetDegredationPreference and SetEncoderSettings. 106 void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); 107 108 // Methods corresponding to different points in the encoding pipeline. 109 void OnFrameDroppedDueToSize(); 110 void OnMaybeEncodeFrame(); 111 void OnEncodeStarted(const VideoFrame& cropped_frame, 112 int64_t time_when_first_seen_us); 113 void OnEncodeCompleted(const EncodedImage& encoded_image, 114 int64_t time_sent_in_us, 115 absl::optional<int> encode_duration_us); 116 void OnFrameDropped(EncodedImageCallback::DropReason reason); 117 118 // Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order 119 // to update legacy getStats(). 120 void MapResourceToReason(rtc::scoped_refptr<Resource> resource, 121 VideoAdaptationReason reason); 122 std::vector<rtc::scoped_refptr<Resource>> MappedResources() const; 123 std::vector<AdaptationConstraint*> AdaptationConstraints() const; 124 std::vector<AdaptationListener*> AdaptationListeners() const; 125 rtc::scoped_refptr<QualityScalerResource> 126 quality_scaler_resource_for_testing(); 127 // If true, the VideoStreamEncoder should eexecute its logic to maybe drop 128 // frames baseed on size and bitrate. 129 bool DropInitialFrames() const; 130 131 // VideoSourceRestrictionsListener implementation. 132 // Updates |video_source_restrictions_|. 133 void OnVideoSourceRestrictionsUpdated( 134 VideoSourceRestrictions restrictions, 135 const VideoAdaptationCounters& adaptation_counters, 136 rtc::scoped_refptr<Resource> reason, 137 const VideoSourceRestrictions& unfiltered_restrictions) override; 138 void OnResourceLimitationChanged( 139 rtc::scoped_refptr<Resource> resource, 140 const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>& 141 resource_limitations) override; 142 143 // QualityRampUpExperimentListener implementation. 144 void OnQualityRampUp() override; 145 146 private: 147 class InitialFrameDropper; 148 149 VideoAdaptationReason GetReasonFromResource( 150 rtc::scoped_refptr<Resource> resource) const; 151 152 CpuOveruseOptions GetCpuOveruseOptions() const; 153 int LastInputFrameSizeOrDefault() const; 154 155 // Calculates an up-to-date value of the target frame rate and informs the 156 // |encode_usage_resource_| of the new value. 157 void MaybeUpdateTargetFrameRate(); 158 159 // Use nullopt to disable quality scaling. 160 void UpdateQualityScalerSettings( 161 absl::optional<VideoEncoder::QpThresholds> qp_thresholds); 162 163 void UpdateStatsAdaptationSettings() const; 164 165 static std::string ActiveCountsToString( 166 const std::map<VideoAdaptationReason, VideoAdaptationCounters>& 167 active_counts); 168 169 // TODO(hbos): Add tests for manager's constraints. 170 // Does not trigger adaptations, only prevents adapting up resolution. 171 class BitrateConstraint : public rtc::RefCountInterface, 172 public AdaptationConstraint { 173 public: 174 explicit BitrateConstraint(VideoStreamEncoderResourceManager* manager); 175 ~BitrateConstraint() override = default; 176 177 void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue); 178 void OnEncoderSettingsUpdated( 179 absl::optional<EncoderSettings> encoder_settings); 180 void OnEncoderTargetBitrateUpdated( 181 absl::optional<uint32_t> encoder_target_bitrate_bps); 182 183 // AdaptationConstraint implementation. Name()184 std::string Name() const override { return "BitrateConstraint"; } 185 bool IsAdaptationUpAllowed( 186 const VideoStreamInputState& input_state, 187 const VideoSourceRestrictions& restrictions_before, 188 const VideoSourceRestrictions& restrictions_after, 189 rtc::scoped_refptr<Resource> reason_resource) const override; 190 191 private: 192 // The |manager_| must be alive as long as this resource is added to the 193 // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called. 194 VideoStreamEncoderResourceManager* const manager_; 195 TaskQueueBase* resource_adaptation_queue_; 196 absl::optional<EncoderSettings> encoder_settings_ 197 RTC_GUARDED_BY(resource_adaptation_queue_); 198 absl::optional<uint32_t> encoder_target_bitrate_bps_ 199 RTC_GUARDED_BY(resource_adaptation_queue_); 200 }; 201 202 // Does not trigger adaptations, only prevents adapting up in BALANCED. 203 class BalancedConstraint : public rtc::RefCountInterface, 204 public AdaptationConstraint { 205 public: 206 BalancedConstraint( 207 VideoStreamEncoderResourceManager* manager, 208 DegradationPreferenceProvider* degradation_preference_provider); 209 ~BalancedConstraint() override = default; 210 211 void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue); 212 void OnEncoderTargetBitrateUpdated( 213 absl::optional<uint32_t> encoder_target_bitrate_bps); 214 215 // AdaptationConstraint implementation. Name()216 std::string Name() const override { return "BalancedConstraint"; } 217 bool IsAdaptationUpAllowed( 218 const VideoStreamInputState& input_state, 219 const VideoSourceRestrictions& restrictions_before, 220 const VideoSourceRestrictions& restrictions_after, 221 rtc::scoped_refptr<Resource> reason_resource) const override; 222 223 private: 224 // The |manager_| must be alive as long as this resource is added to the 225 // ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called. 226 VideoStreamEncoderResourceManager* const manager_; 227 TaskQueueBase* resource_adaptation_queue_; 228 absl::optional<uint32_t> encoder_target_bitrate_bps_ 229 RTC_GUARDED_BY(resource_adaptation_queue_); 230 DegradationPreferenceProvider* degradation_preference_provider_; 231 }; 232 233 DegradationPreferenceProvider* const degradation_preference_provider_; 234 const rtc::scoped_refptr<BitrateConstraint> bitrate_constraint_; 235 const rtc::scoped_refptr<BalancedConstraint> balanced_constraint_; 236 const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_; 237 const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_; 238 239 rtc::TaskQueue* encoder_queue_; 240 rtc::TaskQueue* resource_adaptation_queue_; 241 VideoStreamInputStateProvider* const input_state_provider_ 242 RTC_GUARDED_BY(encoder_queue_); 243 ResourceAdaptationProcessorInterface* adaptation_processor_ 244 RTC_GUARDED_BY(resource_adaptation_queue_); 245 VideoStreamAdapter* stream_adapter_ 246 RTC_GUARDED_BY(resource_adaptation_queue_); 247 // Thread-safe. 248 VideoStreamEncoderObserver* const encoder_stats_observer_; 249 250 DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); 251 VideoSourceRestrictions video_source_restrictions_ 252 RTC_GUARDED_BY(encoder_queue_); 253 254 const BalancedDegradationSettings balanced_settings_; 255 Clock* clock_ RTC_GUARDED_BY(encoder_queue_); 256 const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_); 257 const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_ 258 RTC_GUARDED_BY(encoder_queue_); 259 const bool quality_scaling_experiment_enabled_ RTC_GUARDED_BY(encoder_queue_); 260 absl::optional<uint32_t> encoder_target_bitrate_bps_ 261 RTC_GUARDED_BY(encoder_queue_); 262 absl::optional<VideoEncoder::RateControlParameters> encoder_rates_ 263 RTC_GUARDED_BY(encoder_queue_); 264 std::unique_ptr<QualityRampUpExperimentHelper> quality_rampup_experiment_ 265 RTC_GUARDED_BY(encoder_queue_); 266 absl::optional<EncoderSettings> encoder_settings_ 267 RTC_GUARDED_BY(encoder_queue_); 268 269 // Ties a resource to a reason for statistical reporting. This AdaptReason is 270 // also used by this module to make decisions about how to adapt up/down. 271 struct ResourceAndReason { ResourceAndReasonResourceAndReason272 ResourceAndReason(rtc::scoped_refptr<Resource> resource, 273 VideoAdaptationReason reason) 274 : resource(resource), reason(reason) {} 275 virtual ~ResourceAndReason() = default; 276 277 const rtc::scoped_refptr<Resource> resource; 278 const VideoAdaptationReason reason; 279 }; 280 mutable Mutex resource_lock_; 281 std::vector<ResourceAndReason> resources_ RTC_GUARDED_BY(&resource_lock_); 282 }; 283 284 } // namespace webrtc 285 286 #endif // VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ 287