1 /* 2 * Copyright 2020 The WebRTC Project Authors. All rights reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ 12 #define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ 13 14 #include <memory> 15 #include <utility> 16 #include <vector> 17 18 #include "absl/types/optional.h" 19 #include "absl/types/variant.h" 20 #include "api/adaptation/resource.h" 21 #include "api/rtp_parameters.h" 22 #include "api/video/video_adaptation_counters.h" 23 #include "call/adaptation/adaptation_constraint.h" 24 #include "call/adaptation/adaptation_listener.h" 25 #include "call/adaptation/degradation_preference_provider.h" 26 #include "call/adaptation/video_source_restrictions.h" 27 #include "call/adaptation/video_stream_input_state.h" 28 #include "call/adaptation/video_stream_input_state_provider.h" 29 #include "modules/video_coding/utility/quality_scaler.h" 30 #include "rtc_base/experiments/balanced_degradation_settings.h" 31 #include "rtc_base/thread_annotations.h" 32 33 namespace webrtc { 34 35 // The listener is responsible for carrying out the reconfiguration of the video 36 // source such that the VideoSourceRestrictions are fulfilled. 37 class VideoSourceRestrictionsListener { 38 public: 39 virtual ~VideoSourceRestrictionsListener(); 40 41 // The |restrictions| are filtered by degradation preference but not the 42 // |adaptation_counters|, which are currently only reported for legacy stats 43 // calculation purposes. 44 virtual void OnVideoSourceRestrictionsUpdated( 45 VideoSourceRestrictions restrictions, 46 const VideoAdaptationCounters& adaptation_counters, 47 rtc::scoped_refptr<Resource> reason, 48 const VideoSourceRestrictions& unfiltered_restrictions) = 0; 49 }; 50 51 class VideoStreamAdapter; 52 53 extern const int kMinFrameRateFps; 54 55 VideoSourceRestrictions FilterRestrictionsByDegradationPreference( 56 VideoSourceRestrictions source_restrictions, 57 DegradationPreference degradation_preference); 58 59 int GetHigherResolutionThan(int pixel_count); 60 61 // Either represents the next VideoSourceRestrictions the VideoStreamAdapter 62 // will take, or provides a Status code indicating the reason for not adapting 63 // if the adaptation is not valid. 64 class Adaptation final { 65 public: 66 enum class Status { 67 // Applying this adaptation will have an effect. All other Status codes 68 // indicate that adaptation is not possible and why. 69 kValid, 70 // Cannot adapt. The minimum or maximum adaptation has already been reached. 71 // There are no more steps to take. 72 kLimitReached, 73 // Cannot adapt. The resolution or frame rate requested by a recent 74 // adaptation has not yet been reflected in the input resolution or frame 75 // rate; adaptation is refused to avoid "double-adapting". 76 kAwaitingPreviousAdaptation, 77 // Not enough input. 78 kInsufficientInput, 79 // Adaptation disabled via degradation preference. 80 kAdaptationDisabled, 81 // Adaptation up was rejected by a VideoAdaptationConstraint. 82 kRejectedByConstraint, 83 }; 84 85 static const char* StatusToString(Status status); 86 87 Status status() const; 88 const VideoStreamInputState& input_state() const; 89 const VideoSourceRestrictions& restrictions() const; 90 const VideoAdaptationCounters& counters() const; 91 // Used for stats reporting. 92 bool min_pixel_limit_reached() const; 93 94 private: 95 friend class VideoStreamAdapter; 96 97 // Constructs with a valid adaptation. Status is kValid. 98 Adaptation(int validation_id, 99 VideoSourceRestrictions restrictions, 100 VideoAdaptationCounters counters, 101 VideoStreamInputState input_state, 102 bool min_pixel_limit_reached); 103 // Constructor when adaptation is not valid. Status MUST NOT be kValid. 104 Adaptation(int validation_id, 105 Status invalid_status, 106 VideoStreamInputState input_state, 107 bool min_pixel_limit_reached); 108 109 // An Adaptation can become invalidated if the state of VideoStreamAdapter is 110 // modified before the Adaptation is applied. To guard against this, this ID 111 // has to match VideoStreamAdapter::adaptation_validation_id_ when applied. 112 // TODO(https://crbug.com/webrtc/11700): Remove the validation_id_. 113 const int validation_id_; 114 const Status status_; 115 const bool min_pixel_limit_reached_; 116 // Input state when adaptation was made. 117 const VideoStreamInputState input_state_; 118 const VideoSourceRestrictions restrictions_; 119 const VideoAdaptationCounters counters_; 120 }; 121 122 // Owns the VideoSourceRestriction for a single stream and is responsible for 123 // adapting it up or down when told to do so. This class serves the following 124 // purposes: 125 // 1. Keep track of a stream's restrictions. 126 // 2. Provide valid ways to adapt up or down the stream's restrictions. 127 // 3. Modify the stream's restrictions in one of the valid ways. 128 class VideoStreamAdapter { 129 public: 130 explicit VideoStreamAdapter( 131 VideoStreamInputStateProvider* input_state_provider); 132 ~VideoStreamAdapter(); 133 134 VideoSourceRestrictions source_restrictions() const; 135 const VideoAdaptationCounters& adaptation_counters() const; 136 void ClearRestrictions(); 137 138 void AddRestrictionsListener( 139 VideoSourceRestrictionsListener* restrictions_listener); 140 void RemoveRestrictionsListener( 141 VideoSourceRestrictionsListener* restrictions_listener); 142 void AddAdaptationListener(AdaptationListener* adaptation_listener); 143 void RemoveAdaptationListener(AdaptationListener* adaptation_listener); 144 void AddAdaptationConstraint(AdaptationConstraint* adaptation_constraint); 145 void RemoveAdaptationConstraint(AdaptationConstraint* adaptation_constraint); 146 147 // TODO(hbos): Setting the degradation preference should not clear 148 // restrictions! This is not defined in the spec and is unexpected, there is a 149 // tiny risk that people would discover and rely on this behavior. 150 void SetDegradationPreference(DegradationPreference degradation_preference); 151 152 // Returns an adaptation that we are guaranteed to be able to apply, or a 153 // status code indicating the reason why we cannot adapt. 154 // TODO(https://crbug.com/webrtc/11771) |resource| is needed by the 155 // AdaptationConstraint resources. Remove this parameter when it's removed. 156 Adaptation GetAdaptationUp(rtc::scoped_refptr<Resource> resource); 157 Adaptation GetAdaptationDown(); 158 Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters, 159 const VideoSourceRestrictions& restrictions); 160 // Tries to adapt the resolution one step. This is used for initial frame 161 // dropping. Does nothing if the degradation preference is not BALANCED or 162 // MAINTAIN_FRAMERATE. In the case of BALANCED, it will try twice to reduce 163 // the resolution. If it fails twice it gives up. 164 Adaptation GetAdaptDownResolution(); 165 166 // Updates source_restrictions() the Adaptation. 167 void ApplyAdaptation(const Adaptation& adaptation, 168 rtc::scoped_refptr<Resource> resource); 169 170 struct RestrictionsWithCounters { 171 VideoSourceRestrictions restrictions; 172 VideoAdaptationCounters counters; 173 }; 174 175 private: 176 void BroadcastVideoRestrictionsUpdate( 177 const VideoStreamInputState& input_state, 178 const rtc::scoped_refptr<Resource>& resource); 179 180 bool HasSufficientInputForAdaptation(const VideoStreamInputState& input_state) 181 const RTC_RUN_ON(&sequence_checker_); 182 183 using RestrictionsOrState = 184 absl::variant<RestrictionsWithCounters, Adaptation::Status>; 185 RestrictionsOrState GetAdaptationUpStep( 186 const VideoStreamInputState& input_state) const 187 RTC_RUN_ON(&sequence_checker_); 188 RestrictionsOrState GetAdaptationDownStep( 189 const VideoStreamInputState& input_state) const 190 RTC_RUN_ON(&sequence_checker_); 191 RestrictionsOrState GetAdaptDownResolutionStepForBalanced( 192 const VideoStreamInputState& input_state) const 193 RTC_RUN_ON(&sequence_checker_); 194 195 // TODO(https://crbug.com/webrtc/11771) |resource| is needed by the 196 // AdaptationConstraint resources. Remove this parameter when it's removed. 197 Adaptation GetAdaptationUp(const VideoStreamInputState& input_state, 198 rtc::scoped_refptr<Resource> resource) const 199 RTC_RUN_ON(&sequence_checker_); 200 Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const 201 RTC_RUN_ON(&sequence_checker_); 202 203 static RestrictionsOrState DecreaseResolution( 204 const VideoStreamInputState& input_state, 205 const RestrictionsWithCounters& current_restrictions); 206 static RestrictionsOrState IncreaseResolution( 207 const VideoStreamInputState& input_state, 208 const RestrictionsWithCounters& current_restrictions); 209 // Framerate methods are member functions because they need internal state 210 // if the degradation preference is BALANCED. 211 RestrictionsOrState DecreaseFramerate( 212 const VideoStreamInputState& input_state, 213 const RestrictionsWithCounters& current_restrictions) const 214 RTC_RUN_ON(&sequence_checker_); 215 RestrictionsOrState IncreaseFramerate( 216 const VideoStreamInputState& input_state, 217 const RestrictionsWithCounters& current_restrictions) const 218 RTC_RUN_ON(&sequence_checker_); 219 220 struct RestrictionsOrStateVisitor; 221 Adaptation RestrictionsOrStateToAdaptation( 222 RestrictionsOrState step_or_state, 223 const VideoStreamInputState& input_state) const 224 RTC_RUN_ON(&sequence_checker_); 225 226 SequenceChecker sequence_checker_ RTC_GUARDED_BY(&sequence_checker_); 227 // Gets the input state which is the basis of all adaptations. 228 // Thread safe. 229 VideoStreamInputStateProvider* input_state_provider_; 230 // Decides the next adaptation target in DegradationPreference::BALANCED. 231 const BalancedDegradationSettings balanced_settings_; 232 // To guard against applying adaptations that have become invalidated, an 233 // Adaptation that is applied has to have a matching validation ID. 234 int adaptation_validation_id_ RTC_GUARDED_BY(&sequence_checker_); 235 // When deciding the next target up or down, different strategies are used 236 // depending on the DegradationPreference. 237 // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference 238 DegradationPreference degradation_preference_ 239 RTC_GUARDED_BY(&sequence_checker_); 240 // Used to avoid adapting twice. Stores the resolution at the time of the last 241 // adaptation. 242 // TODO(hbos): Can we implement a more general "cooldown" mechanism of 243 // resources intead? If we already have adapted it seems like we should wait 244 // a while before adapting again, so that we are not acting on usage 245 // measurements that are made obsolete/unreliable by an "ongoing" adaptation. 246 struct AwaitingFrameSizeChange { 247 AwaitingFrameSizeChange(bool pixels_increased, int frame_size); 248 const bool pixels_increased; 249 const int frame_size_pixels; 250 }; 251 absl::optional<AwaitingFrameSizeChange> awaiting_frame_size_change_ 252 RTC_GUARDED_BY(&sequence_checker_); 253 // The previous restrictions value. Starts as unrestricted. 254 VideoSourceRestrictions last_video_source_restrictions_ 255 RTC_GUARDED_BY(&sequence_checker_); 256 VideoSourceRestrictions last_filtered_restrictions_ 257 RTC_GUARDED_BY(&sequence_checker_); 258 259 std::vector<VideoSourceRestrictionsListener*> restrictions_listeners_ 260 RTC_GUARDED_BY(&sequence_checker_); 261 std::vector<AdaptationListener*> adaptation_listeners_ 262 RTC_GUARDED_BY(&sequence_checker_); 263 std::vector<AdaptationConstraint*> adaptation_constraints_ 264 RTC_GUARDED_BY(&sequence_checker_); 265 266 RestrictionsWithCounters current_restrictions_ 267 RTC_GUARDED_BY(&sequence_checker_); 268 }; 269 270 } // namespace webrtc 271 272 #endif // CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ 273