1 /* 2 * Copyright 2020 The WebRTC Project Authors. All rights reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ 12 #define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <utility> 18 #include <vector> 19 20 #include "absl/types/optional.h" 21 #include "api/adaptation/resource.h" 22 #include "api/rtp_parameters.h" 23 #include "api/scoped_refptr.h" 24 #include "api/task_queue/task_queue_base.h" 25 #include "api/video/video_adaptation_counters.h" 26 #include "api/video/video_frame.h" 27 #include "api/video/video_stream_encoder_observer.h" 28 #include "call/adaptation/resource_adaptation_processor_interface.h" 29 #include "call/adaptation/video_source_restrictions.h" 30 #include "call/adaptation/video_stream_adapter.h" 31 #include "call/adaptation/video_stream_input_state.h" 32 #include "call/adaptation/video_stream_input_state_provider.h" 33 34 namespace webrtc { 35 36 // The Resource Adaptation Processor is responsible for reacting to resource 37 // usage measurements (e.g. overusing or underusing CPU). When a resource is 38 // overused the Processor is responsible for performing mitigations in order to 39 // consume less resources. 40 // 41 // Today we have one Processor per VideoStreamEncoder and the Processor is only 42 // capable of restricting resolution or frame rate of the encoded stream. In the 43 // future we should have a single Processor responsible for all encoded streams, 44 // and it should be capable of reconfiguring other things than just 45 // VideoSourceRestrictions (e.g. reduce render frame rate). 46 // See Resource-Adaptation hotlist: 47 // https://bugs.chromium.org/u/590058293/hotlists/Resource-Adaptation 48 // 49 // The ResourceAdaptationProcessor is single-threaded. It may be constructed on 50 // any thread but MUST subsequently be used and destroyed on a single sequence, 51 // i.e. the "resource adaptation task queue". Resources can be added and removed 52 // from any thread. 53 class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, 54 public VideoSourceRestrictionsListener, 55 public ResourceListener { 56 public: 57 ResourceAdaptationProcessor( 58 VideoStreamEncoderObserver* encoder_stats_observer, 59 VideoStreamAdapter* video_stream_adapter); 60 ~ResourceAdaptationProcessor() override; 61 62 void SetResourceAdaptationQueue( 63 TaskQueueBase* resource_adaptation_queue) override; 64 65 // ResourceAdaptationProcessorInterface implementation. 66 void AddResourceLimitationsListener( 67 ResourceLimitationsListener* limitations_listener) override; 68 void RemoveResourceLimitationsListener( 69 ResourceLimitationsListener* limitations_listener) override; 70 void AddResource(rtc::scoped_refptr<Resource> resource) override; 71 std::vector<rtc::scoped_refptr<Resource>> GetResources() const override; 72 void RemoveResource(rtc::scoped_refptr<Resource> resource) override; 73 74 // ResourceListener implementation. 75 // Triggers OnResourceUnderuse() or OnResourceOveruse(). 76 void OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource, 77 ResourceUsageState usage_state) override; 78 79 // VideoSourceRestrictionsListener implementation. 80 void OnVideoSourceRestrictionsUpdated( 81 VideoSourceRestrictions restrictions, 82 const VideoAdaptationCounters& adaptation_counters, 83 rtc::scoped_refptr<Resource> reason, 84 const VideoSourceRestrictions& unfiltered_restrictions) override; 85 86 private: 87 // If resource usage measurements happens off the adaptation task queue, this 88 // class takes care of posting the measurement for the processor to handle it 89 // on the adaptation task queue. 90 class ResourceListenerDelegate : public rtc::RefCountInterface, 91 public ResourceListener { 92 public: 93 explicit ResourceListenerDelegate(ResourceAdaptationProcessor* processor); 94 95 void SetResourceAdaptationQueue(TaskQueueBase* resource_adaptation_queue); 96 void OnProcessorDestroyed(); 97 98 // ResourceListener implementation. 99 void OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource, 100 ResourceUsageState usage_state) override; 101 102 private: 103 TaskQueueBase* resource_adaptation_queue_; 104 ResourceAdaptationProcessor* processor_ 105 RTC_GUARDED_BY(resource_adaptation_queue_); 106 }; 107 108 enum class MitigationResult { 109 kNotMostLimitedResource, 110 kSharedMostLimitedResource, 111 kRejectedByAdapter, 112 kAdaptationApplied, 113 }; 114 115 struct MitigationResultAndLogMessage { 116 MitigationResultAndLogMessage(); 117 MitigationResultAndLogMessage(MitigationResult result, std::string message); 118 MitigationResult result; 119 std::string message; 120 }; 121 122 // Performs the adaptation by getting the next target, applying it and 123 // informing listeners of the new VideoSourceRestriction and adaptation 124 // counters. 125 MitigationResultAndLogMessage OnResourceUnderuse( 126 rtc::scoped_refptr<Resource> reason_resource); 127 MitigationResultAndLogMessage OnResourceOveruse( 128 rtc::scoped_refptr<Resource> reason_resource); 129 130 void UpdateResourceLimitations(rtc::scoped_refptr<Resource> reason_resource, 131 const VideoSourceRestrictions& restrictions, 132 const VideoAdaptationCounters& counters) 133 RTC_RUN_ON(resource_adaptation_queue_); 134 135 // Searches |adaptation_limits_by_resources_| for each resource with the 136 // highest total adaptation counts. Adaptation up may only occur if the 137 // resource performing the adaptation is the only most limited resource. This 138 // function returns the list of all most limited resources as well as the 139 // corresponding adaptation of that resource. 140 std::pair<std::vector<rtc::scoped_refptr<Resource>>, 141 VideoStreamAdapter::RestrictionsWithCounters> 142 FindMostLimitedResources() const RTC_RUN_ON(resource_adaptation_queue_); 143 144 void RemoveLimitationsImposedByResource( 145 rtc::scoped_refptr<Resource> resource); 146 147 TaskQueueBase* resource_adaptation_queue_; 148 rtc::scoped_refptr<ResourceListenerDelegate> resource_listener_delegate_; 149 // Input and output. 150 VideoStreamEncoderObserver* const encoder_stats_observer_ 151 RTC_GUARDED_BY(resource_adaptation_queue_); 152 mutable Mutex resources_lock_; 153 std::vector<rtc::scoped_refptr<Resource>> resources_ 154 RTC_GUARDED_BY(resources_lock_); 155 std::vector<ResourceLimitationsListener*> resource_limitations_listeners_ 156 RTC_GUARDED_BY(resource_adaptation_queue_); 157 // Purely used for statistics, does not ensure mapped resources stay alive. 158 std::map<rtc::scoped_refptr<Resource>, 159 VideoStreamAdapter::RestrictionsWithCounters> 160 adaptation_limits_by_resources_ 161 RTC_GUARDED_BY(resource_adaptation_queue_); 162 // Responsible for generating and applying possible adaptations. 163 VideoStreamAdapter* const stream_adapter_ 164 RTC_GUARDED_BY(resource_adaptation_queue_); 165 VideoSourceRestrictions last_reported_source_restrictions_ 166 RTC_GUARDED_BY(resource_adaptation_queue_); 167 // Keeps track of previous mitigation results per resource since the last 168 // successful adaptation. Used to avoid RTC_LOG spam. 169 std::map<Resource*, MitigationResult> previous_mitigation_results_ 170 RTC_GUARDED_BY(resource_adaptation_queue_); 171 // Prevents recursion. 172 // 173 // This is used to prevent triggering resource adaptation in the process of 174 // already handling resouce adaptation, since that could cause the same states 175 // to be modified in unexpected ways. Example: 176 // 177 // Resource::OnResourceUsageStateMeasured() -> 178 // ResourceAdaptationProcessor::OnResourceOveruse() -> 179 // Resource::OnAdaptationApplied() -> 180 // Resource::OnResourceUsageStateMeasured() -> 181 // ResourceAdaptationProcessor::OnResourceOveruse() // Boom, not allowed. 182 bool processing_in_progress_ RTC_GUARDED_BY(resource_adaptation_queue_); 183 }; 184 185 } // namespace webrtc 186 187 #endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ 188