1 /* 2 * Copyright 2020 The WebRTC Project Authors. All rights reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ 12 #define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <utility> 18 #include <vector> 19 20 #include "absl/strings/string_view.h" 21 #include "absl/types/optional.h" 22 #include "api/adaptation/resource.h" 23 #include "api/rtp_parameters.h" 24 #include "api/scoped_refptr.h" 25 #include "api/task_queue/task_queue_base.h" 26 #include "api/video/video_adaptation_counters.h" 27 #include "api/video/video_frame.h" 28 #include "call/adaptation/resource_adaptation_processor_interface.h" 29 #include "call/adaptation/video_source_restrictions.h" 30 #include "call/adaptation/video_stream_adapter.h" 31 #include "call/adaptation/video_stream_input_state.h" 32 #include "call/adaptation/video_stream_input_state_provider.h" 33 #include "video/video_stream_encoder_observer.h" 34 35 namespace webrtc { 36 37 // The Resource Adaptation Processor is responsible for reacting to resource 38 // usage measurements (e.g. overusing or underusing CPU). When a resource is 39 // overused the Processor is responsible for performing mitigations in order to 40 // consume less resources. 41 // 42 // Today we have one Processor per VideoStreamEncoder and the Processor is only 43 // capable of restricting resolution or frame rate of the encoded stream. In the 44 // future we should have a single Processor responsible for all encoded streams, 45 // and it should be capable of reconfiguring other things than just 46 // VideoSourceRestrictions (e.g. reduce render frame rate). 47 // See Resource-Adaptation hotlist: 48 // https://bugs.chromium.org/u/590058293/hotlists/Resource-Adaptation 49 // 50 // The ResourceAdaptationProcessor is single-threaded. It may be constructed on 51 // any thread but MUST subsequently be used and destroyed on a single sequence, 52 // i.e. the "resource adaptation task queue". Resources can be added and removed 53 // from any thread. 54 class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, 55 public VideoSourceRestrictionsListener, 56 public ResourceListener { 57 public: 58 explicit ResourceAdaptationProcessor( 59 VideoStreamAdapter* video_stream_adapter); 60 ~ResourceAdaptationProcessor() override; 61 62 // ResourceAdaptationProcessorInterface implementation. 63 void AddResourceLimitationsListener( 64 ResourceLimitationsListener* limitations_listener) override; 65 void RemoveResourceLimitationsListener( 66 ResourceLimitationsListener* limitations_listener) override; 67 void AddResource(rtc::scoped_refptr<Resource> resource) override; 68 std::vector<rtc::scoped_refptr<Resource>> GetResources() const override; 69 void RemoveResource(rtc::scoped_refptr<Resource> resource) override; 70 71 // ResourceListener implementation. 72 // Triggers OnResourceUnderuse() or OnResourceOveruse(). 73 void OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource, 74 ResourceUsageState usage_state) override; 75 76 // VideoSourceRestrictionsListener implementation. 77 void OnVideoSourceRestrictionsUpdated( 78 VideoSourceRestrictions restrictions, 79 const VideoAdaptationCounters& adaptation_counters, 80 rtc::scoped_refptr<Resource> reason, 81 const VideoSourceRestrictions& unfiltered_restrictions) override; 82 83 private: 84 // If resource usage measurements happens off the adaptation task queue, this 85 // class takes care of posting the measurement for the processor to handle it 86 // on the adaptation task queue. 87 class ResourceListenerDelegate : public rtc::RefCountInterface, 88 public ResourceListener { 89 public: 90 explicit ResourceListenerDelegate(ResourceAdaptationProcessor* processor); 91 92 void OnProcessorDestroyed(); 93 94 // ResourceListener implementation. 95 void OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource, 96 ResourceUsageState usage_state) override; 97 98 private: 99 TaskQueueBase* task_queue_; 100 ResourceAdaptationProcessor* processor_ RTC_GUARDED_BY(task_queue_); 101 }; 102 103 enum class MitigationResult { 104 kNotMostLimitedResource, 105 kSharedMostLimitedResource, 106 kRejectedByAdapter, 107 kAdaptationApplied, 108 }; 109 110 struct MitigationResultAndLogMessage { 111 MitigationResultAndLogMessage(); 112 MitigationResultAndLogMessage(MitigationResult result, 113 absl::string_view message); 114 MitigationResult result; 115 std::string message; 116 }; 117 118 // Performs the adaptation by getting the next target, applying it and 119 // informing listeners of the new VideoSourceRestriction and adaptation 120 // counters. 121 MitigationResultAndLogMessage OnResourceUnderuse( 122 rtc::scoped_refptr<Resource> reason_resource); 123 MitigationResultAndLogMessage OnResourceOveruse( 124 rtc::scoped_refptr<Resource> reason_resource); 125 126 void UpdateResourceLimitations(rtc::scoped_refptr<Resource> reason_resource, 127 const VideoSourceRestrictions& restrictions, 128 const VideoAdaptationCounters& counters) 129 RTC_RUN_ON(task_queue_); 130 131 // Searches `adaptation_limits_by_resources_` for each resource with the 132 // highest total adaptation counts. Adaptation up may only occur if the 133 // resource performing the adaptation is the only most limited resource. This 134 // function returns the list of all most limited resources as well as the 135 // corresponding adaptation of that resource. 136 std::pair<std::vector<rtc::scoped_refptr<Resource>>, 137 VideoStreamAdapter::RestrictionsWithCounters> 138 FindMostLimitedResources() const RTC_RUN_ON(task_queue_); 139 140 void RemoveLimitationsImposedByResource( 141 rtc::scoped_refptr<Resource> resource); 142 143 TaskQueueBase* task_queue_; 144 rtc::scoped_refptr<ResourceListenerDelegate> resource_listener_delegate_; 145 // Input and output. 146 mutable Mutex resources_lock_; 147 std::vector<rtc::scoped_refptr<Resource>> resources_ 148 RTC_GUARDED_BY(resources_lock_); 149 std::vector<ResourceLimitationsListener*> resource_limitations_listeners_ 150 RTC_GUARDED_BY(task_queue_); 151 // Purely used for statistics, does not ensure mapped resources stay alive. 152 std::map<rtc::scoped_refptr<Resource>, 153 VideoStreamAdapter::RestrictionsWithCounters> 154 adaptation_limits_by_resources_ RTC_GUARDED_BY(task_queue_); 155 // Responsible for generating and applying possible adaptations. 156 VideoStreamAdapter* const stream_adapter_ RTC_GUARDED_BY(task_queue_); 157 VideoSourceRestrictions last_reported_source_restrictions_ 158 RTC_GUARDED_BY(task_queue_); 159 // Keeps track of previous mitigation results per resource since the last 160 // successful adaptation. Used to avoid RTC_LOG spam. 161 std::map<Resource*, MitigationResult> previous_mitigation_results_ 162 RTC_GUARDED_BY(task_queue_); 163 }; 164 165 } // namespace webrtc 166 167 #endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ 168