1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ 12 #define VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ 13 14 #include <list> 15 #include <memory> 16 17 #include "absl/types/optional.h" 18 #include "api/task_queue/task_queue_base.h" 19 #include "api/video/video_stream_encoder_observer.h" 20 #include "rtc_base/constructor_magic.h" 21 #include "rtc_base/experiments/field_trial_parser.h" 22 #include "rtc_base/numerics/exp_filter.h" 23 #include "rtc_base/synchronization/sequence_checker.h" 24 #include "rtc_base/task_utils/repeating_task.h" 25 #include "rtc_base/thread_annotations.h" 26 27 namespace webrtc { 28 29 class VideoFrame; 30 31 struct CpuOveruseOptions { 32 CpuOveruseOptions(); 33 34 int low_encode_usage_threshold_percent; // Threshold for triggering underuse. 35 int high_encode_usage_threshold_percent; // Threshold for triggering overuse. 36 // General settings. 37 int frame_timeout_interval_ms; // The maximum allowed interval between two 38 // frames before resetting estimations. 39 int min_frame_samples; // The minimum number of frames required. 40 int min_process_count; // The number of initial process times required before 41 // triggering an overuse/underuse. 42 int high_threshold_consecutive_count; // The number of consecutive checks 43 // above the high threshold before 44 // triggering an overuse. 45 // New estimator enabled if this is set non-zero. 46 int filter_time_ms; // Time constant for averaging 47 }; 48 49 class OveruseFrameDetectorObserverInterface { 50 public: 51 // Called to signal that we can handle larger or more frequent frames. 52 virtual void AdaptUp() = 0; 53 // Called to signal that the source should reduce the resolution or framerate. 54 virtual void AdaptDown() = 0; 55 56 protected: ~OveruseFrameDetectorObserverInterface()57 virtual ~OveruseFrameDetectorObserverInterface() {} 58 }; 59 60 // Use to detect system overuse based on the send-side processing time of 61 // incoming frames. All methods must be called on a single task queue but it can 62 // be created and destroyed on an arbitrary thread. 63 // OveruseFrameDetector::StartCheckForOveruse must be called to periodically 64 // check for overuse. 65 class OveruseFrameDetector { 66 public: 67 explicit OveruseFrameDetector(CpuOveruseMetricsObserver* metrics_observer); 68 virtual ~OveruseFrameDetector(); 69 70 // Start to periodically check for overuse. 71 void StartCheckForOveruse( 72 TaskQueueBase* task_queue_base, 73 const CpuOveruseOptions& options, 74 OveruseFrameDetectorObserverInterface* overuse_observer); 75 76 // StopCheckForOveruse must be called before destruction if 77 // StartCheckForOveruse has been called. 78 void StopCheckForOveruse(); 79 80 // Defines the current maximum framerate targeted by the capturer. This is 81 // used to make sure the encode usage percent doesn't drop unduly if the 82 // capturer has quiet periods (for instance caused by screen capturers with 83 // variable capture rate depending on content updates), otherwise we might 84 // experience adaptation toggling. 85 virtual void OnTargetFramerateUpdated(int framerate_fps); 86 87 // Called for each captured frame. 88 void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); 89 90 // Called for each sent frame. 91 void FrameSent(uint32_t timestamp, 92 int64_t time_sent_in_us, 93 int64_t capture_time_us, 94 absl::optional<int> encode_duration_us); 95 96 // Interface for cpu load estimation. Intended for internal use only. 97 class ProcessingUsage { 98 public: 99 virtual void Reset() = 0; 100 virtual void SetMaxSampleDiffMs(float diff_ms) = 0; 101 virtual void FrameCaptured(const VideoFrame& frame, 102 int64_t time_when_first_seen_us, 103 int64_t last_capture_time_us) = 0; 104 // Returns encode_time in us, if there's a new measurement. 105 virtual absl::optional<int> FrameSent( 106 // These two argument used by old estimator. 107 uint32_t timestamp, 108 int64_t time_sent_in_us, 109 // And these two by the new estimator. 110 int64_t capture_time_us, 111 absl::optional<int> encode_duration_us) = 0; 112 113 virtual int Value() = 0; 114 virtual ~ProcessingUsage() = default; 115 }; 116 117 protected: 118 // Protected for test purposes. 119 void CheckForOveruse(OveruseFrameDetectorObserverInterface* overuse_observer); 120 void SetOptions(const CpuOveruseOptions& options); 121 122 CpuOveruseOptions options_; 123 124 private: 125 void EncodedFrameTimeMeasured(int encode_duration_ms); 126 bool IsOverusing(int encode_usage_percent); 127 bool IsUnderusing(int encode_usage_percent, int64_t time_now); 128 129 bool FrameTimeoutDetected(int64_t now) const; 130 bool FrameSizeChanged(int num_pixels) const; 131 132 void ResetAll(int num_pixels); 133 134 static std::unique_ptr<ProcessingUsage> CreateProcessingUsage( 135 const CpuOveruseOptions& options); 136 137 SequenceChecker task_checker_; 138 // Owned by the task queue from where StartCheckForOveruse is called. 139 RepeatingTaskHandle check_overuse_task_ RTC_GUARDED_BY(task_checker_); 140 141 // Stats metrics. 142 CpuOveruseMetricsObserver* const metrics_observer_; 143 absl::optional<int> encode_usage_percent_ RTC_GUARDED_BY(task_checker_); 144 145 int64_t num_process_times_ RTC_GUARDED_BY(task_checker_); 146 147 int64_t last_capture_time_us_ RTC_GUARDED_BY(task_checker_); 148 149 // Number of pixels of last captured frame. 150 int num_pixels_ RTC_GUARDED_BY(task_checker_); 151 int max_framerate_ RTC_GUARDED_BY(task_checker_); 152 int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_); 153 int checks_above_threshold_ RTC_GUARDED_BY(task_checker_); 154 int num_overuse_detections_ RTC_GUARDED_BY(task_checker_); 155 int64_t last_rampup_time_ms_ RTC_GUARDED_BY(task_checker_); 156 bool in_quick_rampup_ RTC_GUARDED_BY(task_checker_); 157 int current_rampup_delay_ms_ RTC_GUARDED_BY(task_checker_); 158 159 std::unique_ptr<ProcessingUsage> usage_ RTC_PT_GUARDED_BY(task_checker_); 160 161 // If set by field trial, overrides CpuOveruseOptions::filter_time_ms. 162 FieldTrialOptional<TimeDelta> filter_time_constant_{"tau"}; 163 164 RTC_DISALLOW_COPY_AND_ASSIGN(OveruseFrameDetector); 165 }; 166 167 } // namespace webrtc 168 169 #endif // VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ 170