1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ 12 #define MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ 13 14 #include <stddef.h> 15 #include <stdint.h> 16 17 #include <memory> 18 19 #include "absl/types/optional.h" 20 #include "api/scoped_refptr.h" 21 #include "api/video_codecs/video_encoder.h" 22 #include "rtc_base/experiments/quality_scaling_experiment.h" 23 #include "rtc_base/numerics/moving_average.h" 24 #include "rtc_base/ref_count.h" 25 #include "rtc_base/ref_counted_object.h" 26 #include "rtc_base/synchronization/sequence_checker.h" 27 #include "rtc_base/task_queue.h" 28 29 namespace webrtc { 30 31 class QualityScalerQpUsageHandlerCallbackInterface; 32 class QualityScalerQpUsageHandlerInterface; 33 34 // QualityScaler runs asynchronously and monitors QP values of encoded frames. 35 // It holds a reference to a QualityScalerQpUsageHandlerInterface implementation 36 // to signal an overuse or underuse of QP (which indicate a desire to scale the 37 // video stream down or up). 38 class QualityScaler { 39 public: 40 // Construct a QualityScaler with given |thresholds| and |handler|. 41 // This starts the quality scaler periodically checking what the average QP 42 // has been recently. 43 QualityScaler(QualityScalerQpUsageHandlerInterface* handler, 44 VideoEncoder::QpThresholds thresholds); 45 virtual ~QualityScaler(); 46 // Should be called each time a frame is dropped at encoding. 47 void ReportDroppedFrameByMediaOpt(); 48 void ReportDroppedFrameByEncoder(); 49 // Inform the QualityScaler of the last seen QP. 50 void ReportQp(int qp, int64_t time_sent_us); 51 52 void SetQpThresholds(VideoEncoder::QpThresholds thresholds); 53 bool QpFastFilterLow() const; 54 55 // The following members declared protected for testing purposes. 56 protected: 57 QualityScaler(QualityScalerQpUsageHandlerInterface* handler, 58 VideoEncoder::QpThresholds thresholds, 59 int64_t sampling_period_ms); 60 61 private: 62 class QpSmoother; 63 class CheckQpTask; 64 class CheckQpTaskHandlerCallback; 65 66 enum class CheckQpResult { 67 kInsufficientSamples, 68 kNormalQp, 69 kHighQp, 70 kLowQp, 71 }; 72 73 // Starts checking for QP in a delayed task. When the resulting CheckQpTask 74 // completes, it will invoke this method again, ensuring that we always 75 // periodically check for QP. See CheckQpTask for more details. We never run 76 // more than one CheckQpTask at a time. 77 void StartNextCheckQpTask(); 78 79 CheckQpResult CheckQp() const; 80 void ClearSamples(); 81 82 std::unique_ptr<CheckQpTask> pending_qp_task_ RTC_GUARDED_BY(&task_checker_); 83 QualityScalerQpUsageHandlerInterface* const handler_ 84 RTC_GUARDED_BY(&task_checker_); 85 SequenceChecker task_checker_; 86 87 VideoEncoder::QpThresholds thresholds_ RTC_GUARDED_BY(&task_checker_); 88 const int64_t sampling_period_ms_; 89 bool fast_rampup_ RTC_GUARDED_BY(&task_checker_); 90 rtc::MovingAverage average_qp_ RTC_GUARDED_BY(&task_checker_); 91 rtc::MovingAverage framedrop_percent_media_opt_ 92 RTC_GUARDED_BY(&task_checker_); 93 rtc::MovingAverage framedrop_percent_all_ RTC_GUARDED_BY(&task_checker_); 94 95 // Used by QualityScalingExperiment. 96 const bool experiment_enabled_; 97 QualityScalingExperiment::Config config_ RTC_GUARDED_BY(&task_checker_); 98 std::unique_ptr<QpSmoother> qp_smoother_high_ RTC_GUARDED_BY(&task_checker_); 99 std::unique_ptr<QpSmoother> qp_smoother_low_ RTC_GUARDED_BY(&task_checker_); 100 101 const size_t min_frames_needed_; 102 const double initial_scale_factor_; 103 const absl::optional<double> scale_factor_; 104 }; 105 106 // Reacts to QP being too high or too low. For best quality, when QP is high it 107 // is desired to decrease the resolution or frame rate of the stream and when QP 108 // is low it is desired to increase the resolution or frame rate of the stream. 109 // Whether to reconfigure the stream is ultimately up to the handler, which is 110 // able to respond asynchronously. 111 class QualityScalerQpUsageHandlerInterface { 112 public: 113 virtual ~QualityScalerQpUsageHandlerInterface(); 114 115 // Reacts to QP usage being too high or too low. The |callback| MUST be 116 // invoked when the handler is done, allowing the QualityScaler to resume 117 // checking for QP. 118 virtual void OnReportQpUsageHigh( 119 rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> 120 callback) = 0; 121 virtual void OnReportQpUsageLow( 122 rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> 123 callback) = 0; 124 }; 125 126 // When QP is reported as high or low by the QualityScaler, it pauses checking 127 // for QP until the QP usage has been handled. When OnQpUsageHandled() is 128 // invoked, the QualityScaler resumes checking for QP. This ensures that if the 129 // stream is reconfigured in response to QP usage we do not include QP samples 130 // from before the reconfiguration the next time we check for QP. 131 // 132 // OnQpUsageHandled() MUST be invoked exactly once before this object is 133 // destroyed. 134 class QualityScalerQpUsageHandlerCallbackInterface 135 : public rtc::RefCountedObject<rtc::RefCountInterface> { 136 public: 137 virtual ~QualityScalerQpUsageHandlerCallbackInterface(); 138 139 // If |clear_qp_samples| is true, existing QP samples are cleared before the 140 // next time QualityScaler checks for QP. This is usually a good idea when the 141 // stream is reconfigured. If |clear_qp_samples| is false, samples are not 142 // cleared and QualityScaler increases its frequency of checking for QP. 143 virtual void OnQpUsageHandled(bool clear_qp_samples) = 0; 144 145 protected: 146 QualityScalerQpUsageHandlerCallbackInterface(); 147 }; 148 149 } // namespace webrtc 150 151 #endif // MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ 152