1 /* 2 * libjingle 3 * Copyright 2010 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT 29 #define TALK_MEDIA_BASE_VIDEOADAPTER_H_ 30 31 #include "talk/media/base/videocommon.h" 32 #include "webrtc/base/common.h" // For ASSERT 33 #include "webrtc/base/criticalsection.h" 34 #include "webrtc/base/scoped_ptr.h" 35 #include "webrtc/base/sigslot.h" 36 37 namespace cricket { 38 39 class VideoFrame; 40 41 // VideoAdapter adapts an input video frame to an output frame based on the 42 // specified input and output formats. The adaptation includes dropping frames 43 // to reduce frame rate and scaling frames. VideoAdapter is thread safe. 44 class VideoAdapter { 45 public: 46 VideoAdapter(); 47 virtual ~VideoAdapter(); 48 49 virtual void SetInputFormat(const VideoFormat& format); 50 void SetOutputFormat(const VideoFormat& format); 51 // Constrain output resolution to this many pixels overall 52 void SetOutputNumPixels(int num_pixels); 53 int GetOutputNumPixels() const; 54 55 const VideoFormat& input_format(); 56 // Returns true if the adapter will always return zero size from 57 // AdaptFrameResolution. 58 bool drops_all_frames() const; 59 const VideoFormat& output_format(); 60 61 // Return the adapted resolution given the input resolution. The returned 62 // resolution will be 0x0 if the frame should be dropped. 63 VideoFormat AdaptFrameResolution(int in_width, int in_height); 64 65 void set_scale_third(bool enable); scale_third()66 bool scale_third() const { return scale_third_; } 67 adaptation_changes()68 int adaptation_changes() const { return adaption_changes_; } 69 70 protected: 71 float FindClosestScale(int width, int height, int target_num_pixels); 72 float FindClosestViewScale(int width, int height, int target_num_pixels); 73 float FindLowerScale(int width, int height, int target_num_pixels); 74 75 private: 76 const float* GetViewScaleFactors() const; 77 float FindScale(const float* scale_factors, 78 const float upbias, int width, int height, 79 int target_num_pixels); 80 81 VideoFormat input_format_; 82 VideoFormat output_format_; 83 int output_num_pixels_; 84 bool scale_third_; // True if adapter allows scaling to 1/3 and 2/3. 85 int frames_in_; // Number of input frames. 86 int frames_out_; // Number of output frames. 87 int frames_scaled_; // Number of frames scaled. 88 int adaption_changes_; // Number of changes in scale factor. 89 size_t previous_width_; // Previous adapter output width. 90 size_t previous_height_; // Previous adapter output height. 91 int64_t interval_next_frame_; 92 // The critical section to protect the above variables. 93 rtc::CriticalSection critical_section_; 94 95 RTC_DISALLOW_COPY_AND_ASSIGN(VideoAdapter); 96 }; 97 98 // CoordinatedVideoAdapter adapts the video input to the encoder by coordinating 99 // the format request from the server, the resolution request from the encoder, 100 // and the CPU load. 101 class CoordinatedVideoAdapter 102 : public VideoAdapter, public sigslot::has_slots<> { 103 public: 104 enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE }; 105 enum AdaptReasonEnum { 106 ADAPTREASON_NONE = 0, 107 ADAPTREASON_CPU = 1, 108 ADAPTREASON_BANDWIDTH = 2, 109 ADAPTREASON_VIEW = 4 110 }; 111 typedef int AdaptReason; 112 113 CoordinatedVideoAdapter(); ~CoordinatedVideoAdapter()114 virtual ~CoordinatedVideoAdapter() {} 115 116 virtual void SetInputFormat(const VideoFormat& format); 117 118 // Enable or disable video adaptation due to the change of the CPU load. set_cpu_adaptation(bool enable)119 void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; } cpu_adaptation()120 bool cpu_adaptation() const { return cpu_adaptation_; } 121 // Enable or disable smoothing when doing CPU adaptation. When smoothing is 122 // enabled, system CPU load is tracked using an exponential weighted 123 // average. 124 void set_cpu_smoothing(bool enable); cpu_smoothing()125 bool cpu_smoothing() const { return cpu_smoothing_; } 126 // Enable or disable video adaptation due to the change of the GD set_gd_adaptation(bool enable)127 void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; } gd_adaptation()128 bool gd_adaptation() const { return gd_adaptation_; } 129 // Enable or disable video adaptation due to the change of the View set_view_adaptation(bool enable)130 void set_view_adaptation(bool enable) { view_adaptation_ = enable; } view_adaptation()131 bool view_adaptation() const { return view_adaptation_; } 132 // Enable or disable video adaptation to fast switch View set_view_switch(bool enable)133 void set_view_switch(bool enable) { view_switch_ = enable; } view_switch()134 bool view_switch() const { return view_switch_; } 135 adapt_reason()136 CoordinatedVideoAdapter::AdaptReason adapt_reason() const { 137 return adapt_reason_; 138 } 139 140 // When the video is decreased, set the waiting time for CPU adaptation to 141 // decrease video again. 142 void set_cpu_load_min_samples(int cpu_load_min_samples); cpu_load_min_samples()143 int cpu_load_min_samples() const { return cpu_load_min_samples_; } 144 // CPU system load high threshold for reducing resolution. e.g. 0.85f 145 void set_high_system_threshold(float high_system_threshold); high_system_threshold()146 float high_system_threshold() const { return high_system_threshold_; } 147 // CPU system load low threshold for increasing resolution. e.g. 0.70f 148 void set_low_system_threshold(float low_system_threshold); low_system_threshold()149 float low_system_threshold() const { return low_system_threshold_; } 150 // CPU process load threshold for reducing resolution. e.g. 0.10f 151 void set_process_threshold(float process_threshold); process_threshold()152 float process_threshold() const { return process_threshold_; } 153 154 // Handle the format request from the server via Jingle update message. 155 void OnOutputFormatRequest(const VideoFormat& format); 156 // Handle the resolution request from the encoder due to bandwidth changes. 157 void OnEncoderResolutionRequest(int width, int height, AdaptRequest request); 158 // Handle the resolution request for CPU overuse. 159 void OnCpuResolutionRequest(AdaptRequest request); 160 // Handle the CPU load provided by a CPU monitor. 161 void OnCpuLoadUpdated(int current_cpus, int max_cpus, 162 float process_load, float system_load); 163 164 sigslot::signal0<> SignalCpuAdaptationUnable; 165 166 private: 167 // Adapt to the minimum of the formats the server requests, the CPU wants, and 168 // the encoder wants. Returns true if resolution changed. 169 bool AdaptToMinimumFormat(int* new_width, int* new_height); 170 bool IsMinimumFormat(int pixels); 171 void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request, 172 int* num_pixels); 173 CoordinatedVideoAdapter::AdaptRequest FindCpuRequest( 174 int current_cpus, int max_cpus, 175 float process_load, float system_load); 176 177 bool cpu_adaptation_; // True if cpu adaptation is enabled. 178 bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation). 179 bool gd_adaptation_; // True if gd adaptation is enabled. 180 bool view_adaptation_; // True if view adaptation is enabled. 181 bool view_switch_; // True if view switch is enabled. 182 int cpu_downgrade_count_; 183 int cpu_load_min_samples_; 184 int cpu_load_num_samples_; 185 // cpu system load thresholds relative to max cpus. 186 float high_system_threshold_; 187 float low_system_threshold_; 188 // cpu process load thresholds relative to current cpus. 189 float process_threshold_; 190 // Video formats that the server view requests, the CPU wants, and the encoder 191 // wants respectively. The adapted output format is the minimum of these. 192 int view_desired_num_pixels_; 193 int64_t view_desired_interval_; 194 int encoder_desired_num_pixels_; 195 int cpu_desired_num_pixels_; 196 CoordinatedVideoAdapter::AdaptReason adapt_reason_; 197 // The critical section to protect handling requests. 198 rtc::CriticalSection request_critical_section_; 199 200 // The weighted average of cpu load over time. It's always updated (if cpu 201 // adaptation is on), but only used if cpu_smoothing_ is set. 202 float system_load_average_; 203 204 RTC_DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter); 205 }; 206 207 } // namespace cricket 208 209 #endif // TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT 210