1 // libjingle 2 // Copyright 2010 Google Inc. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // 1. Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // 2. Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // 3. The name of the author may not be used to endorse or promote products 13 // derived from this software without specific prior written permission. 14 // 15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 16 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 17 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 18 // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 24 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 26 #ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT 27 #define TALK_MEDIA_BASE_VIDEOADAPTER_H_ 28 29 #include "talk/media/base/videocommon.h" 30 #include "webrtc/base/common.h" // For ASSERT 31 #include "webrtc/base/criticalsection.h" 32 #include "webrtc/base/scoped_ptr.h" 33 #include "webrtc/base/sigslot.h" 34 35 namespace cricket { 36 37 class VideoFrame; 38 39 // VideoAdapter adapts an input video frame to an output frame based on the 40 // specified input and output formats. The adaptation includes dropping frames 41 // to reduce frame rate and scaling frames. VideoAdapter is thread safe. 42 class VideoAdapter { 43 public: 44 VideoAdapter(); 45 virtual ~VideoAdapter(); 46 47 virtual void SetInputFormat(const VideoFormat& format); 48 void SetOutputFormat(const VideoFormat& format); 49 // Constrain output resolution to this many pixels overall 50 void SetOutputNumPixels(int num_pixels); 51 int GetOutputNumPixels() const; 52 53 const VideoFormat& input_format(); 54 // Returns true if the adapter is dropping frames in calls to AdaptFrame. 55 bool drops_all_frames() const; 56 const VideoFormat& output_format(); 57 // If the parameter black is true, the adapted frames will be black. 58 void SetBlackOutput(bool black); 59 60 // Adapt the input frame from the input format to the output format. Return 61 // true and set the output frame to NULL if the input frame is dropped. Return 62 // true and set the out frame to output_frame_ if the input frame is adapted 63 // successfully. Return false otherwise. 64 // Note that, if no adaptation is required, |out_frame| will refer directly 65 // in_frame. If a copy is always required, the caller must do an explicit 66 // copy. 67 // If a copy has taken place, |output_frame_| is owned by the VideoAdapter 68 // and will remain usable until the adapter is destroyed or AdaptFrame is 69 // called again. 70 bool AdaptFrame(VideoFrame* in_frame, VideoFrame** out_frame); 71 72 void set_scale_third(bool enable); scale_third()73 bool scale_third() const { return scale_third_; } 74 adaptation_changes()75 int adaptation_changes() const { return adaption_changes_; } 76 77 protected: 78 float FindClosestScale(int width, int height, int target_num_pixels); 79 float FindClosestViewScale(int width, int height, int target_num_pixels); 80 float FindLowerScale(int width, int height, int target_num_pixels); 81 82 private: 83 const float* GetViewScaleFactors() const; 84 float FindScale(const float* scale_factors, 85 const float upbias, int width, int height, 86 int target_num_pixels); 87 bool StretchToOutputFrame(const VideoFrame* in_frame); 88 89 VideoFormat input_format_; 90 VideoFormat output_format_; 91 int output_num_pixels_; 92 bool scale_third_; // True if adapter allows scaling to 1/3 and 2/3. 93 int frames_in_; // Number of input frames. 94 int frames_out_; // Number of output frames. 95 int frames_scaled_; // Number of frames scaled. 96 int adaption_changes_; // Number of changes in scale factor. 97 size_t previous_width_; // Previous adapter output width. 98 size_t previous_height_; // Previous adapter output height. 99 bool black_output_; // Flag to tell if we need to black output_frame_. 100 bool is_black_; // Flag to tell if output_frame_ is currently black. 101 int64 interval_next_frame_; 102 rtc::scoped_ptr<VideoFrame> output_frame_; 103 // The critical section to protect the above variables. 104 rtc::CriticalSection critical_section_; 105 106 DISALLOW_COPY_AND_ASSIGN(VideoAdapter); 107 }; 108 109 // CoordinatedVideoAdapter adapts the video input to the encoder by coordinating 110 // the format request from the server, the resolution request from the encoder, 111 // and the CPU load. 112 class CoordinatedVideoAdapter 113 : public VideoAdapter, public sigslot::has_slots<> { 114 public: 115 enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE }; 116 enum AdaptReasonEnum { 117 ADAPTREASON_NONE = 0, 118 ADAPTREASON_CPU = 1, 119 ADAPTREASON_BANDWIDTH = 2, 120 ADAPTREASON_VIEW = 4 121 }; 122 typedef int AdaptReason; 123 124 CoordinatedVideoAdapter(); ~CoordinatedVideoAdapter()125 virtual ~CoordinatedVideoAdapter() {} 126 127 virtual void SetInputFormat(const VideoFormat& format); 128 129 // Enable or disable video adaptation due to the change of the CPU load. set_cpu_adaptation(bool enable)130 void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; } cpu_adaptation()131 bool cpu_adaptation() const { return cpu_adaptation_; } 132 // Enable or disable smoothing when doing CPU adaptation. When smoothing is 133 // enabled, system CPU load is tracked using an exponential weighted 134 // average. 135 void set_cpu_smoothing(bool enable); cpu_smoothing()136 bool cpu_smoothing() const { return cpu_smoothing_; } 137 // Enable or disable video adaptation due to the change of the GD set_gd_adaptation(bool enable)138 void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; } gd_adaptation()139 bool gd_adaptation() const { return gd_adaptation_; } 140 // Enable or disable video adaptation due to the change of the View set_view_adaptation(bool enable)141 void set_view_adaptation(bool enable) { view_adaptation_ = enable; } view_adaptation()142 bool view_adaptation() const { return view_adaptation_; } 143 // Enable or disable video adaptation to fast switch View set_view_switch(bool enable)144 void set_view_switch(bool enable) { view_switch_ = enable; } view_switch()145 bool view_switch() const { return view_switch_; } 146 adapt_reason()147 CoordinatedVideoAdapter::AdaptReason adapt_reason() const { 148 return adapt_reason_; 149 } 150 151 // When the video is decreased, set the waiting time for CPU adaptation to 152 // decrease video again. 153 void set_cpu_load_min_samples(int cpu_load_min_samples); cpu_load_min_samples()154 int cpu_load_min_samples() const { return cpu_load_min_samples_; } 155 // CPU system load high threshold for reducing resolution. e.g. 0.85f 156 void set_high_system_threshold(float high_system_threshold); high_system_threshold()157 float high_system_threshold() const { return high_system_threshold_; } 158 // CPU system load low threshold for increasing resolution. e.g. 0.70f 159 void set_low_system_threshold(float low_system_threshold); low_system_threshold()160 float low_system_threshold() const { return low_system_threshold_; } 161 // CPU process load threshold for reducing resolution. e.g. 0.10f 162 void set_process_threshold(float process_threshold); process_threshold()163 float process_threshold() const { return process_threshold_; } 164 165 // Handle the format request from the server via Jingle update message. 166 void OnOutputFormatRequest(const VideoFormat& format); 167 // Handle the resolution request from the encoder due to bandwidth changes. 168 void OnEncoderResolutionRequest(int width, int height, AdaptRequest request); 169 // Handle the resolution request for CPU overuse. 170 void OnCpuResolutionRequest(AdaptRequest request); 171 // Handle the CPU load provided by a CPU monitor. 172 void OnCpuLoadUpdated(int current_cpus, int max_cpus, 173 float process_load, float system_load); 174 175 sigslot::signal0<> SignalCpuAdaptationUnable; 176 177 private: 178 // Adapt to the minimum of the formats the server requests, the CPU wants, and 179 // the encoder wants. Returns true if resolution changed. 180 bool AdaptToMinimumFormat(int* new_width, int* new_height); 181 bool IsMinimumFormat(int pixels); 182 void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request, 183 int* num_pixels); 184 CoordinatedVideoAdapter::AdaptRequest FindCpuRequest( 185 int current_cpus, int max_cpus, 186 float process_load, float system_load); 187 188 bool cpu_adaptation_; // True if cpu adaptation is enabled. 189 bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation). 190 bool gd_adaptation_; // True if gd adaptation is enabled. 191 bool view_adaptation_; // True if view adaptation is enabled. 192 bool view_switch_; // True if view switch is enabled. 193 int cpu_downgrade_count_; 194 int cpu_load_min_samples_; 195 int cpu_load_num_samples_; 196 // cpu system load thresholds relative to max cpus. 197 float high_system_threshold_; 198 float low_system_threshold_; 199 // cpu process load thresholds relative to current cpus. 200 float process_threshold_; 201 // Video formats that the server view requests, the CPU wants, and the encoder 202 // wants respectively. The adapted output format is the minimum of these. 203 int view_desired_num_pixels_; 204 int64 view_desired_interval_; 205 int encoder_desired_num_pixels_; 206 int cpu_desired_num_pixels_; 207 CoordinatedVideoAdapter::AdaptReason adapt_reason_; 208 // The critical section to protect handling requests. 209 rtc::CriticalSection request_critical_section_; 210 211 // The weighted average of cpu load over time. It's always updated (if cpu 212 // adaptation is on), but only used if cpu_smoothing_ is set. 213 float system_load_average_; 214 215 DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter); 216 }; 217 218 } // namespace cricket 219 220 #endif // TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT 221