• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * libjingle
3  * Copyright 2010 Google Inc.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  *  1. Redistributions of source code must retain the above copyright notice,
9  *     this list of conditions and the following disclaimer.
10  *  2. Redistributions in binary form must reproduce the above copyright notice,
11  *     this list of conditions and the following disclaimer in the documentation
12  *     and/or other materials provided with the distribution.
13  *  3. The name of the author may not be used to endorse or promote products
14  *     derived from this software without specific prior written permission.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #include "talk/media/base/videoadapter.h"
29 
30 #include <limits.h>  // For INT_MAX
31 #include <algorithm>
32 
33 #include "talk/media/base/constants.h"
34 #include "talk/media/base/videocommon.h"
35 #include "talk/media/base/videoframe.h"
36 #include "webrtc/base/logging.h"
37 #include "webrtc/base/timeutils.h"
38 
39 namespace cricket {
40 
41 // TODO(fbarchard): Make downgrades settable
42 static const int kMaxCpuDowngrades = 2;  // Downgrade at most 2 times for CPU.
43 // The number of cpu samples to require before adapting. This value depends on
44 // the cpu monitor sampling frequency being 2000ms.
45 static const int kCpuLoadMinSamples = 3;
46 // The amount of weight to give to each new cpu load sample. The lower the
47 // value, the slower we'll adapt to changing cpu conditions.
48 static const float kCpuLoadWeightCoefficient = 0.4f;
49 // The seed value for the cpu load moving average.
50 static const float kCpuLoadInitialAverage = 0.5f;
51 
52 // Desktop needs 1/8 scale for HD (1280 x 720) to QQVGA (160 x 90)
53 static const float kScaleFactors[] = {
54   1.f / 1.f,   // Full size.
55   3.f / 4.f,   // 3/4 scale.
56   1.f / 2.f,   // 1/2 scale.
57   3.f / 8.f,   // 3/8 scale.
58   1.f / 4.f,   // 1/4 scale.
59   3.f / 16.f,  // 3/16 scale.
60   1.f / 8.f,   // 1/8 scale.
61   0.f  // End of table.
62 };
63 
64 // TODO(fbarchard): Use this table (optionally) for CPU and GD as well.
65 static const float kViewScaleFactors[] = {
66   1.f / 1.f,   // Full size.
67   3.f / 4.f,   // 3/4 scale.
68   2.f / 3.f,   // 2/3 scale.  // Allow 1080p to 720p.
69   1.f / 2.f,   // 1/2 scale.
70   3.f / 8.f,   // 3/8 scale.
71   1.f / 3.f,   // 1/3 scale.  // Allow 1080p to 360p.
72   1.f / 4.f,   // 1/4 scale.
73   3.f / 16.f,  // 3/16 scale.
74   1.f / 8.f,   // 1/8 scale.
75   0.f  // End of table.
76 };
77 
GetViewScaleFactors() const78 const float* VideoAdapter::GetViewScaleFactors() const {
79   return scale_third_ ? kViewScaleFactors : kScaleFactors;
80 }
81 
82 // For resolutions that would scale down a little instead of up a little,
83 // bias toward scaling up a little.  This will tend to choose 3/4 scale instead
84 // of 2/3 scale, when the 2/3 is not an exact match.
85 static const float kUpBias = -0.9f;
86 // Find the scale factor that, when applied to width and height, is closest
87 // to num_pixels.
FindScale(const float * scale_factors,const float upbias,int width,int height,int target_num_pixels)88 float VideoAdapter::FindScale(const float* scale_factors,
89                               const float upbias,
90                               int width, int height,
91                               int target_num_pixels) {
92   const float kMinNumPixels = 160 * 90;
93   if (!target_num_pixels) {
94     return 0.f;
95   }
96   float best_distance = static_cast<float>(INT_MAX);
97   float best_scale = 1.f;  // Default to unscaled if nothing matches.
98   float pixels = static_cast<float>(width * height);
99   for (int i = 0; ; ++i) {
100     float scale = scale_factors[i];
101     float test_num_pixels = pixels * scale * scale;
102     // Do not consider scale factors that produce too small images.
103     // Scale factor of 0 at end of table will also exit here.
104     if (test_num_pixels < kMinNumPixels) {
105       break;
106     }
107     float diff = target_num_pixels - test_num_pixels;
108     // If resolution is higher than desired, bias the difference based on
109     // preference for slightly larger for nearest, or avoid completely if
110     // looking for lower resolutions only.
111     if (diff < 0) {
112       diff = diff * kUpBias;
113     }
114     if (diff < best_distance) {
115       best_distance = diff;
116       best_scale = scale;
117       if (best_distance == 0) {  // Found exact match.
118         break;
119       }
120     }
121   }
122   return best_scale;
123 }
124 
125 // Find the closest scale factor.
FindClosestScale(int width,int height,int target_num_pixels)126 float VideoAdapter::FindClosestScale(int width, int height,
127                                          int target_num_pixels) {
128   return FindScale(kScaleFactors, kUpBias,
129                    width, height, target_num_pixels);
130 }
131 
132 // Find the closest view scale factor.
FindClosestViewScale(int width,int height,int target_num_pixels)133 float VideoAdapter::FindClosestViewScale(int width, int height,
134                                          int target_num_pixels) {
135   return FindScale(GetViewScaleFactors(), kUpBias,
136                    width, height, target_num_pixels);
137 }
138 
139 // Finds the scale factor that, when applied to width and height, produces
140 // fewer than num_pixels.
141 static const float kUpAvoidBias = -1000000000.f;
FindLowerScale(int width,int height,int target_num_pixels)142 float VideoAdapter::FindLowerScale(int width, int height,
143                                    int target_num_pixels) {
144   return FindScale(GetViewScaleFactors(), kUpAvoidBias,
145                    width, height, target_num_pixels);
146 }
147 
148 // There are several frame sizes used by Adapter.  This explains them
149 // input_format - set once by server to frame size expected from the camera.
150 //   The input frame size is also updated in AdaptFrameResolution.
151 // output_format - size that output would like to be.  Includes framerate.
152 //   The output frame size is also updated in AdaptFrameResolution.
153 // output_num_pixels - size that output should be constrained to.  Used to
154 //   compute output_format from in_frame.
155 // in_frame - actual camera captured frame size, which is typically the same
156 //   as input_format.  This can also be rotated or cropped for aspect ratio.
157 // out_frame - actual frame output by adapter.  Should be a direct scale of
158 //   in_frame maintaining rotation and aspect ratio.
159 // OnOutputFormatRequest - server requests you send this resolution based on
160 //   view requests.
161 // OnEncoderResolutionRequest - encoder requests you send this resolution based
162 //   on bandwidth
163 // OnCpuLoadUpdated - cpu monitor requests you send this resolution based on
164 //   cpu load.
165 
166 ///////////////////////////////////////////////////////////////////////
167 // Implementation of VideoAdapter
VideoAdapter()168 VideoAdapter::VideoAdapter()
169     : output_num_pixels_(INT_MAX),
170       scale_third_(false),
171       frames_in_(0),
172       frames_out_(0),
173       frames_scaled_(0),
174       adaption_changes_(0),
175       previous_width_(0),
176       previous_height_(0),
177       interval_next_frame_(0) {
178 }
179 
~VideoAdapter()180 VideoAdapter::~VideoAdapter() {
181 }
182 
SetInputFormat(const VideoFormat & format)183 void VideoAdapter::SetInputFormat(const VideoFormat& format) {
184   rtc::CritScope cs(&critical_section_);
185   int64_t old_input_interval = input_format_.interval;
186   input_format_ = format;
187   output_format_.interval =
188       std::max(output_format_.interval, input_format_.interval);
189   if (old_input_interval != input_format_.interval) {
190     LOG(LS_INFO) << "VAdapt input interval changed from "
191       << old_input_interval << " to " << input_format_.interval;
192   }
193 }
194 
SetInputFormat(const VideoFormat & format)195 void CoordinatedVideoAdapter::SetInputFormat(const VideoFormat& format) {
196   int previous_width = input_format().width;
197   int previous_height = input_format().height;
198   bool is_resolution_change = previous_width > 0 && format.width > 0 &&
199                               (previous_width != format.width ||
200                                previous_height != format.height);
201   VideoAdapter::SetInputFormat(format);
202   if (is_resolution_change) {
203     int width, height;
204     // Trigger the adaptation logic again, to potentially reset the adaptation
205     // state for things like view requests that may not longer be capping
206     // output (or may now cap output).
207     AdaptToMinimumFormat(&width, &height);
208     LOG(LS_INFO) << "VAdapt Input Resolution Change: "
209                  << "Previous input resolution: "
210                  << previous_width << "x" << previous_height
211                  << " New input resolution: "
212                  << format.width << "x" << format.height
213                  << " New output resolution: "
214                  << width << "x" << height;
215   }
216 }
217 
set_cpu_smoothing(bool enable)218 void CoordinatedVideoAdapter::set_cpu_smoothing(bool enable) {
219   LOG(LS_INFO) << "CPU smoothing is now "
220                << (enable ? "enabled" : "disabled");
221   cpu_smoothing_ = enable;
222 }
223 
SetOutputFormat(const VideoFormat & format)224 void VideoAdapter::SetOutputFormat(const VideoFormat& format) {
225   rtc::CritScope cs(&critical_section_);
226   int64_t old_output_interval = output_format_.interval;
227   output_format_ = format;
228   output_num_pixels_ = output_format_.width * output_format_.height;
229   output_format_.interval =
230       std::max(output_format_.interval, input_format_.interval);
231   if (old_output_interval != output_format_.interval) {
232     LOG(LS_INFO) << "VAdapt output interval changed from "
233       << old_output_interval << " to " << output_format_.interval;
234   }
235 }
236 
input_format()237 const VideoFormat& VideoAdapter::input_format() {
238   rtc::CritScope cs(&critical_section_);
239   return input_format_;
240 }
241 
drops_all_frames() const242 bool VideoAdapter::drops_all_frames() const {
243   return output_num_pixels_ == 0;
244 }
245 
output_format()246 const VideoFormat& VideoAdapter::output_format() {
247   rtc::CritScope cs(&critical_section_);
248   return output_format_;
249 }
250 
251 // Constrain output resolution to this many pixels overall
SetOutputNumPixels(int num_pixels)252 void VideoAdapter::SetOutputNumPixels(int num_pixels) {
253   output_num_pixels_ = num_pixels;
254 }
255 
GetOutputNumPixels() const256 int VideoAdapter::GetOutputNumPixels() const {
257   return output_num_pixels_;
258 }
259 
AdaptFrameResolution(int in_width,int in_height)260 VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) {
261   rtc::CritScope cs(&critical_section_);
262   ++frames_in_;
263 
264   SetInputFormat(VideoFormat(
265       in_width, in_height, input_format_.interval, input_format_.fourcc));
266 
267   // Drop the input frame if necessary.
268   bool should_drop = false;
269   if (!output_num_pixels_) {
270     // Drop all frames as the output format is 0x0.
271     should_drop = true;
272   } else {
273     // Drop some frames based on input fps and output fps.
274     // Normally output fps is less than input fps.
275     // TODO(fbarchard): Consider adjusting interval to reflect the adjusted
276     // interval between frames after dropping some frames.
277     interval_next_frame_ += input_format_.interval;
278     if (output_format_.interval > 0) {
279       if (interval_next_frame_ >= output_format_.interval) {
280         interval_next_frame_ %= output_format_.interval;
281       } else {
282         should_drop = true;
283       }
284     }
285   }
286   if (should_drop) {
287     // Show VAdapt log every 90 frames dropped. (3 seconds)
288     if ((frames_in_ - frames_out_) % 90 == 0) {
289       // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
290       // in default calls.
291       LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_
292                    << " / out " << frames_out_
293                    << " / in " << frames_in_
294                    << " Changes: " << adaption_changes_
295                    << " Input: " << in_width
296                    << "x" << in_height
297                    << " i" << input_format_.interval
298                    << " Output: i" << output_format_.interval;
299     }
300 
301     return VideoFormat();  // Drop frame.
302   }
303 
304   const float scale = VideoAdapter::FindClosestViewScale(
305       in_width, in_height, output_num_pixels_);
306   const int output_width = static_cast<int>(in_width * scale + .5f);
307   const int output_height = static_cast<int>(in_height * scale + .5f);
308 
309   ++frames_out_;
310   if (scale != 1)
311     ++frames_scaled_;
312   // Show VAdapt log every 90 frames output. (3 seconds)
313   // TODO(fbarchard): Consider GetLogSeverity() to change interval to less
314   // for LS_VERBOSE and more for LS_INFO.
315   bool show = (frames_out_) % 90 == 0;
316 
317   // TODO(fbarchard): LOG the previous output resolution and track input
318   // resolution changes as well.  Consider dropping the statistics into their
319   // own class which could be queried publically.
320   bool changed = false;
321   if (previous_width_ && (previous_width_ != output_width ||
322                           previous_height_ != output_height)) {
323     show = true;
324     ++adaption_changes_;
325     changed = true;
326   }
327   if (show) {
328     // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
329     // in default calls.
330     LOG(LS_INFO) << "VAdapt Frame: scaled " << frames_scaled_
331                  << " / out " << frames_out_
332                  << " / in " << frames_in_
333                  << " Changes: " << adaption_changes_
334                  << " Input: " << in_width
335                  << "x" << in_height
336                  << " i" << input_format_.interval
337                  << " Scale: " << scale
338                  << " Output: " << output_width
339                  << "x" << output_height
340                  << " i" << output_format_.interval
341                  << " Changed: " << (changed ? "true" : "false");
342   }
343 
344   output_format_.width = output_width;
345   output_format_.height = output_height;
346   previous_width_ = output_width;
347   previous_height_ = output_height;
348 
349   return output_format_;
350 }
351 
set_scale_third(bool enable)352 void VideoAdapter::set_scale_third(bool enable) {
353   LOG(LS_INFO) << "Video Adapter third scaling is now "
354                << (enable ? "enabled" : "disabled");
355   scale_third_ = enable;
356 }
357 
358 ///////////////////////////////////////////////////////////////////////
359 // Implementation of CoordinatedVideoAdapter
CoordinatedVideoAdapter()360 CoordinatedVideoAdapter::CoordinatedVideoAdapter()
361     : cpu_adaptation_(true),
362       cpu_smoothing_(false),
363       gd_adaptation_(true),
364       view_adaptation_(true),
365       view_switch_(false),
366       cpu_downgrade_count_(0),
367       cpu_load_min_samples_(kCpuLoadMinSamples),
368       cpu_load_num_samples_(0),
369       high_system_threshold_(kHighSystemCpuThreshold),
370       low_system_threshold_(kLowSystemCpuThreshold),
371       process_threshold_(kProcessCpuThreshold),
372       view_desired_num_pixels_(INT_MAX),
373       view_desired_interval_(0),
374       encoder_desired_num_pixels_(INT_MAX),
375       cpu_desired_num_pixels_(INT_MAX),
376       adapt_reason_(ADAPTREASON_NONE),
377       system_load_average_(kCpuLoadInitialAverage) {
378 }
379 
380 // Helper function to UPGRADE or DOWNGRADE a number of pixels
StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,int * num_pixels)381 void CoordinatedVideoAdapter::StepPixelCount(
382     CoordinatedVideoAdapter::AdaptRequest request,
383     int* num_pixels) {
384   switch (request) {
385     case CoordinatedVideoAdapter::DOWNGRADE:
386       *num_pixels /= 2;
387       break;
388 
389     case CoordinatedVideoAdapter::UPGRADE:
390       *num_pixels *= 2;
391       break;
392 
393     default:  // No change in pixel count
394       break;
395   }
396   return;
397 }
398 
399 // Find the adaptation request of the cpu based on the load. Return UPGRADE if
400 // the load is low, DOWNGRADE if the load is high, and KEEP otherwise.
FindCpuRequest(int current_cpus,int max_cpus,float process_load,float system_load)401 CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest(
402     int current_cpus, int max_cpus,
403     float process_load, float system_load) {
404   // Downgrade if system is high and plugin is at least more than midrange.
405   if (system_load >= high_system_threshold_ * max_cpus &&
406       process_load >= process_threshold_ * current_cpus) {
407     return CoordinatedVideoAdapter::DOWNGRADE;
408   // Upgrade if system is low.
409   } else if (system_load < low_system_threshold_ * max_cpus) {
410     return CoordinatedVideoAdapter::UPGRADE;
411   }
412   return CoordinatedVideoAdapter::KEEP;
413 }
414 
415 // A remote view request for a new resolution.
OnOutputFormatRequest(const VideoFormat & format)416 void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
417   rtc::CritScope cs(&request_critical_section_);
418   if (!view_adaptation_) {
419     return;
420   }
421   // Set output for initial aspect ratio in mediachannel unittests.
422   int old_num_pixels = GetOutputNumPixels();
423   SetOutputFormat(format);
424   SetOutputNumPixels(old_num_pixels);
425   view_desired_num_pixels_ = format.width * format.height;
426   view_desired_interval_ = format.interval;
427   int new_width, new_height;
428   bool changed = AdaptToMinimumFormat(&new_width, &new_height);
429   LOG(LS_INFO) << "VAdapt View Request: "
430                << format.width << "x" << format.height
431                << " Pixels: " << view_desired_num_pixels_
432                << " Changed: " << (changed ? "true" : "false")
433                << " To: " << new_width << "x" << new_height;
434 }
435 
set_cpu_load_min_samples(int cpu_load_min_samples)436 void CoordinatedVideoAdapter::set_cpu_load_min_samples(
437     int cpu_load_min_samples) {
438   if (cpu_load_min_samples_ != cpu_load_min_samples) {
439     LOG(LS_INFO) << "VAdapt Change Cpu Adapt Min Samples from: "
440                  << cpu_load_min_samples_ << " to "
441                  << cpu_load_min_samples;
442     cpu_load_min_samples_ = cpu_load_min_samples;
443   }
444 }
445 
set_high_system_threshold(float high_system_threshold)446 void CoordinatedVideoAdapter::set_high_system_threshold(
447     float high_system_threshold) {
448   ASSERT(high_system_threshold <= 1.0f);
449   ASSERT(high_system_threshold >= 0.0f);
450   if (high_system_threshold_ != high_system_threshold) {
451     LOG(LS_INFO) << "VAdapt Change High System Threshold from: "
452                  << high_system_threshold_ << " to " << high_system_threshold;
453     high_system_threshold_ = high_system_threshold;
454   }
455 }
456 
set_low_system_threshold(float low_system_threshold)457 void CoordinatedVideoAdapter::set_low_system_threshold(
458     float low_system_threshold) {
459   ASSERT(low_system_threshold <= 1.0f);
460   ASSERT(low_system_threshold >= 0.0f);
461   if (low_system_threshold_ != low_system_threshold) {
462     LOG(LS_INFO) << "VAdapt Change Low System Threshold from: "
463                  << low_system_threshold_ << " to " << low_system_threshold;
464     low_system_threshold_ = low_system_threshold;
465   }
466 }
467 
set_process_threshold(float process_threshold)468 void CoordinatedVideoAdapter::set_process_threshold(float process_threshold) {
469   ASSERT(process_threshold <= 1.0f);
470   ASSERT(process_threshold >= 0.0f);
471   if (process_threshold_ != process_threshold) {
472     LOG(LS_INFO) << "VAdapt Change High Process Threshold from: "
473                  << process_threshold_ << " to " << process_threshold;
474     process_threshold_ = process_threshold;
475   }
476 }
477 
478 // A Bandwidth GD request for new resolution
OnEncoderResolutionRequest(int width,int height,AdaptRequest request)479 void CoordinatedVideoAdapter::OnEncoderResolutionRequest(
480     int width, int height, AdaptRequest request) {
481   rtc::CritScope cs(&request_critical_section_);
482   if (!gd_adaptation_) {
483     return;
484   }
485   int old_encoder_desired_num_pixels = encoder_desired_num_pixels_;
486   if (KEEP != request) {
487     int new_encoder_desired_num_pixels = width * height;
488     int old_num_pixels = GetOutputNumPixels();
489     if (new_encoder_desired_num_pixels != old_num_pixels) {
490       LOG(LS_VERBOSE) << "VAdapt GD resolution stale.  Ignored";
491     } else {
492       // Update the encoder desired format based on the request.
493       encoder_desired_num_pixels_ = new_encoder_desired_num_pixels;
494       StepPixelCount(request, &encoder_desired_num_pixels_);
495     }
496   }
497   int new_width, new_height;
498   bool changed = AdaptToMinimumFormat(&new_width, &new_height);
499 
500   // Ignore up or keep if no change.
501   if (DOWNGRADE != request && view_switch_ && !changed) {
502     encoder_desired_num_pixels_ = old_encoder_desired_num_pixels;
503     LOG(LS_VERBOSE) << "VAdapt ignoring GD request.";
504   }
505 
506   LOG(LS_INFO) << "VAdapt GD Request: "
507                << (DOWNGRADE == request ? "down" :
508                    (UPGRADE == request ? "up" : "keep"))
509                << " From: " << width << "x" << height
510                << " Pixels: " << encoder_desired_num_pixels_
511                << " Changed: " << (changed ? "true" : "false")
512                << " To: " << new_width << "x" << new_height;
513 }
514 
515 // A Bandwidth GD request for new resolution
OnCpuResolutionRequest(AdaptRequest request)516 void CoordinatedVideoAdapter::OnCpuResolutionRequest(AdaptRequest request) {
517   rtc::CritScope cs(&request_critical_section_);
518   if (!cpu_adaptation_) {
519     return;
520   }
521   // Update how many times we have downgraded due to the cpu load.
522   switch (request) {
523     case DOWNGRADE:
524       // Ignore downgrades if we have downgraded the maximum times.
525       if (cpu_downgrade_count_ < kMaxCpuDowngrades) {
526         ++cpu_downgrade_count_;
527       } else {
528         LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade "
529                            "because maximum downgrades reached";
530         SignalCpuAdaptationUnable();
531       }
532       break;
533     case UPGRADE:
534       if (cpu_downgrade_count_ > 0) {
535         bool is_min = IsMinimumFormat(cpu_desired_num_pixels_);
536         if (is_min) {
537           --cpu_downgrade_count_;
538         } else {
539           LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
540                              "because cpu is not limiting resolution";
541         }
542       } else {
543         LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
544                            "because minimum downgrades reached";
545       }
546       break;
547     case KEEP:
548     default:
549       break;
550   }
551   if (KEEP != request) {
552     // TODO(fbarchard): compute stepping up/down from OutputNumPixels but
553     // clamp to inputpixels / 4 (2 steps)
554     cpu_desired_num_pixels_ =  cpu_downgrade_count_ == 0 ? INT_MAX :
555         static_cast<int>(input_format().width * input_format().height >>
556                          cpu_downgrade_count_);
557   }
558   int new_width, new_height;
559   bool changed = AdaptToMinimumFormat(&new_width, &new_height);
560   LOG(LS_INFO) << "VAdapt CPU Request: "
561                << (DOWNGRADE == request ? "down" :
562                    (UPGRADE == request ? "up" : "keep"))
563                << " Steps: " << cpu_downgrade_count_
564                << " Changed: " << (changed ? "true" : "false")
565                << " To: " << new_width << "x" << new_height;
566 }
567 
568 // A CPU request for new resolution
569 // TODO(fbarchard): Move outside adapter.
OnCpuLoadUpdated(int current_cpus,int max_cpus,float process_load,float system_load)570 void CoordinatedVideoAdapter::OnCpuLoadUpdated(
571     int current_cpus, int max_cpus, float process_load, float system_load) {
572   rtc::CritScope cs(&request_critical_section_);
573   if (!cpu_adaptation_) {
574     return;
575   }
576   // Update the moving average of system load. Even if we aren't smoothing,
577   // we'll still calculate this information, in case smoothing is later enabled.
578   system_load_average_ = kCpuLoadWeightCoefficient * system_load +
579       (1.0f - kCpuLoadWeightCoefficient) * system_load_average_;
580   ++cpu_load_num_samples_;
581   if (cpu_smoothing_) {
582     system_load = system_load_average_;
583   }
584   AdaptRequest request = FindCpuRequest(current_cpus, max_cpus,
585                                         process_load, system_load);
586   // Make sure we're not adapting too quickly.
587   if (request != KEEP) {
588     if (cpu_load_num_samples_ < cpu_load_min_samples_) {
589       LOG(LS_VERBOSE) << "VAdapt CPU load high/low but do not adapt until "
590                       << (cpu_load_min_samples_ - cpu_load_num_samples_)
591                       << " more samples";
592       request = KEEP;
593     }
594   }
595 
596   OnCpuResolutionRequest(request);
597 }
598 
599 // Called by cpu adapter on up requests.
IsMinimumFormat(int pixels)600 bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) {
601   // Find closest scale factor that matches input resolution to min_num_pixels
602   // and set that for output resolution.  This is not needed for VideoAdapter,
603   // but provides feedback to unittests and users on expected resolution.
604   // Actual resolution is based on input frame.
605   VideoFormat new_output = output_format();
606   VideoFormat input = input_format();
607   if (input_format().IsSize0x0()) {
608     input = new_output;
609   }
610   float scale = 1.0f;
611   if (!input.IsSize0x0()) {
612     scale = FindClosestScale(input.width,
613                              input.height,
614                              pixels);
615   }
616   new_output.width = static_cast<int>(input.width * scale + .5f);
617   new_output.height = static_cast<int>(input.height * scale + .5f);
618   int new_pixels = new_output.width * new_output.height;
619   int num_pixels = GetOutputNumPixels();
620   return new_pixels <= num_pixels;
621 }
622 
623 // Called by all coordinators when there is a change.
AdaptToMinimumFormat(int * new_width,int * new_height)624 bool CoordinatedVideoAdapter::AdaptToMinimumFormat(int* new_width,
625                                                    int* new_height) {
626   VideoFormat new_output = output_format();
627   VideoFormat input = input_format();
628   if (input_format().IsSize0x0()) {
629     input = new_output;
630   }
631   int old_num_pixels = GetOutputNumPixels();
632   int min_num_pixels = INT_MAX;
633   adapt_reason_ = ADAPTREASON_NONE;
634 
635   // Reduce resolution based on encoder bandwidth (GD).
636   if (encoder_desired_num_pixels_ &&
637       (encoder_desired_num_pixels_ < min_num_pixels)) {
638     adapt_reason_ |= ADAPTREASON_BANDWIDTH;
639     min_num_pixels = encoder_desired_num_pixels_;
640   }
641   // Reduce resolution based on CPU.
642   if (cpu_adaptation_ && cpu_desired_num_pixels_ &&
643       (cpu_desired_num_pixels_ <= min_num_pixels)) {
644     if (cpu_desired_num_pixels_ < min_num_pixels) {
645       adapt_reason_ = ADAPTREASON_CPU;
646     } else {
647       adapt_reason_ |= ADAPTREASON_CPU;
648     }
649     min_num_pixels = cpu_desired_num_pixels_;
650   }
651   // Round resolution for GD or CPU to allow 1/2 to map to 9/16.
652   if (!input.IsSize0x0() && min_num_pixels != INT_MAX) {
653     float scale = FindClosestScale(input.width, input.height, min_num_pixels);
654     min_num_pixels = static_cast<int>(input.width * scale + .5f) *
655         static_cast<int>(input.height * scale + .5f);
656   }
657   // Reduce resolution based on View Request.
658   if (view_desired_num_pixels_ <= min_num_pixels) {
659     if (view_desired_num_pixels_ < min_num_pixels) {
660       adapt_reason_ = ADAPTREASON_VIEW;
661     } else {
662       adapt_reason_ |= ADAPTREASON_VIEW;
663     }
664     min_num_pixels = view_desired_num_pixels_;
665   }
666   // Snap to a scale factor.
667   float scale = 1.0f;
668   if (!input.IsSize0x0()) {
669     scale = FindLowerScale(input.width, input.height, min_num_pixels);
670     min_num_pixels = static_cast<int>(input.width * scale + .5f) *
671         static_cast<int>(input.height * scale + .5f);
672   }
673   if (scale == 1.0f) {
674     adapt_reason_ = ADAPTREASON_NONE;
675   }
676   *new_width = new_output.width = static_cast<int>(input.width * scale + .5f);
677   *new_height = new_output.height = static_cast<int>(input.height * scale +
678                                                      .5f);
679   SetOutputNumPixels(min_num_pixels);
680 
681   new_output.interval = view_desired_interval_;
682   SetOutputFormat(new_output);
683   int new_num_pixels = GetOutputNumPixels();
684   bool changed = new_num_pixels != old_num_pixels;
685 
686   static const char* kReasons[8] = {
687     "None",
688     "CPU",
689     "BANDWIDTH",
690     "CPU+BANDWIDTH",
691     "VIEW",
692     "CPU+VIEW",
693     "BANDWIDTH+VIEW",
694     "CPU+BANDWIDTH+VIEW",
695   };
696 
697   LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_
698                   << " GD: " << encoder_desired_num_pixels_
699                   << " CPU: " << cpu_desired_num_pixels_
700                   << " Pixels: " << min_num_pixels
701                   << " Input: " << input.width
702                   << "x" << input.height
703                   << " Scale: " << scale
704                   << " Resolution: " << new_output.width
705                   << "x" << new_output.height
706                   << " Changed: " << (changed ? "true" : "false")
707                   << " Reason: " << kReasons[adapt_reason_];
708 
709   if (changed) {
710     // When any adaptation occurs, historic CPU load levels are no longer
711     // accurate. Clear out our state so we can re-learn at the new normal.
712     cpu_load_num_samples_ = 0;
713     system_load_average_ = kCpuLoadInitialAverage;
714   }
715 
716   return changed;
717 }
718 
719 }  // namespace cricket
720