1 // libjingle
2 // Copyright 2010 Google Inc.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // 1. Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // 2. Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // 3. The name of the author may not be used to endorse or promote products
13 // derived from this software without specific prior written permission.
14 //
15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
16 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
17 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
18 // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
24 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 #include "talk/media/base/videoadapter.h"
27
28 #include <limits.h> // For INT_MAX
29
30 #include "talk/base/logging.h"
31 #include "talk/base/timeutils.h"
32 #include "talk/media/base/constants.h"
33 #include "talk/media/base/videoframe.h"
34
35 namespace cricket {
36
37 // TODO(fbarchard): Make downgrades settable
38 static const int kMaxCpuDowngrades = 2; // Downgrade at most 2 times for CPU.
39 // The number of milliseconds of data to require before acting on cpu sampling
40 // information.
41 static const size_t kCpuLoadMinSampleTime = 5000;
42 // The amount of weight to give to each new cpu load sample. The lower the
43 // value, the slower we'll adapt to changing cpu conditions.
44 static const float kCpuLoadWeightCoefficient = 0.4f;
45 // The seed value for the cpu load moving average.
46 static const float kCpuLoadInitialAverage = 0.5f;
47
48 // Desktop needs 1/8 scale for HD (1280 x 720) to QQVGA (160 x 90)
49 static const float kScaleFactors[] = {
50 1.f / 1.f, // Full size.
51 3.f / 4.f, // 3/4 scale.
52 1.f / 2.f, // 1/2 scale.
53 3.f / 8.f, // 3/8 scale.
54 1.f / 4.f, // 1/4 scale.
55 3.f / 16.f, // 3/16 scale.
56 1.f / 8.f, // 1/8 scale.
57 0.f // End of table.
58 };
59
60 // TODO(fbarchard): Use this table (optionally) for CPU and GD as well.
61 static const float kViewScaleFactors[] = {
62 1.f / 1.f, // Full size.
63 3.f / 4.f, // 3/4 scale.
64 2.f / 3.f, // 2/3 scale. // Allow 1080p to 720p.
65 1.f / 2.f, // 1/2 scale.
66 3.f / 8.f, // 3/8 scale.
67 1.f / 3.f, // 1/3 scale. // Allow 1080p to 360p.
68 1.f / 4.f, // 1/4 scale.
69 3.f / 16.f, // 3/16 scale.
70 1.f / 8.f, // 1/8 scale.
71 0.f // End of table.
72 };
73
GetViewScaleFactors() const74 const float* VideoAdapter::GetViewScaleFactors() const {
75 return scale_third_ ? kViewScaleFactors : kScaleFactors;
76 }
77
78 // For resolutions that would scale down a little instead of up a little,
79 // bias toward scaling up a little. This will tend to choose 3/4 scale instead
80 // of 2/3 scale, when the 2/3 is not an exact match.
81 static const float kUpBias = -0.9f;
82 // Find the scale factor that, when applied to width and height, is closest
83 // to num_pixels.
FindScale(const float * scale_factors,const float upbias,int width,int height,int target_num_pixels)84 float VideoAdapter::FindScale(const float* scale_factors,
85 const float upbias,
86 int width, int height,
87 int target_num_pixels) {
88 const float kMinNumPixels = 160 * 90;
89 if (!target_num_pixels) {
90 return 0.f;
91 }
92 float best_distance = static_cast<float>(INT_MAX);
93 float best_scale = 1.f; // Default to unscaled if nothing matches.
94 float pixels = static_cast<float>(width * height);
95 for (int i = 0; ; ++i) {
96 float scale = scale_factors[i];
97 float test_num_pixels = pixels * scale * scale;
98 // Do not consider scale factors that produce too small images.
99 // Scale factor of 0 at end of table will also exit here.
100 if (test_num_pixels < kMinNumPixels) {
101 break;
102 }
103 float diff = target_num_pixels - test_num_pixels;
104 // If resolution is higher than desired, bias the difference based on
105 // preference for slightly larger for nearest, or avoid completely if
106 // looking for lower resolutions only.
107 if (diff < 0) {
108 diff = diff * kUpBias;
109 }
110 if (diff < best_distance) {
111 best_distance = diff;
112 best_scale = scale;
113 if (best_distance == 0) { // Found exact match.
114 break;
115 }
116 }
117 }
118 return best_scale;
119 }
120
121 // Find the closest scale factor.
FindClosestScale(int width,int height,int target_num_pixels)122 float VideoAdapter::FindClosestScale(int width, int height,
123 int target_num_pixels) {
124 return FindScale(kScaleFactors, kUpBias,
125 width, height, target_num_pixels);
126 }
127
128 // Find the closest view scale factor.
FindClosestViewScale(int width,int height,int target_num_pixels)129 float VideoAdapter::FindClosestViewScale(int width, int height,
130 int target_num_pixels) {
131 return FindScale(GetViewScaleFactors(), kUpBias,
132 width, height, target_num_pixels);
133 }
134
135 // Finds the scale factor that, when applied to width and height, produces
136 // fewer than num_pixels.
137 static const float kUpAvoidBias = -1000000000.f;
FindLowerScale(int width,int height,int target_num_pixels)138 float VideoAdapter::FindLowerScale(int width, int height,
139 int target_num_pixels) {
140 return FindScale(GetViewScaleFactors(), kUpAvoidBias,
141 width, height, target_num_pixels);
142 }
143
144 // There are several frame sizes used by Adapter. This explains them
145 // input_format - set once by server to frame size expected from the camera.
146 // output_format - size that output would like to be. Includes framerate.
147 // output_num_pixels - size that output should be constrained to. Used to
148 // compute output_format from in_frame.
149 // in_frame - actual camera captured frame size, which is typically the same
150 // as input_format. This can also be rotated or cropped for aspect ratio.
151 // out_frame - actual frame output by adapter. Should be a direct scale of
152 // in_frame maintaining rotation and aspect ratio.
153 // OnOutputFormatRequest - server requests you send this resolution based on
154 // view requests.
155 // OnEncoderResolutionRequest - encoder requests you send this resolution based
156 // on bandwidth
157 // OnCpuLoadUpdated - cpu monitor requests you send this resolution based on
158 // cpu load.
159
160 ///////////////////////////////////////////////////////////////////////
161 // Implementation of VideoAdapter
VideoAdapter()162 VideoAdapter::VideoAdapter()
163 : output_num_pixels_(INT_MAX),
164 scale_third_(false),
165 frames_(0),
166 adapted_frames_(0),
167 adaption_changes_(0),
168 previous_width_(0),
169 previous_height_(0),
170 black_output_(false),
171 is_black_(false),
172 interval_next_frame_(0) {
173 }
174
~VideoAdapter()175 VideoAdapter::~VideoAdapter() {
176 }
177
SetInputFormat(const VideoFormat & format)178 void VideoAdapter::SetInputFormat(const VideoFormat& format) {
179 talk_base::CritScope cs(&critical_section_);
180 input_format_ = format;
181 output_format_.interval = talk_base::_max(
182 output_format_.interval, input_format_.interval);
183 }
184
SetInputFormat(const VideoFormat & format)185 void CoordinatedVideoAdapter::SetInputFormat(const VideoFormat& format) {
186 int previous_width = input_format().width;
187 int previous_height = input_format().height;
188 bool is_resolution_change = previous_width > 0 && format.width > 0 &&
189 (previous_width != format.width ||
190 previous_height != format.height);
191 VideoAdapter::SetInputFormat(format);
192 if (is_resolution_change) {
193 int width, height;
194 // Trigger the adaptation logic again, to potentially reset the adaptation
195 // state for things like view requests that may not longer be capping
196 // output (or may now cap output).
197 AdaptToMinimumFormat(&width, &height);
198 LOG(LS_INFO) << "VAdapt Input Resolution Change: "
199 << "Previous input resolution: "
200 << previous_width << "x" << previous_height
201 << " New input resolution: "
202 << format.width << "x" << format.height
203 << " New output resolution: "
204 << width << "x" << height;
205 }
206 }
207
SetOutputFormat(const VideoFormat & format)208 void VideoAdapter::SetOutputFormat(const VideoFormat& format) {
209 talk_base::CritScope cs(&critical_section_);
210 output_format_ = format;
211 output_num_pixels_ = output_format_.width * output_format_.height;
212 output_format_.interval = talk_base::_max(
213 output_format_.interval, input_format_.interval);
214 }
215
input_format()216 const VideoFormat& VideoAdapter::input_format() {
217 talk_base::CritScope cs(&critical_section_);
218 return input_format_;
219 }
220
output_format()221 const VideoFormat& VideoAdapter::output_format() {
222 talk_base::CritScope cs(&critical_section_);
223 return output_format_;
224 }
225
SetBlackOutput(bool black)226 void VideoAdapter::SetBlackOutput(bool black) {
227 talk_base::CritScope cs(&critical_section_);
228 black_output_ = black;
229 }
230
231 // Constrain output resolution to this many pixels overall
SetOutputNumPixels(int num_pixels)232 void VideoAdapter::SetOutputNumPixels(int num_pixels) {
233 output_num_pixels_ = num_pixels;
234 }
235
GetOutputNumPixels() const236 int VideoAdapter::GetOutputNumPixels() const {
237 return output_num_pixels_;
238 }
239
240 // TODO(fbarchard): Add AdaptFrameRate function that only drops frames but
241 // not resolution.
AdaptFrame(const VideoFrame * in_frame,VideoFrame ** out_frame)242 bool VideoAdapter::AdaptFrame(const VideoFrame* in_frame,
243 VideoFrame** out_frame) {
244 if (!in_frame || !out_frame) {
245 return false;
246 }
247 talk_base::CritScope cs(&critical_section_);
248 ++frames_;
249
250 // Update input to actual frame dimensions.
251 VideoFormat format(static_cast<int>(in_frame->GetWidth()),
252 static_cast<int>(in_frame->GetHeight()),
253 input_format_.interval, input_format_.fourcc);
254 SetInputFormat(format);
255
256 // Drop the input frame if necessary.
257 bool should_drop = false;
258 if (!output_num_pixels_) {
259 // Drop all frames as the output format is 0x0.
260 should_drop = true;
261 } else {
262 // Drop some frames based on input fps and output fps.
263 // Normally output fps is less than input fps.
264 // TODO(fbarchard): Consider adjusting interval to reflect the adjusted
265 // interval between frames after dropping some frames.
266 interval_next_frame_ += input_format_.interval;
267 if (output_format_.interval > 0) {
268 if (interval_next_frame_ >= output_format_.interval) {
269 interval_next_frame_ %= output_format_.interval;
270 } else {
271 should_drop = true;
272 }
273 }
274 }
275 if (should_drop) {
276 *out_frame = NULL;
277 return true;
278 }
279
280 float scale = 1.f;
281 if (output_num_pixels_) {
282 scale = VideoAdapter::FindClosestViewScale(
283 static_cast<int>(in_frame->GetWidth()),
284 static_cast<int>(in_frame->GetHeight()),
285 output_num_pixels_);
286 output_format_.width = static_cast<int>(in_frame->GetWidth() * scale + .5f);
287 output_format_.height = static_cast<int>(in_frame->GetHeight() * scale +
288 .5f);
289 }
290
291 if (!StretchToOutputFrame(in_frame)) {
292 return false;
293 }
294
295 *out_frame = output_frame_.get();
296
297 // Show VAdapt log every 300 frames. (10 seconds)
298 // TODO(fbarchard): Consider GetLogSeverity() to change interval to less
299 // for LS_VERBOSE and more for LS_INFO.
300 bool show = frames_ % 300 == 0;
301 if (in_frame->GetWidth() != (*out_frame)->GetWidth() ||
302 in_frame->GetHeight() != (*out_frame)->GetHeight()) {
303 ++adapted_frames_;
304 }
305 // TODO(fbarchard): LOG the previous output resolution and track input
306 // resolution changes as well. Consider dropping the statistics into their
307 // own class which could be queried publically.
308 bool changed = false;
309 if (previous_width_ && (previous_width_ != (*out_frame)->GetWidth() ||
310 previous_height_ != (*out_frame)->GetHeight())) {
311 show = true;
312 ++adaption_changes_;
313 changed = true;
314 }
315 if (show) {
316 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
317 // in default calls.
318 LOG(LS_INFO) << "VAdapt Frame: " << adapted_frames_
319 << " / " << frames_
320 << " Changes: " << adaption_changes_
321 << " Input: " << in_frame->GetWidth()
322 << "x" << in_frame->GetHeight()
323 << " Scale: " << scale
324 << " Output: " << (*out_frame)->GetWidth()
325 << "x" << (*out_frame)->GetHeight()
326 << " Changed: " << (changed ? "true" : "false");
327 }
328 previous_width_ = (*out_frame)->GetWidth();
329 previous_height_ = (*out_frame)->GetHeight();
330
331 return true;
332 }
333
334 // Scale or Blacken the frame. Returns true if successful.
StretchToOutputFrame(const VideoFrame * in_frame)335 bool VideoAdapter::StretchToOutputFrame(const VideoFrame* in_frame) {
336 int output_width = output_format_.width;
337 int output_height = output_format_.height;
338
339 // Create and stretch the output frame if it has not been created yet or its
340 // size is not same as the expected.
341 bool stretched = false;
342 if (!output_frame_ ||
343 output_frame_->GetWidth() != static_cast<size_t>(output_width) ||
344 output_frame_->GetHeight() != static_cast<size_t>(output_height)) {
345 output_frame_.reset(
346 in_frame->Stretch(output_width, output_height, true, true));
347 if (!output_frame_) {
348 LOG(LS_WARNING) << "Adapter failed to stretch frame to "
349 << output_width << "x" << output_height;
350 return false;
351 }
352 stretched = true;
353 is_black_ = false;
354 }
355
356 if (!black_output_) {
357 if (!stretched) {
358 // The output frame does not need to be blacken and has not been stretched
359 // from the input frame yet, stretch the input frame. This is the most
360 // common case.
361 in_frame->StretchToFrame(output_frame_.get(), true, true);
362 }
363 is_black_ = false;
364 } else {
365 if (!is_black_) {
366 output_frame_->SetToBlack();
367 is_black_ = true;
368 }
369 output_frame_->SetElapsedTime(in_frame->GetElapsedTime());
370 output_frame_->SetTimeStamp(in_frame->GetTimeStamp());
371 }
372
373 return true;
374 }
375
376 ///////////////////////////////////////////////////////////////////////
377 // Implementation of CoordinatedVideoAdapter
CoordinatedVideoAdapter()378 CoordinatedVideoAdapter::CoordinatedVideoAdapter()
379 : cpu_adaptation_(true),
380 cpu_smoothing_(false),
381 gd_adaptation_(true),
382 view_adaptation_(true),
383 view_switch_(false),
384 cpu_downgrade_count_(0),
385 cpu_adapt_wait_time_(0),
386 high_system_threshold_(kHighSystemCpuThreshold),
387 low_system_threshold_(kLowSystemCpuThreshold),
388 process_threshold_(kProcessCpuThreshold),
389 view_desired_num_pixels_(INT_MAX),
390 view_desired_interval_(0),
391 encoder_desired_num_pixels_(INT_MAX),
392 cpu_desired_num_pixels_(INT_MAX),
393 adapt_reason_(0),
394 system_load_average_(kCpuLoadInitialAverage) {
395 }
396
397 // Helper function to UPGRADE or DOWNGRADE a number of pixels
StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,int * num_pixels)398 void CoordinatedVideoAdapter::StepPixelCount(
399 CoordinatedVideoAdapter::AdaptRequest request,
400 int* num_pixels) {
401 switch (request) {
402 case CoordinatedVideoAdapter::DOWNGRADE:
403 *num_pixels /= 2;
404 break;
405
406 case CoordinatedVideoAdapter::UPGRADE:
407 *num_pixels *= 2;
408 break;
409
410 default: // No change in pixel count
411 break;
412 }
413 return;
414 }
415
416 // Find the adaptation request of the cpu based on the load. Return UPGRADE if
417 // the load is low, DOWNGRADE if the load is high, and KEEP otherwise.
FindCpuRequest(int current_cpus,int max_cpus,float process_load,float system_load)418 CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest(
419 int current_cpus, int max_cpus,
420 float process_load, float system_load) {
421 // Downgrade if system is high and plugin is at least more than midrange.
422 if (system_load >= high_system_threshold_ * max_cpus &&
423 process_load >= process_threshold_ * current_cpus) {
424 return CoordinatedVideoAdapter::DOWNGRADE;
425 // Upgrade if system is low.
426 } else if (system_load < low_system_threshold_ * max_cpus) {
427 return CoordinatedVideoAdapter::UPGRADE;
428 }
429 return CoordinatedVideoAdapter::KEEP;
430 }
431
432 // A remote view request for a new resolution.
OnOutputFormatRequest(const VideoFormat & format)433 void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
434 talk_base::CritScope cs(&request_critical_section_);
435 if (!view_adaptation_) {
436 return;
437 }
438 // Set output for initial aspect ratio in mediachannel unittests.
439 int old_num_pixels = GetOutputNumPixels();
440 SetOutputFormat(format);
441 SetOutputNumPixels(old_num_pixels);
442 view_desired_num_pixels_ = format.width * format.height;
443 view_desired_interval_ = format.interval;
444 int new_width, new_height;
445 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
446 LOG(LS_INFO) << "VAdapt View Request: "
447 << format.width << "x" << format.height
448 << " Pixels: " << view_desired_num_pixels_
449 << " Changed: " << (changed ? "true" : "false")
450 << " To: " << new_width << "x" << new_height;
451 }
452
453 // A Bandwidth GD request for new resolution
OnEncoderResolutionRequest(int width,int height,AdaptRequest request)454 void CoordinatedVideoAdapter::OnEncoderResolutionRequest(
455 int width, int height, AdaptRequest request) {
456 talk_base::CritScope cs(&request_critical_section_);
457 if (!gd_adaptation_) {
458 return;
459 }
460 int old_encoder_desired_num_pixels = encoder_desired_num_pixels_;
461 if (KEEP != request) {
462 int new_encoder_desired_num_pixels = width * height;
463 int old_num_pixels = GetOutputNumPixels();
464 if (new_encoder_desired_num_pixels != old_num_pixels) {
465 LOG(LS_VERBOSE) << "VAdapt GD resolution stale. Ignored";
466 } else {
467 // Update the encoder desired format based on the request.
468 encoder_desired_num_pixels_ = new_encoder_desired_num_pixels;
469 StepPixelCount(request, &encoder_desired_num_pixels_);
470 }
471 }
472 int new_width, new_height;
473 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
474
475 // Ignore up or keep if no change.
476 if (DOWNGRADE != request && view_switch_ && !changed) {
477 encoder_desired_num_pixels_ = old_encoder_desired_num_pixels;
478 LOG(LS_VERBOSE) << "VAdapt ignoring GD request.";
479 }
480
481 LOG(LS_INFO) << "VAdapt GD Request: "
482 << (DOWNGRADE == request ? "down" :
483 (UPGRADE == request ? "up" : "keep"))
484 << " From: " << width << "x" << height
485 << " Pixels: " << encoder_desired_num_pixels_
486 << " Changed: " << (changed ? "true" : "false")
487 << " To: " << new_width << "x" << new_height;
488 }
489
490 // A Bandwidth GD request for new resolution
OnCpuResolutionRequest(AdaptRequest request)491 void CoordinatedVideoAdapter::OnCpuResolutionRequest(AdaptRequest request) {
492 talk_base::CritScope cs(&request_critical_section_);
493 if (!cpu_adaptation_) {
494 return;
495 }
496 // Update how many times we have downgraded due to the cpu load.
497 switch (request) {
498 case DOWNGRADE:
499 // Ignore downgrades if we have downgraded the maximum times.
500 if (cpu_downgrade_count_ < kMaxCpuDowngrades) {
501 ++cpu_downgrade_count_;
502 } else {
503 LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade "
504 "because maximum downgrades reached";
505 SignalCpuAdaptationUnable();
506 }
507 break;
508 case UPGRADE:
509 if (cpu_downgrade_count_ > 0) {
510 bool is_min = IsMinimumFormat(cpu_desired_num_pixels_);
511 if (is_min) {
512 --cpu_downgrade_count_;
513 } else {
514 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
515 "because cpu is not limiting resolution";
516 }
517 } else {
518 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
519 "because minimum downgrades reached";
520 }
521 break;
522 case KEEP:
523 default:
524 break;
525 }
526 if (KEEP != request) {
527 // TODO(fbarchard): compute stepping up/down from OutputNumPixels but
528 // clamp to inputpixels / 4 (2 steps)
529 cpu_desired_num_pixels_ = cpu_downgrade_count_ == 0 ? INT_MAX :
530 static_cast<int>(input_format().width * input_format().height >>
531 cpu_downgrade_count_);
532 }
533 int new_width, new_height;
534 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
535 LOG(LS_INFO) << "VAdapt CPU Request: "
536 << (DOWNGRADE == request ? "down" :
537 (UPGRADE == request ? "up" : "keep"))
538 << " Steps: " << cpu_downgrade_count_
539 << " Changed: " << (changed ? "true" : "false")
540 << " To: " << new_width << "x" << new_height;
541 }
542
543 // A CPU request for new resolution
544 // TODO(fbarchard): Move outside adapter.
OnCpuLoadUpdated(int current_cpus,int max_cpus,float process_load,float system_load)545 void CoordinatedVideoAdapter::OnCpuLoadUpdated(
546 int current_cpus, int max_cpus, float process_load, float system_load) {
547 talk_base::CritScope cs(&request_critical_section_);
548 if (!cpu_adaptation_) {
549 return;
550 }
551 // Update the moving average of system load. Even if we aren't smoothing,
552 // we'll still calculate this information, in case smoothing is later enabled.
553 system_load_average_ = kCpuLoadWeightCoefficient * system_load +
554 (1.0f - kCpuLoadWeightCoefficient) * system_load_average_;
555 if (cpu_smoothing_) {
556 system_load = system_load_average_;
557 }
558 // If we haven't started taking samples yet, wait until we have at least
559 // the correct number of samples per the wait time.
560 if (cpu_adapt_wait_time_ == 0) {
561 cpu_adapt_wait_time_ = talk_base::TimeAfter(kCpuLoadMinSampleTime);
562 }
563 AdaptRequest request = FindCpuRequest(current_cpus, max_cpus,
564 process_load, system_load);
565 // Make sure we're not adapting too quickly.
566 if (request != KEEP) {
567 if (talk_base::TimeIsLater(talk_base::Time(),
568 cpu_adapt_wait_time_)) {
569 LOG(LS_VERBOSE) << "VAdapt CPU load high/low but do not adapt until "
570 << talk_base::TimeUntil(cpu_adapt_wait_time_) << " ms";
571 request = KEEP;
572 }
573 }
574
575 OnCpuResolutionRequest(request);
576 }
577
578 // Called by cpu adapter on up requests.
IsMinimumFormat(int pixels)579 bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) {
580 // Find closest scale factor that matches input resolution to min_num_pixels
581 // and set that for output resolution. This is not needed for VideoAdapter,
582 // but provides feedback to unittests and users on expected resolution.
583 // Actual resolution is based on input frame.
584 VideoFormat new_output = output_format();
585 VideoFormat input = input_format();
586 if (input_format().IsSize0x0()) {
587 input = new_output;
588 }
589 float scale = 1.0f;
590 if (!input.IsSize0x0()) {
591 scale = FindClosestScale(input.width,
592 input.height,
593 pixels);
594 }
595 new_output.width = static_cast<int>(input.width * scale + .5f);
596 new_output.height = static_cast<int>(input.height * scale + .5f);
597 int new_pixels = new_output.width * new_output.height;
598 int num_pixels = GetOutputNumPixels();
599 return new_pixels <= num_pixels;
600 }
601
602 // Called by all coordinators when there is a change.
AdaptToMinimumFormat(int * new_width,int * new_height)603 bool CoordinatedVideoAdapter::AdaptToMinimumFormat(int* new_width,
604 int* new_height) {
605 VideoFormat new_output = output_format();
606 VideoFormat input = input_format();
607 if (input_format().IsSize0x0()) {
608 input = new_output;
609 }
610 int old_num_pixels = GetOutputNumPixels();
611 int min_num_pixels = INT_MAX;
612 adapt_reason_ = 0;
613
614 // Reduce resolution based on encoder bandwidth (GD).
615 if (encoder_desired_num_pixels_ &&
616 (encoder_desired_num_pixels_ < min_num_pixels)) {
617 adapt_reason_ |= ADAPTREASON_BANDWIDTH;
618 min_num_pixels = encoder_desired_num_pixels_;
619 }
620 // Reduce resolution based on CPU.
621 if (cpu_adaptation_ && cpu_desired_num_pixels_ &&
622 (cpu_desired_num_pixels_ <= min_num_pixels)) {
623 if (cpu_desired_num_pixels_ < min_num_pixels) {
624 adapt_reason_ = ADAPTREASON_CPU;
625 } else {
626 adapt_reason_ |= ADAPTREASON_CPU;
627 }
628 min_num_pixels = cpu_desired_num_pixels_;
629 }
630 // Round resolution for GD or CPU to allow 1/2 to map to 9/16.
631 if (!input.IsSize0x0() && min_num_pixels != INT_MAX) {
632 float scale = FindClosestScale(input.width, input.height, min_num_pixels);
633 min_num_pixels = static_cast<int>(input.width * scale + .5f) *
634 static_cast<int>(input.height * scale + .5f);
635 }
636 // Reduce resolution based on View Request.
637 if (view_desired_num_pixels_ <= min_num_pixels) {
638 if (view_desired_num_pixels_ < min_num_pixels) {
639 adapt_reason_ = ADAPTREASON_VIEW;
640 } else {
641 adapt_reason_ |= ADAPTREASON_VIEW;
642 }
643 min_num_pixels = view_desired_num_pixels_;
644 }
645 // Snap to a scale factor.
646 float scale = 1.0f;
647 if (!input.IsSize0x0()) {
648 scale = FindLowerScale(input.width, input.height, min_num_pixels);
649 min_num_pixels = static_cast<int>(input.width * scale + .5f) *
650 static_cast<int>(input.height * scale + .5f);
651 }
652 if (scale == 1.0f) {
653 adapt_reason_ = 0;
654 }
655 *new_width = new_output.width = static_cast<int>(input.width * scale + .5f);
656 *new_height = new_output.height = static_cast<int>(input.height * scale +
657 .5f);
658 SetOutputNumPixels(min_num_pixels);
659
660 new_output.interval = view_desired_interval_;
661 SetOutputFormat(new_output);
662 int new_num_pixels = GetOutputNumPixels();
663 bool changed = new_num_pixels != old_num_pixels;
664
665 static const char* kReasons[8] = {
666 "None",
667 "CPU",
668 "BANDWIDTH",
669 "CPU+BANDWIDTH",
670 "VIEW",
671 "CPU+VIEW",
672 "BANDWIDTH+VIEW",
673 "CPU+BANDWIDTH+VIEW",
674 };
675
676 LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_
677 << " GD: " << encoder_desired_num_pixels_
678 << " CPU: " << cpu_desired_num_pixels_
679 << " Pixels: " << min_num_pixels
680 << " Input: " << input.width
681 << "x" << input.height
682 << " Scale: " << scale
683 << " Resolution: " << new_output.width
684 << "x" << new_output.height
685 << " Changed: " << (changed ? "true" : "false")
686 << " Reason: " << kReasons[adapt_reason_];
687
688 if (changed) {
689 // When any adaptation occurs, historic CPU load levels are no longer
690 // accurate. Clear out our state so we can re-learn at the new normal.
691 cpu_adapt_wait_time_ = talk_base::TimeAfter(kCpuLoadMinSampleTime);
692 system_load_average_ = kCpuLoadInitialAverage;
693 }
694
695 return changed;
696 }
697
698 } // namespace cricket
699