• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
12 
13 #include <assert.h>
14 #include <string.h>
15 
16 #include <algorithm>
17 #include <cstdint>
18 #include <iterator>
19 #include <memory>
20 #include <string>
21 #include <utility>
22 #include <vector>
23 
24 #include "api/scoped_refptr.h"
25 #include "api/video/video_content_type.h"
26 #include "api/video/video_frame_buffer.h"
27 #include "api/video/video_timing.h"
28 #include "api/video_codecs/vp8_temporal_layers.h"
29 #include "api/video_codecs/vp8_temporal_layers_factory.h"
30 #include "modules/video_coding/codecs/interface/common_constants.h"
31 #include "modules/video_coding/codecs/vp8/include/vp8.h"
32 #include "modules/video_coding/include/video_error_codes.h"
33 #include "modules/video_coding/utility/simulcast_rate_allocator.h"
34 #include "modules/video_coding/utility/simulcast_utility.h"
35 #include "rtc_base/checks.h"
36 #include "rtc_base/experiments/field_trial_parser.h"
37 #include "rtc_base/experiments/field_trial_units.h"
38 #include "rtc_base/logging.h"
39 #include "rtc_base/trace_event.h"
40 #include "system_wrappers/include/field_trial.h"
41 #include "third_party/libyuv/include/libyuv/scale.h"
42 #include "vpx/vp8cx.h"
43 
44 namespace webrtc {
45 namespace {
46 #if defined(WEBRTC_IOS)
47 const char kVP8IosMaxNumberOfThreadFieldTrial[] =
48     "WebRTC-VP8IosMaxNumberOfThread";
49 const char kVP8IosMaxNumberOfThreadFieldTrialParameter[] = "max_thread";
50 #endif
51 
52 const char kVp8ForcePartitionResilience[] =
53     "WebRTC-VP8-ForcePartitionResilience";
54 
55 // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
56 // bitstream range of [0, 127] and not the user-level range of [0,63].
57 constexpr int kLowVp8QpThreshold = 29;
58 constexpr int kHighVp8QpThreshold = 95;
59 
60 constexpr int kTokenPartitions = VP8_ONE_TOKENPARTITION;
61 constexpr uint32_t kVp832ByteAlign = 32u;
62 
63 constexpr int kRtpTicksPerSecond = 90000;
64 constexpr int kRtpTicksPerMs = kRtpTicksPerSecond / 1000;
65 
66 constexpr double kLowRateFactor = 1.0;
67 constexpr double kHighRateFactor = 2.0;
68 
69 // VP8 denoiser states.
70 enum denoiserState : uint32_t {
71   kDenoiserOff,
72   kDenoiserOnYOnly,
73   kDenoiserOnYUV,
74   kDenoiserOnYUVAggressive,
75   // Adaptive mode defaults to kDenoiserOnYUV on key frame, but may switch
76   // to kDenoiserOnYUVAggressive based on a computed noise metric.
77   kDenoiserOnAdaptive
78 };
79 
80 // These settings correspond to the settings in vpx_codec_enc_cfg.
81 struct Vp8RateSettings {
82   uint32_t rc_undershoot_pct;
83   uint32_t rc_overshoot_pct;
84   uint32_t rc_buf_sz;
85   uint32_t rc_buf_optimal_sz;
86   uint32_t rc_dropframe_thresh;
87 };
88 
89 // Greatest common divisior
GCD(int a,int b)90 int GCD(int a, int b) {
91   int c = a % b;
92   while (c != 0) {
93     a = b;
94     b = c;
95     c = a % b;
96   }
97   return b;
98 }
99 
Interpolate(uint32_t low,uint32_t high,double bandwidth_headroom_factor)100 uint32_t Interpolate(uint32_t low,
101                      uint32_t high,
102                      double bandwidth_headroom_factor) {
103   RTC_DCHECK_GE(bandwidth_headroom_factor, kLowRateFactor);
104   RTC_DCHECK_LE(bandwidth_headroom_factor, kHighRateFactor);
105 
106   // |factor| is between 0.0 and 1.0.
107   const double factor = bandwidth_headroom_factor - kLowRateFactor;
108 
109   return static_cast<uint32_t>(((1.0 - factor) * low) + (factor * high) + 0.5);
110 }
111 
GetRateSettings(double bandwidth_headroom_factor)112 Vp8RateSettings GetRateSettings(double bandwidth_headroom_factor) {
113   static const Vp8RateSettings low_settings{1000u, 0u, 100u, 30u, 40u};
114   static const Vp8RateSettings high_settings{100u, 15u, 1000u, 600u, 5u};
115 
116   if (bandwidth_headroom_factor <= kLowRateFactor) {
117     return low_settings;
118   } else if (bandwidth_headroom_factor >= kHighRateFactor) {
119     return high_settings;
120   }
121 
122   Vp8RateSettings settings;
123   settings.rc_undershoot_pct =
124       Interpolate(low_settings.rc_undershoot_pct,
125                   high_settings.rc_undershoot_pct, bandwidth_headroom_factor);
126   settings.rc_overshoot_pct =
127       Interpolate(low_settings.rc_overshoot_pct, high_settings.rc_overshoot_pct,
128                   bandwidth_headroom_factor);
129   settings.rc_buf_sz =
130       Interpolate(low_settings.rc_buf_sz, high_settings.rc_buf_sz,
131                   bandwidth_headroom_factor);
132   settings.rc_buf_optimal_sz =
133       Interpolate(low_settings.rc_buf_optimal_sz,
134                   high_settings.rc_buf_optimal_sz, bandwidth_headroom_factor);
135   settings.rc_dropframe_thresh =
136       Interpolate(low_settings.rc_dropframe_thresh,
137                   high_settings.rc_dropframe_thresh, bandwidth_headroom_factor);
138   return settings;
139 }
140 
UpdateRateSettings(vpx_codec_enc_cfg_t * config,const Vp8RateSettings & new_settings)141 void UpdateRateSettings(vpx_codec_enc_cfg_t* config,
142                         const Vp8RateSettings& new_settings) {
143   config->rc_undershoot_pct = new_settings.rc_undershoot_pct;
144   config->rc_overshoot_pct = new_settings.rc_overshoot_pct;
145   config->rc_buf_sz = new_settings.rc_buf_sz;
146   config->rc_buf_optimal_sz = new_settings.rc_buf_optimal_sz;
147   config->rc_dropframe_thresh = new_settings.rc_dropframe_thresh;
148 }
149 
150 static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxPeriodicity ==
151                   VPX_TS_MAX_PERIODICITY,
152               "Vp8EncoderConfig::kMaxPeriodicity must be kept in sync with the "
153               "constant in libvpx.");
154 static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxLayers ==
155                   VPX_TS_MAX_LAYERS,
156               "Vp8EncoderConfig::kMaxLayers must be kept in sync with the "
157               "constant in libvpx.");
158 
159 // Allow a newer value to override a current value only if the new value
160 // is set.
161 template <typename T>
MaybeSetNewValue(const absl::optional<T> & new_value,absl::optional<T> * base_value)162 bool MaybeSetNewValue(const absl::optional<T>& new_value,
163                       absl::optional<T>* base_value) {
164   if (new_value.has_value() && new_value != *base_value) {
165     *base_value = new_value;
166     return true;
167   } else {
168     return false;
169   }
170 }
171 
172 // Adds configuration from |new_config| to |base_config|. Both configs consist
173 // of optionals, and only optionals which are set in |new_config| can have
174 // an effect. (That is, set values in |base_config| cannot be unset.)
175 // Returns |true| iff any changes were made to |base_config|.
MaybeExtendVp8EncoderConfig(const Vp8EncoderConfig & new_config,Vp8EncoderConfig * base_config)176 bool MaybeExtendVp8EncoderConfig(const Vp8EncoderConfig& new_config,
177                                  Vp8EncoderConfig* base_config) {
178   bool changes_made = false;
179   changes_made |= MaybeSetNewValue(new_config.temporal_layer_config,
180                                    &base_config->temporal_layer_config);
181   changes_made |= MaybeSetNewValue(new_config.rc_target_bitrate,
182                                    &base_config->rc_target_bitrate);
183   changes_made |= MaybeSetNewValue(new_config.rc_max_quantizer,
184                                    &base_config->rc_max_quantizer);
185   changes_made |= MaybeSetNewValue(new_config.g_error_resilient,
186                                    &base_config->g_error_resilient);
187   return changes_made;
188 }
189 
ApplyVp8EncoderConfigToVpxConfig(const Vp8EncoderConfig & encoder_config,vpx_codec_enc_cfg_t * vpx_config)190 void ApplyVp8EncoderConfigToVpxConfig(const Vp8EncoderConfig& encoder_config,
191                                       vpx_codec_enc_cfg_t* vpx_config) {
192   if (encoder_config.temporal_layer_config.has_value()) {
193     const Vp8EncoderConfig::TemporalLayerConfig& ts_config =
194         encoder_config.temporal_layer_config.value();
195     vpx_config->ts_number_layers = ts_config.ts_number_layers;
196     std::copy(ts_config.ts_target_bitrate.begin(),
197               ts_config.ts_target_bitrate.end(),
198               std::begin(vpx_config->ts_target_bitrate));
199     std::copy(ts_config.ts_rate_decimator.begin(),
200               ts_config.ts_rate_decimator.end(),
201               std::begin(vpx_config->ts_rate_decimator));
202     vpx_config->ts_periodicity = ts_config.ts_periodicity;
203     std::copy(ts_config.ts_layer_id.begin(), ts_config.ts_layer_id.end(),
204               std::begin(vpx_config->ts_layer_id));
205   } else {
206     vpx_config->ts_number_layers = 1;
207     vpx_config->ts_rate_decimator[0] = 1;
208     vpx_config->ts_periodicity = 1;
209     vpx_config->ts_layer_id[0] = 0;
210   }
211 
212   if (encoder_config.rc_target_bitrate.has_value()) {
213     vpx_config->rc_target_bitrate = encoder_config.rc_target_bitrate.value();
214   }
215 
216   if (encoder_config.rc_max_quantizer.has_value()) {
217     vpx_config->rc_max_quantizer = encoder_config.rc_max_quantizer.value();
218   }
219 
220   if (encoder_config.g_error_resilient.has_value()) {
221     vpx_config->g_error_resilient = encoder_config.g_error_resilient.value();
222   }
223 }
224 
225 }  // namespace
226 
Create()227 std::unique_ptr<VideoEncoder> VP8Encoder::Create() {
228   return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::CreateEncoder(),
229                                             VP8Encoder::Settings());
230 }
231 
Create(VP8Encoder::Settings settings)232 std::unique_ptr<VideoEncoder> VP8Encoder::Create(
233     VP8Encoder::Settings settings) {
234   return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::CreateEncoder(),
235                                             std::move(settings));
236 }
237 
Create(std::unique_ptr<Vp8FrameBufferControllerFactory> frame_buffer_controller_factory)238 std::unique_ptr<VideoEncoder> VP8Encoder::Create(
239     std::unique_ptr<Vp8FrameBufferControllerFactory>
240         frame_buffer_controller_factory) {
241   VP8Encoder::Settings settings;
242   settings.frame_buffer_controller_factory =
243       std::move(frame_buffer_controller_factory);
244   return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::CreateEncoder(),
245                                             std::move(settings));
246 }
247 
EncodeFlags(const Vp8FrameConfig & references)248 vpx_enc_frame_flags_t LibvpxVp8Encoder::EncodeFlags(
249     const Vp8FrameConfig& references) {
250   RTC_DCHECK(!references.drop_frame);
251 
252   vpx_enc_frame_flags_t flags = 0;
253 
254   if ((references.last_buffer_flags &
255        Vp8FrameConfig::BufferFlags::kReference) == 0)
256     flags |= VP8_EFLAG_NO_REF_LAST;
257   if ((references.last_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) ==
258       0)
259     flags |= VP8_EFLAG_NO_UPD_LAST;
260   if ((references.golden_buffer_flags &
261        Vp8FrameConfig::BufferFlags::kReference) == 0)
262     flags |= VP8_EFLAG_NO_REF_GF;
263   if ((references.golden_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) ==
264       0)
265     flags |= VP8_EFLAG_NO_UPD_GF;
266   if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kReference) ==
267       0)
268     flags |= VP8_EFLAG_NO_REF_ARF;
269   if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == 0)
270     flags |= VP8_EFLAG_NO_UPD_ARF;
271   if (references.freeze_entropy)
272     flags |= VP8_EFLAG_NO_UPD_ENTROPY;
273 
274   return flags;
275 }
276 
LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,VP8Encoder::Settings settings)277 LibvpxVp8Encoder::LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,
278                                    VP8Encoder::Settings settings)
279     : libvpx_(std::move(interface)),
280       experimental_cpu_speed_config_arm_(CpuSpeedExperiment::GetConfigs()),
281       rate_control_settings_(RateControlSettings::ParseFromFieldTrials()),
282       frame_buffer_controller_factory_(
283           std::move(settings.frame_buffer_controller_factory)),
284       resolution_bitrate_limits_(std::move(settings.resolution_bitrate_limits)),
285       key_frame_request_(kMaxSimulcastStreams, false),
286       variable_framerate_experiment_(ParseVariableFramerateConfig(
287           "WebRTC-VP8VariableFramerateScreenshare")),
288       framerate_controller_(variable_framerate_experiment_.framerate_limit) {
289   // TODO(eladalon/ilnik): These reservations might be wasting memory.
290   // InitEncode() is resizing to the actual size, which might be smaller.
291   raw_images_.reserve(kMaxSimulcastStreams);
292   encoded_images_.reserve(kMaxSimulcastStreams);
293   send_stream_.reserve(kMaxSimulcastStreams);
294   cpu_speed_.assign(kMaxSimulcastStreams, cpu_speed_default_);
295   encoders_.reserve(kMaxSimulcastStreams);
296   vpx_configs_.reserve(kMaxSimulcastStreams);
297   config_overrides_.reserve(kMaxSimulcastStreams);
298   downsampling_factors_.reserve(kMaxSimulcastStreams);
299 }
300 
~LibvpxVp8Encoder()301 LibvpxVp8Encoder::~LibvpxVp8Encoder() {
302   Release();
303 }
304 
Release()305 int LibvpxVp8Encoder::Release() {
306   int ret_val = WEBRTC_VIDEO_CODEC_OK;
307 
308   encoded_images_.clear();
309 
310   if (inited_) {
311     for (auto it = encoders_.rbegin(); it != encoders_.rend(); ++it) {
312       if (libvpx_->codec_destroy(&*it)) {
313         ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
314       }
315     }
316   }
317   encoders_.clear();
318 
319   vpx_configs_.clear();
320   config_overrides_.clear();
321   send_stream_.clear();
322   cpu_speed_.clear();
323 
324   for (auto it = raw_images_.rbegin(); it != raw_images_.rend(); ++it) {
325     libvpx_->img_free(&*it);
326   }
327   raw_images_.clear();
328 
329   frame_buffer_controller_.reset();
330   inited_ = false;
331   return ret_val;
332 }
333 
SetRates(const RateControlParameters & parameters)334 void LibvpxVp8Encoder::SetRates(const RateControlParameters& parameters) {
335   if (!inited_) {
336     RTC_LOG(LS_WARNING) << "SetRates() while not initialize";
337     return;
338   }
339 
340   if (encoders_[0].err) {
341     RTC_LOG(LS_WARNING) << "Encoder in error state.";
342     return;
343   }
344 
345   if (parameters.framerate_fps < 1.0) {
346     RTC_LOG(LS_WARNING) << "Unsupported framerate (must be >= 1.0): "
347                         << parameters.framerate_fps;
348     return;
349   }
350 
351   if (parameters.bitrate.get_sum_bps() == 0) {
352     // Encoder paused, turn off all encoding.
353     const int num_streams = static_cast<size_t>(encoders_.size());
354     for (int i = 0; i < num_streams; ++i)
355       SetStreamState(false, i);
356     return;
357   }
358 
359   codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
360 
361   if (encoders_.size() > 1) {
362     // If we have more than 1 stream, reduce the qp_max for the low resolution
363     // stream if frame rate is not too low. The trade-off with lower qp_max is
364     // possibly more dropped frames, so we only do this if the frame rate is
365     // above some threshold (base temporal layer is down to 1/4 for 3 layers).
366     // We may want to condition this on bitrate later.
367     if (rate_control_settings_.Vp8BoostBaseLayerQuality() &&
368         parameters.framerate_fps > 20.0) {
369       vpx_configs_[encoders_.size() - 1].rc_max_quantizer = 45;
370     } else {
371       // Go back to default value set in InitEncode.
372       vpx_configs_[encoders_.size() - 1].rc_max_quantizer = qp_max_;
373     }
374   }
375 
376   for (size_t i = 0; i < encoders_.size(); ++i) {
377     const size_t stream_idx = encoders_.size() - 1 - i;
378 
379     unsigned int target_bitrate_kbps =
380         parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;
381 
382     bool send_stream = target_bitrate_kbps > 0;
383     if (send_stream || encoders_.size() > 1)
384       SetStreamState(send_stream, stream_idx);
385 
386     vpx_configs_[i].rc_target_bitrate = target_bitrate_kbps;
387     if (send_stream) {
388       frame_buffer_controller_->OnRatesUpdated(
389           stream_idx, parameters.bitrate.GetTemporalLayerAllocation(stream_idx),
390           static_cast<int>(parameters.framerate_fps + 0.5));
391     }
392 
393     UpdateVpxConfiguration(stream_idx);
394 
395     if (rate_control_settings_.Vp8DynamicRateSettings()) {
396       // Tweak rate control settings based on available network headroom.
397       UpdateRateSettings(
398           &vpx_configs_[i],
399           GetRateSettings(parameters.bandwidth_allocation.bps<double>() /
400                           parameters.bitrate.get_sum_bps()));
401     }
402 
403     vpx_codec_err_t err =
404         libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i]);
405     if (err != VPX_CODEC_OK) {
406       RTC_LOG(LS_WARNING) << "Error configuring codec, error code: " << err;
407     }
408   }
409 }
410 
OnPacketLossRateUpdate(float packet_loss_rate)411 void LibvpxVp8Encoder::OnPacketLossRateUpdate(float packet_loss_rate) {
412   // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK.
413   if (frame_buffer_controller_) {
414     frame_buffer_controller_->OnPacketLossRateUpdate(packet_loss_rate);
415   }
416 }
417 
OnRttUpdate(int64_t rtt_ms)418 void LibvpxVp8Encoder::OnRttUpdate(int64_t rtt_ms) {
419   // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK.
420   if (frame_buffer_controller_) {
421     frame_buffer_controller_->OnRttUpdate(rtt_ms);
422   }
423 }
424 
OnLossNotification(const LossNotification & loss_notification)425 void LibvpxVp8Encoder::OnLossNotification(
426     const LossNotification& loss_notification) {
427   if (frame_buffer_controller_) {
428     frame_buffer_controller_->OnLossNotification(loss_notification);
429   }
430 }
431 
SetStreamState(bool send_stream,int stream_idx)432 void LibvpxVp8Encoder::SetStreamState(bool send_stream, int stream_idx) {
433   if (send_stream && !send_stream_[stream_idx]) {
434     // Need a key frame if we have not sent this stream before.
435     key_frame_request_[stream_idx] = true;
436   }
437   send_stream_[stream_idx] = send_stream;
438 }
439 
SetFecControllerOverride(FecControllerOverride * fec_controller_override)440 void LibvpxVp8Encoder::SetFecControllerOverride(
441     FecControllerOverride* fec_controller_override) {
442   // TODO(bugs.webrtc.org/10769): Update downstream and remove ability to
443   // pass nullptr.
444   // RTC_DCHECK(fec_controller_override);
445   RTC_DCHECK(!fec_controller_override_);
446   fec_controller_override_ = fec_controller_override;
447 }
448 
449 // TODO(eladalon): s/inst/codec_settings/g.
InitEncode(const VideoCodec * inst,const VideoEncoder::Settings & settings)450 int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst,
451                                  const VideoEncoder::Settings& settings) {
452   if (inst == NULL) {
453     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
454   }
455   if (inst->maxFramerate < 1) {
456     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
457   }
458   // allow zero to represent an unspecified maxBitRate
459   if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
460     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
461   }
462   if (inst->width < 1 || inst->height < 1) {
463     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
464   }
465   if (settings.number_of_cores < 1) {
466     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
467   }
468 
469   num_active_streams_ = 0;
470   for (int i = 0; i < inst->numberOfSimulcastStreams; ++i) {
471     if (inst->simulcastStream[i].active) {
472       ++num_active_streams_;
473     }
474   }
475   if (inst->numberOfSimulcastStreams == 0 && inst->active) {
476     num_active_streams_ = 1;
477   }
478 
479   if (inst->VP8().automaticResizeOn && num_active_streams_ > 1) {
480     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
481   }
482 
483   int retVal = Release();
484   if (retVal < 0) {
485     return retVal;
486   }
487 
488   int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
489   if (number_of_streams > 1 &&
490       !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
491     return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
492   }
493 
494   RTC_DCHECK(!frame_buffer_controller_);
495   if (frame_buffer_controller_factory_) {
496     frame_buffer_controller_ = frame_buffer_controller_factory_->Create(
497         *inst, settings, fec_controller_override_);
498   } else {
499     Vp8TemporalLayersFactory factory;
500     frame_buffer_controller_ =
501         factory.Create(*inst, settings, fec_controller_override_);
502   }
503   RTC_DCHECK(frame_buffer_controller_);
504 
505   number_of_cores_ = settings.number_of_cores;
506   timestamp_ = 0;
507   codec_ = *inst;
508 
509   // Code expects simulcastStream resolutions to be correct, make sure they are
510   // filled even when there are no simulcast layers.
511   if (codec_.numberOfSimulcastStreams == 0) {
512     codec_.simulcastStream[0].width = codec_.width;
513     codec_.simulcastStream[0].height = codec_.height;
514   }
515 
516   encoded_images_.resize(number_of_streams);
517   encoders_.resize(number_of_streams);
518   vpx_configs_.resize(number_of_streams);
519   config_overrides_.resize(number_of_streams);
520   downsampling_factors_.resize(number_of_streams);
521   raw_images_.resize(number_of_streams);
522   send_stream_.resize(number_of_streams);
523   send_stream_[0] = true;  // For non-simulcast case.
524   cpu_speed_.resize(number_of_streams);
525   std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
526 
527   int idx = number_of_streams - 1;
528   for (int i = 0; i < (number_of_streams - 1); ++i, --idx) {
529     int gcd = GCD(inst->simulcastStream[idx].width,
530                   inst->simulcastStream[idx - 1].width);
531     downsampling_factors_[i].num = inst->simulcastStream[idx].width / gcd;
532     downsampling_factors_[i].den = inst->simulcastStream[idx - 1].width / gcd;
533     send_stream_[i] = false;
534   }
535   if (number_of_streams > 1) {
536     send_stream_[number_of_streams - 1] = false;
537     downsampling_factors_[number_of_streams - 1].num = 1;
538     downsampling_factors_[number_of_streams - 1].den = 1;
539   }
540   for (int i = 0; i < number_of_streams; ++i) {
541     encoded_images_[i]._completeFrame = true;
542   }
543   // populate encoder configuration with default values
544   if (libvpx_->codec_enc_config_default(vpx_codec_vp8_cx(), &vpx_configs_[0],
545                                         0)) {
546     return WEBRTC_VIDEO_CODEC_ERROR;
547   }
548   // setting the time base of the codec
549   vpx_configs_[0].g_timebase.num = 1;
550   vpx_configs_[0].g_timebase.den = kRtpTicksPerSecond;
551   vpx_configs_[0].g_lag_in_frames = 0;  // 0- no frame lagging
552 
553   // Set the error resilience mode for temporal layers (but not simulcast).
554   vpx_configs_[0].g_error_resilient =
555       (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1)
556           ? VPX_ERROR_RESILIENT_DEFAULT
557           : 0;
558 
559   // Override the error resilience mode if this is not simulcast, but we are
560   // using temporal layers.
561   if (field_trial::IsEnabled(kVp8ForcePartitionResilience) &&
562       (number_of_streams == 1) &&
563       (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1)) {
564     RTC_LOG(LS_INFO) << "Overriding g_error_resilient from "
565                      << vpx_configs_[0].g_error_resilient << " to "
566                      << VPX_ERROR_RESILIENT_PARTITIONS;
567     vpx_configs_[0].g_error_resilient = VPX_ERROR_RESILIENT_PARTITIONS;
568   }
569 
570   // rate control settings
571   vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(0);
572   vpx_configs_[0].rc_end_usage = VPX_CBR;
573   vpx_configs_[0].g_pass = VPX_RC_ONE_PASS;
574   // Handle resizing outside of libvpx.
575   vpx_configs_[0].rc_resize_allowed = 0;
576   vpx_configs_[0].rc_min_quantizer =
577       codec_.mode == VideoCodecMode::kScreensharing ? 12 : 2;
578   if (inst->qpMax >= vpx_configs_[0].rc_min_quantizer) {
579     qp_max_ = inst->qpMax;
580   }
581   if (rate_control_settings_.LibvpxVp8QpMax()) {
582     qp_max_ = std::max(rate_control_settings_.LibvpxVp8QpMax().value(),
583                        static_cast<int>(vpx_configs_[0].rc_min_quantizer));
584   }
585   vpx_configs_[0].rc_max_quantizer = qp_max_;
586   vpx_configs_[0].rc_undershoot_pct = 100;
587   vpx_configs_[0].rc_overshoot_pct = 15;
588   vpx_configs_[0].rc_buf_initial_sz = 500;
589   vpx_configs_[0].rc_buf_optimal_sz = 600;
590   vpx_configs_[0].rc_buf_sz = 1000;
591 
592   // Set the maximum target size of any key-frame.
593   rc_max_intra_target_ = MaxIntraTarget(vpx_configs_[0].rc_buf_optimal_sz);
594 
595   if (inst->VP8().keyFrameInterval > 0) {
596     vpx_configs_[0].kf_mode = VPX_KF_AUTO;
597     vpx_configs_[0].kf_max_dist = inst->VP8().keyFrameInterval;
598   } else {
599     vpx_configs_[0].kf_mode = VPX_KF_DISABLED;
600   }
601 
602   // Allow the user to set the complexity for the base stream.
603   switch (inst->VP8().complexity) {
604     case VideoCodecComplexity::kComplexityHigh:
605       cpu_speed_[0] = -5;
606       break;
607     case VideoCodecComplexity::kComplexityHigher:
608       cpu_speed_[0] = -4;
609       break;
610     case VideoCodecComplexity::kComplexityMax:
611       cpu_speed_[0] = -3;
612       break;
613     default:
614       cpu_speed_[0] = -6;
615       break;
616   }
617   cpu_speed_default_ = cpu_speed_[0];
618   // Set encoding complexity (cpu_speed) based on resolution and/or platform.
619   cpu_speed_[0] = GetCpuSpeed(inst->width, inst->height);
620   for (int i = 1; i < number_of_streams; ++i) {
621     cpu_speed_[i] =
622         GetCpuSpeed(inst->simulcastStream[number_of_streams - 1 - i].width,
623                     inst->simulcastStream[number_of_streams - 1 - i].height);
624   }
625   vpx_configs_[0].g_w = inst->width;
626   vpx_configs_[0].g_h = inst->height;
627 
628   // Determine number of threads based on the image size and #cores.
629   // TODO(fbarchard): Consider number of Simulcast layers.
630   vpx_configs_[0].g_threads = NumberOfThreads(
631       vpx_configs_[0].g_w, vpx_configs_[0].g_h, settings.number_of_cores);
632 
633   // Creating a wrapper to the image - setting image data to NULL.
634   // Actual pointer will be set in encode. Setting align to 1, as it
635   // is meaningless (no memory allocation is done here).
636   libvpx_->img_wrap(&raw_images_[0], VPX_IMG_FMT_I420, inst->width,
637                     inst->height, 1, NULL);
638 
639   // Note the order we use is different from webm, we have lowest resolution
640   // at position 0 and they have highest resolution at position 0.
641   const size_t stream_idx_cfg_0 = encoders_.size() - 1;
642   SimulcastRateAllocator init_allocator(codec_);
643   VideoBitrateAllocation allocation =
644       init_allocator.Allocate(VideoBitrateAllocationParameters(
645           inst->startBitrate * 1000, inst->maxFramerate));
646   std::vector<uint32_t> stream_bitrates;
647   for (int i = 0; i == 0 || i < inst->numberOfSimulcastStreams; ++i) {
648     uint32_t bitrate = allocation.GetSpatialLayerSum(i) / 1000;
649     stream_bitrates.push_back(bitrate);
650   }
651 
652   vpx_configs_[0].rc_target_bitrate = stream_bitrates[stream_idx_cfg_0];
653   if (stream_bitrates[stream_idx_cfg_0] > 0) {
654     uint32_t maxFramerate =
655         inst->simulcastStream[stream_idx_cfg_0].maxFramerate;
656     if (!maxFramerate) {
657       maxFramerate = inst->maxFramerate;
658     }
659 
660     frame_buffer_controller_->OnRatesUpdated(
661         stream_idx_cfg_0,
662         allocation.GetTemporalLayerAllocation(stream_idx_cfg_0), maxFramerate);
663   }
664   frame_buffer_controller_->SetQpLimits(stream_idx_cfg_0,
665                                         vpx_configs_[0].rc_min_quantizer,
666                                         vpx_configs_[0].rc_max_quantizer);
667   UpdateVpxConfiguration(stream_idx_cfg_0);
668   vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(stream_idx_cfg_0);
669 
670   for (size_t i = 1; i < encoders_.size(); ++i) {
671     const size_t stream_idx = encoders_.size() - 1 - i;
672     memcpy(&vpx_configs_[i], &vpx_configs_[0], sizeof(vpx_configs_[0]));
673 
674     vpx_configs_[i].g_w = inst->simulcastStream[stream_idx].width;
675     vpx_configs_[i].g_h = inst->simulcastStream[stream_idx].height;
676 
677     // Use 1 thread for lower resolutions.
678     vpx_configs_[i].g_threads = 1;
679 
680     vpx_configs_[i].rc_dropframe_thresh = FrameDropThreshold(stream_idx);
681 
682     // Setting alignment to 32 - as that ensures at least 16 for all
683     // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for
684     // the y plane, but only half of it to the u and v planes.
685     libvpx_->img_alloc(&raw_images_[i], VPX_IMG_FMT_I420,
686                        inst->simulcastStream[stream_idx].width,
687                        inst->simulcastStream[stream_idx].height,
688                        kVp832ByteAlign);
689     SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx);
690     vpx_configs_[i].rc_target_bitrate = stream_bitrates[stream_idx];
691     if (stream_bitrates[stream_idx] > 0) {
692       uint32_t maxFramerate = inst->simulcastStream[stream_idx].maxFramerate;
693       if (!maxFramerate) {
694         maxFramerate = inst->maxFramerate;
695       }
696       frame_buffer_controller_->OnRatesUpdated(
697           stream_idx, allocation.GetTemporalLayerAllocation(stream_idx),
698           maxFramerate);
699     }
700     frame_buffer_controller_->SetQpLimits(stream_idx,
701                                           vpx_configs_[i].rc_min_quantizer,
702                                           vpx_configs_[i].rc_max_quantizer);
703     UpdateVpxConfiguration(stream_idx);
704   }
705 
706   return InitAndSetControlSettings();
707 }
708 
GetCpuSpeed(int width,int height)709 int LibvpxVp8Encoder::GetCpuSpeed(int width, int height) {
710 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
711     defined(WEBRTC_ANDROID)
712   // On mobile platform, use a lower speed setting for lower resolutions for
713   // CPUs with 4 or more cores.
714   RTC_DCHECK_GT(number_of_cores_, 0);
715   if (number_of_cores_ <= 3)
716     return -12;
717 
718   if (experimental_cpu_speed_config_arm_) {
719     return CpuSpeedExperiment::GetValue(width * height,
720                                         *experimental_cpu_speed_config_arm_);
721   }
722 
723   if (width * height <= 352 * 288)
724     return -8;
725   else if (width * height <= 640 * 480)
726     return -10;
727   else
728     return -12;
729 #else
730   // For non-ARM, increase encoding complexity (i.e., use lower speed setting)
731   // if resolution is below CIF. Otherwise, keep the default/user setting
732   // (|cpu_speed_default_|) set on InitEncode via VP8().complexity.
733   if (width * height < 352 * 288)
734     return (cpu_speed_default_ < -4) ? -4 : cpu_speed_default_;
735   else
736     return cpu_speed_default_;
737 #endif
738 }
739 
NumberOfThreads(int width,int height,int cpus)740 int LibvpxVp8Encoder::NumberOfThreads(int width, int height, int cpus) {
741 #if defined(WEBRTC_ANDROID)
742   if (width * height >= 320 * 180) {
743     if (cpus >= 4) {
744       // 3 threads for CPUs with 4 and more cores since most of times only 4
745       // cores will be active.
746       return 3;
747     } else if (cpus == 3 || cpus == 2) {
748       return 2;
749     } else {
750       return 1;
751     }
752   }
753   return 1;
754 #else
755 #if defined(WEBRTC_IOS)
756   std::string trial_string =
757       field_trial::FindFullName(kVP8IosMaxNumberOfThreadFieldTrial);
758   FieldTrialParameter<int> max_thread_number(
759       kVP8IosMaxNumberOfThreadFieldTrialParameter, 0);
760   ParseFieldTrial({&max_thread_number}, trial_string);
761   if (max_thread_number.Get() > 0) {
762     if (width * height < 320 * 180) {
763       return 1;  // Use single thread for small screens
764     }
765     // thread number must be less than or equal to the number of CPUs.
766     return std::min(cpus, max_thread_number.Get());
767   }
768 #endif  // defined(WEBRTC_IOS)
769   if (width * height >= 1920 * 1080 && cpus > 8) {
770     return 8;  // 8 threads for 1080p on high perf machines.
771   } else if (width * height > 1280 * 960 && cpus >= 6) {
772     // 3 threads for 1080p.
773     return 3;
774   } else if (width * height > 640 * 480 && cpus >= 3) {
775     // Default 2 threads for qHD/HD, but allow 3 if core count is high enough,
776     // as this will allow more margin for high-core/low clock machines or if
777     // not built with highest optimization.
778     if (cpus >= 6) {
779       return 3;
780     }
781     return 2;
782   } else {
783     // 1 thread for VGA or less.
784     return 1;
785   }
786 #endif
787 }
788 
InitAndSetControlSettings()789 int LibvpxVp8Encoder::InitAndSetControlSettings() {
790   vpx_codec_flags_t flags = 0;
791   flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
792 
793   if (encoders_.size() > 1) {
794     int error = libvpx_->codec_enc_init_multi(
795         &encoders_[0], vpx_codec_vp8_cx(), &vpx_configs_[0], encoders_.size(),
796         flags, &downsampling_factors_[0]);
797     if (error) {
798       return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
799     }
800   } else {
801     if (libvpx_->codec_enc_init(&encoders_[0], vpx_codec_vp8_cx(),
802                                 &vpx_configs_[0], flags)) {
803       return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
804     }
805   }
806   // Enable denoising for the highest resolution stream, and for
807   // the second highest resolution if we are doing more than 2
808   // spatial layers/streams.
809   // TODO(holmer): Investigate possibility of adding a libvpx API
810   // for getting the denoised frame from the encoder and using that
811   // when encoding lower resolution streams. Would it work with the
812   // multi-res encoding feature?
813   denoiserState denoiser_state = kDenoiserOnYOnly;
814 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
815     defined(WEBRTC_ANDROID)
816   denoiser_state = kDenoiserOnYOnly;
817 #else
818   denoiser_state = kDenoiserOnAdaptive;
819 #endif
820   libvpx_->codec_control(
821       &encoders_[0], VP8E_SET_NOISE_SENSITIVITY,
822       codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
823   if (encoders_.size() > 2) {
824     libvpx_->codec_control(
825         &encoders_[1], VP8E_SET_NOISE_SENSITIVITY,
826         codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
827   }
828   for (size_t i = 0; i < encoders_.size(); ++i) {
829     // Allow more screen content to be detected as static.
830     libvpx_->codec_control(
831         &(encoders_[i]), VP8E_SET_STATIC_THRESHOLD,
832         codec_.mode == VideoCodecMode::kScreensharing ? 100u : 1u);
833     libvpx_->codec_control(&(encoders_[i]), VP8E_SET_CPUUSED, cpu_speed_[i]);
834     libvpx_->codec_control(
835         &(encoders_[i]), VP8E_SET_TOKEN_PARTITIONS,
836         static_cast<vp8e_token_partitions>(kTokenPartitions));
837     libvpx_->codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
838                            rc_max_intra_target_);
839     // VP8E_SET_SCREEN_CONTENT_MODE 2 = screen content with more aggressive
840     // rate control (drop frames on large target bitrate overshoot)
841     libvpx_->codec_control(
842         &(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE,
843         codec_.mode == VideoCodecMode::kScreensharing ? 2u : 0u);
844   }
845   inited_ = true;
846   return WEBRTC_VIDEO_CODEC_OK;
847 }
848 
MaxIntraTarget(uint32_t optimalBuffersize)849 uint32_t LibvpxVp8Encoder::MaxIntraTarget(uint32_t optimalBuffersize) {
850   // Set max to the optimal buffer level (normalized by target BR),
851   // and scaled by a scalePar.
852   // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
853   // This values is presented in percentage of perFrameBw:
854   // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
855   // The target in % is as follows:
856 
857   float scalePar = 0.5;
858   uint32_t targetPct = optimalBuffersize * scalePar * codec_.maxFramerate / 10;
859 
860   // Don't go below 3 times the per frame bandwidth.
861   const uint32_t minIntraTh = 300;
862   return (targetPct < minIntraTh) ? minIntraTh : targetPct;
863 }
864 
FrameDropThreshold(size_t spatial_idx) const865 uint32_t LibvpxVp8Encoder::FrameDropThreshold(size_t spatial_idx) const {
866   bool enable_frame_dropping = codec_.VP8().frameDroppingOn;
867   // If temporal layers are used, they get to override the frame dropping
868   // setting, as eg. ScreenshareLayers does not work as intended with frame
869   // dropping on and DefaultTemporalLayers will have performance issues with
870   // frame dropping off.
871   RTC_DCHECK(frame_buffer_controller_);
872   RTC_DCHECK_LT(spatial_idx, frame_buffer_controller_->StreamCount());
873   enable_frame_dropping =
874       frame_buffer_controller_->SupportsEncoderFrameDropping(spatial_idx);
875   return enable_frame_dropping ? 30 : 0;
876 }
877 
SteadyStateSize(int sid,int tid)878 size_t LibvpxVp8Encoder::SteadyStateSize(int sid, int tid) {
879   const int encoder_id = encoders_.size() - 1 - sid;
880   size_t bitrate_bps;
881   float fps;
882   if (SimulcastUtility::IsConferenceModeScreenshare(codec_) ||
883       vpx_configs_[encoder_id].ts_number_layers <= 1) {
884     // In conference screenshare there's no defined per temporal layer bitrate
885     // and framerate.
886     bitrate_bps = vpx_configs_[encoder_id].rc_target_bitrate * 1000;
887     fps = codec_.maxFramerate;
888   } else {
889     bitrate_bps = vpx_configs_[encoder_id].ts_target_bitrate[tid] * 1000;
890     fps = codec_.maxFramerate /
891           fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid], 1.0);
892     if (tid > 0) {
893       // Layer bitrate and fps are counted as a partial sums.
894       bitrate_bps -= vpx_configs_[encoder_id].ts_target_bitrate[tid - 1] * 1000;
895       fps = codec_.maxFramerate /
896             fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid - 1], 1.0);
897     }
898   }
899 
900   if (fps < 1e-9)
901     return 0;
902   return static_cast<size_t>(
903       bitrate_bps / (8 * fps) *
904           (100 -
905            variable_framerate_experiment_.steady_state_undershoot_percentage) /
906           100 +
907       0.5);
908 }
909 
UpdateVpxConfiguration(size_t stream_index)910 bool LibvpxVp8Encoder::UpdateVpxConfiguration(size_t stream_index) {
911   RTC_DCHECK(frame_buffer_controller_);
912 
913   const size_t config_index = vpx_configs_.size() - 1 - stream_index;
914 
915   RTC_DCHECK_LT(config_index, config_overrides_.size());
916   Vp8EncoderConfig* config = &config_overrides_[config_index];
917 
918   const Vp8EncoderConfig new_config =
919       frame_buffer_controller_->UpdateConfiguration(stream_index);
920 
921   if (new_config.reset_previous_configuration_overrides) {
922     *config = new_config;
923     return true;
924   }
925 
926   const bool changes_made = MaybeExtendVp8EncoderConfig(new_config, config);
927 
928   // Note that overrides must be applied even if they haven't changed.
929   RTC_DCHECK_LT(config_index, vpx_configs_.size());
930   vpx_codec_enc_cfg_t* vpx_config = &vpx_configs_[config_index];
931   ApplyVp8EncoderConfigToVpxConfig(*config, vpx_config);
932 
933   return changes_made;
934 }
935 
Encode(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)936 int LibvpxVp8Encoder::Encode(const VideoFrame& frame,
937                              const std::vector<VideoFrameType>* frame_types) {
938   RTC_DCHECK_EQ(frame.width(), codec_.width);
939   RTC_DCHECK_EQ(frame.height(), codec_.height);
940 
941   if (!inited_)
942     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
943   if (encoded_complete_callback_ == NULL)
944     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
945 
946   bool key_frame_requested = false;
947   for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size();
948        ++i) {
949     if (key_frame_request_[i] && send_stream_[i]) {
950       key_frame_requested = true;
951       break;
952     }
953   }
954   if (!key_frame_requested && frame_types) {
955     for (size_t i = 0; i < frame_types->size() && i < send_stream_.size();
956          ++i) {
957       if ((*frame_types)[i] == VideoFrameType::kVideoFrameKey &&
958           send_stream_[i]) {
959         key_frame_requested = true;
960         break;
961       }
962     }
963   }
964 
965   if (frame.update_rect().IsEmpty() && num_steady_state_frames_ >= 3 &&
966       !key_frame_requested) {
967     if (variable_framerate_experiment_.enabled &&
968         framerate_controller_.DropFrame(frame.timestamp() / kRtpTicksPerMs)) {
969       return WEBRTC_VIDEO_CODEC_OK;
970     }
971     framerate_controller_.AddFrame(frame.timestamp() / kRtpTicksPerMs);
972   }
973 
974   bool send_key_frame = key_frame_requested;
975   bool drop_frame = false;
976   bool retransmission_allowed = true;
977   Vp8FrameConfig tl_configs[kMaxSimulcastStreams];
978   for (size_t i = 0; i < encoders_.size(); ++i) {
979     tl_configs[i] =
980         frame_buffer_controller_->NextFrameConfig(i, frame.timestamp());
981     send_key_frame |= tl_configs[i].IntraFrame();
982     drop_frame |= tl_configs[i].drop_frame;
983     RTC_DCHECK(i == 0 ||
984                retransmission_allowed == tl_configs[i].retransmission_allowed);
985     retransmission_allowed = tl_configs[i].retransmission_allowed;
986   }
987 
988   if (drop_frame && !send_key_frame) {
989     return WEBRTC_VIDEO_CODEC_OK;
990   }
991 
992   vpx_enc_frame_flags_t flags[kMaxSimulcastStreams];
993   for (size_t i = 0; i < encoders_.size(); ++i) {
994     flags[i] = send_key_frame ? VPX_EFLAG_FORCE_KF : EncodeFlags(tl_configs[i]);
995   }
996 
997   rtc::scoped_refptr<I420BufferInterface> input_image =
998       frame.video_frame_buffer()->ToI420();
999   // Since we are extracting raw pointers from |input_image| to
1000   // |raw_images_[0]|, the resolution of these frames must match.
1001   RTC_DCHECK_EQ(input_image->width(), raw_images_[0].d_w);
1002   RTC_DCHECK_EQ(input_image->height(), raw_images_[0].d_h);
1003 
1004   // Image in vpx_image_t format.
1005   // Input image is const. VP8's raw image is not defined as const.
1006   raw_images_[0].planes[VPX_PLANE_Y] =
1007       const_cast<uint8_t*>(input_image->DataY());
1008   raw_images_[0].planes[VPX_PLANE_U] =
1009       const_cast<uint8_t*>(input_image->DataU());
1010   raw_images_[0].planes[VPX_PLANE_V] =
1011       const_cast<uint8_t*>(input_image->DataV());
1012 
1013   raw_images_[0].stride[VPX_PLANE_Y] = input_image->StrideY();
1014   raw_images_[0].stride[VPX_PLANE_U] = input_image->StrideU();
1015   raw_images_[0].stride[VPX_PLANE_V] = input_image->StrideV();
1016 
1017   struct CleanUpOnExit {
1018     explicit CleanUpOnExit(vpx_image_t& raw_image) : raw_image_(raw_image) {}
1019     ~CleanUpOnExit() {
1020       raw_image_.planes[VPX_PLANE_Y] = nullptr;
1021       raw_image_.planes[VPX_PLANE_U] = nullptr;
1022       raw_image_.planes[VPX_PLANE_V] = nullptr;
1023     }
1024     vpx_image_t& raw_image_;
1025   } clean_up_on_exit(raw_images_[0]);
1026 
1027   for (size_t i = 1; i < encoders_.size(); ++i) {
1028     // Scale the image down a number of times by downsampling factor
1029     libyuv::I420Scale(
1030         raw_images_[i - 1].planes[VPX_PLANE_Y],
1031         raw_images_[i - 1].stride[VPX_PLANE_Y],
1032         raw_images_[i - 1].planes[VPX_PLANE_U],
1033         raw_images_[i - 1].stride[VPX_PLANE_U],
1034         raw_images_[i - 1].planes[VPX_PLANE_V],
1035         raw_images_[i - 1].stride[VPX_PLANE_V], raw_images_[i - 1].d_w,
1036         raw_images_[i - 1].d_h, raw_images_[i].planes[VPX_PLANE_Y],
1037         raw_images_[i].stride[VPX_PLANE_Y], raw_images_[i].planes[VPX_PLANE_U],
1038         raw_images_[i].stride[VPX_PLANE_U], raw_images_[i].planes[VPX_PLANE_V],
1039         raw_images_[i].stride[VPX_PLANE_V], raw_images_[i].d_w,
1040         raw_images_[i].d_h, libyuv::kFilterBilinear);
1041   }
1042 
1043   if (send_key_frame) {
1044     // Adapt the size of the key frame when in screenshare with 1 temporal
1045     // layer.
1046     if (encoders_.size() == 1 &&
1047         codec_.mode == VideoCodecMode::kScreensharing &&
1048         codec_.VP8()->numberOfTemporalLayers <= 1) {
1049       const uint32_t forceKeyFrameIntraTh = 100;
1050       libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
1051                              forceKeyFrameIntraTh);
1052     }
1053 
1054     std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
1055   }
1056 
1057   // Set the encoder frame flags and temporal layer_id for each spatial stream.
1058   // Note that streams are defined starting from lowest resolution at
1059   // position 0 to highest resolution at position |encoders_.size() - 1|,
1060   // whereas |encoder_| is from highest to lowest resolution.
1061   for (size_t i = 0; i < encoders_.size(); ++i) {
1062     const size_t stream_idx = encoders_.size() - 1 - i;
1063 
1064     if (UpdateVpxConfiguration(stream_idx)) {
1065       if (libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i]))
1066         return WEBRTC_VIDEO_CODEC_ERROR;
1067     }
1068 
1069     libvpx_->codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS,
1070                            static_cast<int>(flags[stream_idx]));
1071     libvpx_->codec_control(&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID,
1072                            tl_configs[i].encoder_layer_id);
1073   }
1074   // TODO(holmer): Ideally the duration should be the timestamp diff of this
1075   // frame and the next frame to be encoded, which we don't have. Instead we
1076   // would like to use the duration of the previous frame. Unfortunately the
1077   // rate control seems to be off with that setup. Using the average input
1078   // frame rate to calculate an average duration for now.
1079   assert(codec_.maxFramerate > 0);
1080   uint32_t duration = kRtpTicksPerSecond / codec_.maxFramerate;
1081 
1082   int error = WEBRTC_VIDEO_CODEC_OK;
1083   int num_tries = 0;
1084   // If the first try returns WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT
1085   // the frame must be reencoded with the same parameters again because
1086   // target bitrate is exceeded and encoder state has been reset.
1087   while (num_tries == 0 ||
1088          (num_tries == 1 &&
1089           error == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT)) {
1090     ++num_tries;
1091     // Note we must pass 0 for |flags| field in encode call below since they are
1092     // set above in |libvpx_interface_->vpx_codec_control_| function for each
1093     // encoder/spatial layer.
1094     error = libvpx_->codec_encode(&encoders_[0], &raw_images_[0], timestamp_,
1095                                   duration, 0, VPX_DL_REALTIME);
1096     // Reset specific intra frame thresholds, following the key frame.
1097     if (send_key_frame) {
1098       libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
1099                              rc_max_intra_target_);
1100     }
1101     if (error)
1102       return WEBRTC_VIDEO_CODEC_ERROR;
1103     // Examines frame timestamps only.
1104     error = GetEncodedPartitions(frame, retransmission_allowed);
1105   }
1106   // TODO(sprang): Shouldn't we use the frame timestamp instead?
1107   timestamp_ += duration;
1108   return error;
1109 }
1110 
PopulateCodecSpecific(CodecSpecificInfo * codec_specific,const vpx_codec_cx_pkt_t & pkt,int stream_idx,int encoder_idx,uint32_t timestamp)1111 void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
1112                                              const vpx_codec_cx_pkt_t& pkt,
1113                                              int stream_idx,
1114                                              int encoder_idx,
1115                                              uint32_t timestamp) {
1116   assert(codec_specific != NULL);
1117   codec_specific->codecType = kVideoCodecVP8;
1118   codec_specific->codecSpecific.VP8.keyIdx =
1119       kNoKeyIdx;  // TODO(hlundin) populate this
1120   codec_specific->codecSpecific.VP8.nonReference =
1121       (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
1122 
1123   int qp = 0;
1124   vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
1125   frame_buffer_controller_->OnEncodeDone(
1126       stream_idx, timestamp, encoded_images_[encoder_idx].size(),
1127       (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, codec_specific);
1128 }
1129 
GetEncodedPartitions(const VideoFrame & input_image,bool retransmission_allowed)1130 int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image,
1131                                            bool retransmission_allowed) {
1132   int stream_idx = static_cast<int>(encoders_.size()) - 1;
1133   int result = WEBRTC_VIDEO_CODEC_OK;
1134   for (size_t encoder_idx = 0; encoder_idx < encoders_.size();
1135        ++encoder_idx, --stream_idx) {
1136     vpx_codec_iter_t iter = NULL;
1137     encoded_images_[encoder_idx].set_size(0);
1138     encoded_images_[encoder_idx]._frameType = VideoFrameType::kVideoFrameDelta;
1139     CodecSpecificInfo codec_specific;
1140     const vpx_codec_cx_pkt_t* pkt = NULL;
1141 
1142     size_t encoded_size = 0;
1143     while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) !=
1144            NULL) {
1145       if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) {
1146         encoded_size += pkt->data.frame.sz;
1147       }
1148     }
1149 
1150     // TODO(nisse): Introduce some buffer cache or buffer pool, to reduce
1151     // allocations and/or copy operations.
1152     auto buffer = EncodedImageBuffer::Create(encoded_size);
1153 
1154     iter = NULL;
1155     size_t encoded_pos = 0;
1156     while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) !=
1157            NULL) {
1158       switch (pkt->kind) {
1159         case VPX_CODEC_CX_FRAME_PKT: {
1160           RTC_CHECK_LE(encoded_pos + pkt->data.frame.sz, buffer->size());
1161           memcpy(&buffer->data()[encoded_pos], pkt->data.frame.buf,
1162                  pkt->data.frame.sz);
1163           encoded_pos += pkt->data.frame.sz;
1164           break;
1165         }
1166         default:
1167           break;
1168       }
1169       // End of frame
1170       if ((pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT) == 0) {
1171         // check if encoded frame is a key frame
1172         if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
1173           encoded_images_[encoder_idx]._frameType =
1174               VideoFrameType::kVideoFrameKey;
1175         }
1176         encoded_images_[encoder_idx].SetEncodedData(buffer);
1177         encoded_images_[encoder_idx].set_size(encoded_pos);
1178         encoded_images_[encoder_idx].SetSpatialIndex(stream_idx);
1179         PopulateCodecSpecific(&codec_specific, *pkt, stream_idx, encoder_idx,
1180                               input_image.timestamp());
1181         break;
1182       }
1183     }
1184     encoded_images_[encoder_idx].SetTimestamp(input_image.timestamp());
1185     encoded_images_[encoder_idx].SetRetransmissionAllowed(
1186         retransmission_allowed);
1187 
1188     if (send_stream_[stream_idx]) {
1189       if (encoded_images_[encoder_idx].size() > 0) {
1190         TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,
1191                           encoded_images_[encoder_idx].size());
1192         encoded_images_[encoder_idx]._encodedHeight =
1193             codec_.simulcastStream[stream_idx].height;
1194         encoded_images_[encoder_idx]._encodedWidth =
1195             codec_.simulcastStream[stream_idx].width;
1196         int qp_128 = -1;
1197         libvpx_->codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER,
1198                                &qp_128);
1199         encoded_images_[encoder_idx].qp_ = qp_128;
1200         encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx],
1201                                                    &codec_specific, nullptr);
1202         const size_t steady_state_size = SteadyStateSize(
1203             stream_idx, codec_specific.codecSpecific.VP8.temporalIdx);
1204         if (qp_128 > variable_framerate_experiment_.steady_state_qp ||
1205             encoded_images_[encoder_idx].size() > steady_state_size) {
1206           num_steady_state_frames_ = 0;
1207         } else {
1208           ++num_steady_state_frames_;
1209         }
1210       } else if (!frame_buffer_controller_->SupportsEncoderFrameDropping(
1211                      stream_idx)) {
1212         result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
1213         if (encoded_images_[encoder_idx].size() == 0) {
1214           // Dropped frame that will be re-encoded.
1215           frame_buffer_controller_->OnFrameDropped(stream_idx,
1216                                                    input_image.timestamp());
1217         }
1218       }
1219     }
1220   }
1221   return result;
1222 }
1223 
GetEncoderInfo() const1224 VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const {
1225   EncoderInfo info;
1226   info.supports_native_handle = false;
1227   info.implementation_name = "libvpx";
1228   info.has_trusted_rate_controller =
1229       rate_control_settings_.LibvpxVp8TrustedRateController();
1230   info.is_hardware_accelerated = false;
1231   info.has_internal_source = false;
1232   info.supports_simulcast = true;
1233   if (!resolution_bitrate_limits_.empty()) {
1234     info.resolution_bitrate_limits = resolution_bitrate_limits_;
1235   }
1236 
1237   const bool enable_scaling =
1238       num_active_streams_ == 1 &&
1239       (vpx_configs_.empty() || vpx_configs_[0].rc_dropframe_thresh > 0) &&
1240       codec_.VP8().automaticResizeOn;
1241 
1242   info.scaling_settings = enable_scaling
1243                               ? VideoEncoder::ScalingSettings(
1244                                     kLowVp8QpThreshold, kHighVp8QpThreshold)
1245                               : VideoEncoder::ScalingSettings::kOff;
1246   if (rate_control_settings_.LibvpxVp8MinPixels()) {
1247     info.scaling_settings.min_pixels_per_frame =
1248         rate_control_settings_.LibvpxVp8MinPixels().value();
1249   }
1250 
1251   if (inited_) {
1252     // |encoder_idx| is libvpx index where 0 is highest resolution.
1253     // |si| is simulcast index, where 0 is lowest resolution.
1254     for (size_t si = 0, encoder_idx = encoders_.size() - 1;
1255          si < encoders_.size(); ++si, --encoder_idx) {
1256       info.fps_allocation[si].clear();
1257       if ((codec_.numberOfSimulcastStreams > si &&
1258            !codec_.simulcastStream[si].active) ||
1259           (si == 0 && SimulcastUtility::IsConferenceModeScreenshare(codec_))) {
1260         // No defined frame rate fractions if not active or if using
1261         // ScreenshareLayers, leave vector empty and continue;
1262         continue;
1263       }
1264       if (vpx_configs_[encoder_idx].ts_number_layers <= 1) {
1265         info.fps_allocation[si].push_back(EncoderInfo::kMaxFramerateFraction);
1266       } else {
1267         for (size_t ti = 0; ti < vpx_configs_[encoder_idx].ts_number_layers;
1268              ++ti) {
1269           RTC_DCHECK_GT(vpx_configs_[encoder_idx].ts_rate_decimator[ti], 0);
1270           info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>(
1271               EncoderInfo::kMaxFramerateFraction /
1272                   vpx_configs_[encoder_idx].ts_rate_decimator[ti] +
1273               0.5));
1274         }
1275       }
1276     }
1277   }
1278 
1279   return info;
1280 }
1281 
RegisterEncodeCompleteCallback(EncodedImageCallback * callback)1282 int LibvpxVp8Encoder::RegisterEncodeCompleteCallback(
1283     EncodedImageCallback* callback) {
1284   encoded_complete_callback_ = callback;
1285   return WEBRTC_VIDEO_CODEC_OK;
1286 }
1287 
1288 // static
1289 LibvpxVp8Encoder::VariableFramerateExperiment
ParseVariableFramerateConfig(std::string group_name)1290 LibvpxVp8Encoder::ParseVariableFramerateConfig(std::string group_name) {
1291   FieldTrialFlag enabled = FieldTrialFlag("Enabled");
1292   FieldTrialParameter<double> framerate_limit("min_fps", 5.0);
1293   FieldTrialParameter<int> qp("min_qp", 15);
1294   FieldTrialParameter<int> undershoot_percentage("undershoot", 30);
1295   ParseFieldTrial({&enabled, &framerate_limit, &qp, &undershoot_percentage},
1296                   field_trial::FindFullName(group_name));
1297   VariableFramerateExperiment config;
1298   config.enabled = enabled.Get();
1299   config.framerate_limit = framerate_limit.Get();
1300   config.steady_state_qp = qp.Get();
1301   config.steady_state_undershoot_percentage = undershoot_percentage.Get();
1302 
1303   return config;
1304 }
1305 
1306 }  // namespace webrtc
1307