1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
12
13 #include <assert.h>
14 #include <string.h>
15
16 #include <algorithm>
17 #include <cstdint>
18 #include <iterator>
19 #include <memory>
20 #include <string>
21 #include <utility>
22 #include <vector>
23
24 #include "api/scoped_refptr.h"
25 #include "api/video/video_content_type.h"
26 #include "api/video/video_frame_buffer.h"
27 #include "api/video/video_timing.h"
28 #include "api/video_codecs/vp8_temporal_layers.h"
29 #include "api/video_codecs/vp8_temporal_layers_factory.h"
30 #include "modules/video_coding/codecs/interface/common_constants.h"
31 #include "modules/video_coding/codecs/vp8/include/vp8.h"
32 #include "modules/video_coding/include/video_error_codes.h"
33 #include "modules/video_coding/utility/simulcast_rate_allocator.h"
34 #include "modules/video_coding/utility/simulcast_utility.h"
35 #include "rtc_base/checks.h"
36 #include "rtc_base/experiments/field_trial_parser.h"
37 #include "rtc_base/experiments/field_trial_units.h"
38 #include "rtc_base/logging.h"
39 #include "rtc_base/trace_event.h"
40 #include "system_wrappers/include/field_trial.h"
41 #include "third_party/libyuv/include/libyuv/scale.h"
42 #include "vpx/vp8cx.h"
43
44 namespace webrtc {
45 namespace {
46 #if defined(WEBRTC_IOS)
47 const char kVP8IosMaxNumberOfThreadFieldTrial[] =
48 "WebRTC-VP8IosMaxNumberOfThread";
49 const char kVP8IosMaxNumberOfThreadFieldTrialParameter[] = "max_thread";
50 #endif
51
52 const char kVp8ForcePartitionResilience[] =
53 "WebRTC-VP8-ForcePartitionResilience";
54
55 // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
56 // bitstream range of [0, 127] and not the user-level range of [0,63].
57 constexpr int kLowVp8QpThreshold = 29;
58 constexpr int kHighVp8QpThreshold = 95;
59
60 constexpr int kTokenPartitions = VP8_ONE_TOKENPARTITION;
61 constexpr uint32_t kVp832ByteAlign = 32u;
62
63 constexpr int kRtpTicksPerSecond = 90000;
64 constexpr int kRtpTicksPerMs = kRtpTicksPerSecond / 1000;
65
66 constexpr double kLowRateFactor = 1.0;
67 constexpr double kHighRateFactor = 2.0;
68
69 // VP8 denoiser states.
70 enum denoiserState : uint32_t {
71 kDenoiserOff,
72 kDenoiserOnYOnly,
73 kDenoiserOnYUV,
74 kDenoiserOnYUVAggressive,
75 // Adaptive mode defaults to kDenoiserOnYUV on key frame, but may switch
76 // to kDenoiserOnYUVAggressive based on a computed noise metric.
77 kDenoiserOnAdaptive
78 };
79
80 // These settings correspond to the settings in vpx_codec_enc_cfg.
81 struct Vp8RateSettings {
82 uint32_t rc_undershoot_pct;
83 uint32_t rc_overshoot_pct;
84 uint32_t rc_buf_sz;
85 uint32_t rc_buf_optimal_sz;
86 uint32_t rc_dropframe_thresh;
87 };
88
89 // Greatest common divisior
GCD(int a,int b)90 int GCD(int a, int b) {
91 int c = a % b;
92 while (c != 0) {
93 a = b;
94 b = c;
95 c = a % b;
96 }
97 return b;
98 }
99
Interpolate(uint32_t low,uint32_t high,double bandwidth_headroom_factor)100 uint32_t Interpolate(uint32_t low,
101 uint32_t high,
102 double bandwidth_headroom_factor) {
103 RTC_DCHECK_GE(bandwidth_headroom_factor, kLowRateFactor);
104 RTC_DCHECK_LE(bandwidth_headroom_factor, kHighRateFactor);
105
106 // |factor| is between 0.0 and 1.0.
107 const double factor = bandwidth_headroom_factor - kLowRateFactor;
108
109 return static_cast<uint32_t>(((1.0 - factor) * low) + (factor * high) + 0.5);
110 }
111
GetRateSettings(double bandwidth_headroom_factor)112 Vp8RateSettings GetRateSettings(double bandwidth_headroom_factor) {
113 static const Vp8RateSettings low_settings{1000u, 0u, 100u, 30u, 40u};
114 static const Vp8RateSettings high_settings{100u, 15u, 1000u, 600u, 5u};
115
116 if (bandwidth_headroom_factor <= kLowRateFactor) {
117 return low_settings;
118 } else if (bandwidth_headroom_factor >= kHighRateFactor) {
119 return high_settings;
120 }
121
122 Vp8RateSettings settings;
123 settings.rc_undershoot_pct =
124 Interpolate(low_settings.rc_undershoot_pct,
125 high_settings.rc_undershoot_pct, bandwidth_headroom_factor);
126 settings.rc_overshoot_pct =
127 Interpolate(low_settings.rc_overshoot_pct, high_settings.rc_overshoot_pct,
128 bandwidth_headroom_factor);
129 settings.rc_buf_sz =
130 Interpolate(low_settings.rc_buf_sz, high_settings.rc_buf_sz,
131 bandwidth_headroom_factor);
132 settings.rc_buf_optimal_sz =
133 Interpolate(low_settings.rc_buf_optimal_sz,
134 high_settings.rc_buf_optimal_sz, bandwidth_headroom_factor);
135 settings.rc_dropframe_thresh =
136 Interpolate(low_settings.rc_dropframe_thresh,
137 high_settings.rc_dropframe_thresh, bandwidth_headroom_factor);
138 return settings;
139 }
140
UpdateRateSettings(vpx_codec_enc_cfg_t * config,const Vp8RateSettings & new_settings)141 void UpdateRateSettings(vpx_codec_enc_cfg_t* config,
142 const Vp8RateSettings& new_settings) {
143 config->rc_undershoot_pct = new_settings.rc_undershoot_pct;
144 config->rc_overshoot_pct = new_settings.rc_overshoot_pct;
145 config->rc_buf_sz = new_settings.rc_buf_sz;
146 config->rc_buf_optimal_sz = new_settings.rc_buf_optimal_sz;
147 config->rc_dropframe_thresh = new_settings.rc_dropframe_thresh;
148 }
149
150 static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxPeriodicity ==
151 VPX_TS_MAX_PERIODICITY,
152 "Vp8EncoderConfig::kMaxPeriodicity must be kept in sync with the "
153 "constant in libvpx.");
154 static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxLayers ==
155 VPX_TS_MAX_LAYERS,
156 "Vp8EncoderConfig::kMaxLayers must be kept in sync with the "
157 "constant in libvpx.");
158
159 // Allow a newer value to override a current value only if the new value
160 // is set.
161 template <typename T>
MaybeSetNewValue(const absl::optional<T> & new_value,absl::optional<T> * base_value)162 bool MaybeSetNewValue(const absl::optional<T>& new_value,
163 absl::optional<T>* base_value) {
164 if (new_value.has_value() && new_value != *base_value) {
165 *base_value = new_value;
166 return true;
167 } else {
168 return false;
169 }
170 }
171
172 // Adds configuration from |new_config| to |base_config|. Both configs consist
173 // of optionals, and only optionals which are set in |new_config| can have
174 // an effect. (That is, set values in |base_config| cannot be unset.)
175 // Returns |true| iff any changes were made to |base_config|.
MaybeExtendVp8EncoderConfig(const Vp8EncoderConfig & new_config,Vp8EncoderConfig * base_config)176 bool MaybeExtendVp8EncoderConfig(const Vp8EncoderConfig& new_config,
177 Vp8EncoderConfig* base_config) {
178 bool changes_made = false;
179 changes_made |= MaybeSetNewValue(new_config.temporal_layer_config,
180 &base_config->temporal_layer_config);
181 changes_made |= MaybeSetNewValue(new_config.rc_target_bitrate,
182 &base_config->rc_target_bitrate);
183 changes_made |= MaybeSetNewValue(new_config.rc_max_quantizer,
184 &base_config->rc_max_quantizer);
185 changes_made |= MaybeSetNewValue(new_config.g_error_resilient,
186 &base_config->g_error_resilient);
187 return changes_made;
188 }
189
ApplyVp8EncoderConfigToVpxConfig(const Vp8EncoderConfig & encoder_config,vpx_codec_enc_cfg_t * vpx_config)190 void ApplyVp8EncoderConfigToVpxConfig(const Vp8EncoderConfig& encoder_config,
191 vpx_codec_enc_cfg_t* vpx_config) {
192 if (encoder_config.temporal_layer_config.has_value()) {
193 const Vp8EncoderConfig::TemporalLayerConfig& ts_config =
194 encoder_config.temporal_layer_config.value();
195 vpx_config->ts_number_layers = ts_config.ts_number_layers;
196 std::copy(ts_config.ts_target_bitrate.begin(),
197 ts_config.ts_target_bitrate.end(),
198 std::begin(vpx_config->ts_target_bitrate));
199 std::copy(ts_config.ts_rate_decimator.begin(),
200 ts_config.ts_rate_decimator.end(),
201 std::begin(vpx_config->ts_rate_decimator));
202 vpx_config->ts_periodicity = ts_config.ts_periodicity;
203 std::copy(ts_config.ts_layer_id.begin(), ts_config.ts_layer_id.end(),
204 std::begin(vpx_config->ts_layer_id));
205 } else {
206 vpx_config->ts_number_layers = 1;
207 vpx_config->ts_rate_decimator[0] = 1;
208 vpx_config->ts_periodicity = 1;
209 vpx_config->ts_layer_id[0] = 0;
210 }
211
212 if (encoder_config.rc_target_bitrate.has_value()) {
213 vpx_config->rc_target_bitrate = encoder_config.rc_target_bitrate.value();
214 }
215
216 if (encoder_config.rc_max_quantizer.has_value()) {
217 vpx_config->rc_max_quantizer = encoder_config.rc_max_quantizer.value();
218 }
219
220 if (encoder_config.g_error_resilient.has_value()) {
221 vpx_config->g_error_resilient = encoder_config.g_error_resilient.value();
222 }
223 }
224
225 } // namespace
226
Create()227 std::unique_ptr<VideoEncoder> VP8Encoder::Create() {
228 return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::CreateEncoder(),
229 VP8Encoder::Settings());
230 }
231
Create(VP8Encoder::Settings settings)232 std::unique_ptr<VideoEncoder> VP8Encoder::Create(
233 VP8Encoder::Settings settings) {
234 return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::CreateEncoder(),
235 std::move(settings));
236 }
237
Create(std::unique_ptr<Vp8FrameBufferControllerFactory> frame_buffer_controller_factory)238 std::unique_ptr<VideoEncoder> VP8Encoder::Create(
239 std::unique_ptr<Vp8FrameBufferControllerFactory>
240 frame_buffer_controller_factory) {
241 VP8Encoder::Settings settings;
242 settings.frame_buffer_controller_factory =
243 std::move(frame_buffer_controller_factory);
244 return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::CreateEncoder(),
245 std::move(settings));
246 }
247
EncodeFlags(const Vp8FrameConfig & references)248 vpx_enc_frame_flags_t LibvpxVp8Encoder::EncodeFlags(
249 const Vp8FrameConfig& references) {
250 RTC_DCHECK(!references.drop_frame);
251
252 vpx_enc_frame_flags_t flags = 0;
253
254 if ((references.last_buffer_flags &
255 Vp8FrameConfig::BufferFlags::kReference) == 0)
256 flags |= VP8_EFLAG_NO_REF_LAST;
257 if ((references.last_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) ==
258 0)
259 flags |= VP8_EFLAG_NO_UPD_LAST;
260 if ((references.golden_buffer_flags &
261 Vp8FrameConfig::BufferFlags::kReference) == 0)
262 flags |= VP8_EFLAG_NO_REF_GF;
263 if ((references.golden_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) ==
264 0)
265 flags |= VP8_EFLAG_NO_UPD_GF;
266 if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kReference) ==
267 0)
268 flags |= VP8_EFLAG_NO_REF_ARF;
269 if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == 0)
270 flags |= VP8_EFLAG_NO_UPD_ARF;
271 if (references.freeze_entropy)
272 flags |= VP8_EFLAG_NO_UPD_ENTROPY;
273
274 return flags;
275 }
276
LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,VP8Encoder::Settings settings)277 LibvpxVp8Encoder::LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,
278 VP8Encoder::Settings settings)
279 : libvpx_(std::move(interface)),
280 experimental_cpu_speed_config_arm_(CpuSpeedExperiment::GetConfigs()),
281 rate_control_settings_(RateControlSettings::ParseFromFieldTrials()),
282 frame_buffer_controller_factory_(
283 std::move(settings.frame_buffer_controller_factory)),
284 resolution_bitrate_limits_(std::move(settings.resolution_bitrate_limits)),
285 key_frame_request_(kMaxSimulcastStreams, false),
286 variable_framerate_experiment_(ParseVariableFramerateConfig(
287 "WebRTC-VP8VariableFramerateScreenshare")),
288 framerate_controller_(variable_framerate_experiment_.framerate_limit) {
289 // TODO(eladalon/ilnik): These reservations might be wasting memory.
290 // InitEncode() is resizing to the actual size, which might be smaller.
291 raw_images_.reserve(kMaxSimulcastStreams);
292 encoded_images_.reserve(kMaxSimulcastStreams);
293 send_stream_.reserve(kMaxSimulcastStreams);
294 cpu_speed_.assign(kMaxSimulcastStreams, cpu_speed_default_);
295 encoders_.reserve(kMaxSimulcastStreams);
296 vpx_configs_.reserve(kMaxSimulcastStreams);
297 config_overrides_.reserve(kMaxSimulcastStreams);
298 downsampling_factors_.reserve(kMaxSimulcastStreams);
299 }
300
~LibvpxVp8Encoder()301 LibvpxVp8Encoder::~LibvpxVp8Encoder() {
302 Release();
303 }
304
Release()305 int LibvpxVp8Encoder::Release() {
306 int ret_val = WEBRTC_VIDEO_CODEC_OK;
307
308 encoded_images_.clear();
309
310 if (inited_) {
311 for (auto it = encoders_.rbegin(); it != encoders_.rend(); ++it) {
312 if (libvpx_->codec_destroy(&*it)) {
313 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
314 }
315 }
316 }
317 encoders_.clear();
318
319 vpx_configs_.clear();
320 config_overrides_.clear();
321 send_stream_.clear();
322 cpu_speed_.clear();
323
324 for (auto it = raw_images_.rbegin(); it != raw_images_.rend(); ++it) {
325 libvpx_->img_free(&*it);
326 }
327 raw_images_.clear();
328
329 frame_buffer_controller_.reset();
330 inited_ = false;
331 return ret_val;
332 }
333
SetRates(const RateControlParameters & parameters)334 void LibvpxVp8Encoder::SetRates(const RateControlParameters& parameters) {
335 if (!inited_) {
336 RTC_LOG(LS_WARNING) << "SetRates() while not initialize";
337 return;
338 }
339
340 if (encoders_[0].err) {
341 RTC_LOG(LS_WARNING) << "Encoder in error state.";
342 return;
343 }
344
345 if (parameters.framerate_fps < 1.0) {
346 RTC_LOG(LS_WARNING) << "Unsupported framerate (must be >= 1.0): "
347 << parameters.framerate_fps;
348 return;
349 }
350
351 if (parameters.bitrate.get_sum_bps() == 0) {
352 // Encoder paused, turn off all encoding.
353 const int num_streams = static_cast<size_t>(encoders_.size());
354 for (int i = 0; i < num_streams; ++i)
355 SetStreamState(false, i);
356 return;
357 }
358
359 codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
360
361 if (encoders_.size() > 1) {
362 // If we have more than 1 stream, reduce the qp_max for the low resolution
363 // stream if frame rate is not too low. The trade-off with lower qp_max is
364 // possibly more dropped frames, so we only do this if the frame rate is
365 // above some threshold (base temporal layer is down to 1/4 for 3 layers).
366 // We may want to condition this on bitrate later.
367 if (rate_control_settings_.Vp8BoostBaseLayerQuality() &&
368 parameters.framerate_fps > 20.0) {
369 vpx_configs_[encoders_.size() - 1].rc_max_quantizer = 45;
370 } else {
371 // Go back to default value set in InitEncode.
372 vpx_configs_[encoders_.size() - 1].rc_max_quantizer = qp_max_;
373 }
374 }
375
376 for (size_t i = 0; i < encoders_.size(); ++i) {
377 const size_t stream_idx = encoders_.size() - 1 - i;
378
379 unsigned int target_bitrate_kbps =
380 parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;
381
382 bool send_stream = target_bitrate_kbps > 0;
383 if (send_stream || encoders_.size() > 1)
384 SetStreamState(send_stream, stream_idx);
385
386 vpx_configs_[i].rc_target_bitrate = target_bitrate_kbps;
387 if (send_stream) {
388 frame_buffer_controller_->OnRatesUpdated(
389 stream_idx, parameters.bitrate.GetTemporalLayerAllocation(stream_idx),
390 static_cast<int>(parameters.framerate_fps + 0.5));
391 }
392
393 UpdateVpxConfiguration(stream_idx);
394
395 if (rate_control_settings_.Vp8DynamicRateSettings()) {
396 // Tweak rate control settings based on available network headroom.
397 UpdateRateSettings(
398 &vpx_configs_[i],
399 GetRateSettings(parameters.bandwidth_allocation.bps<double>() /
400 parameters.bitrate.get_sum_bps()));
401 }
402
403 vpx_codec_err_t err =
404 libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i]);
405 if (err != VPX_CODEC_OK) {
406 RTC_LOG(LS_WARNING) << "Error configuring codec, error code: " << err;
407 }
408 }
409 }
410
OnPacketLossRateUpdate(float packet_loss_rate)411 void LibvpxVp8Encoder::OnPacketLossRateUpdate(float packet_loss_rate) {
412 // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK.
413 if (frame_buffer_controller_) {
414 frame_buffer_controller_->OnPacketLossRateUpdate(packet_loss_rate);
415 }
416 }
417
OnRttUpdate(int64_t rtt_ms)418 void LibvpxVp8Encoder::OnRttUpdate(int64_t rtt_ms) {
419 // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK.
420 if (frame_buffer_controller_) {
421 frame_buffer_controller_->OnRttUpdate(rtt_ms);
422 }
423 }
424
OnLossNotification(const LossNotification & loss_notification)425 void LibvpxVp8Encoder::OnLossNotification(
426 const LossNotification& loss_notification) {
427 if (frame_buffer_controller_) {
428 frame_buffer_controller_->OnLossNotification(loss_notification);
429 }
430 }
431
SetStreamState(bool send_stream,int stream_idx)432 void LibvpxVp8Encoder::SetStreamState(bool send_stream, int stream_idx) {
433 if (send_stream && !send_stream_[stream_idx]) {
434 // Need a key frame if we have not sent this stream before.
435 key_frame_request_[stream_idx] = true;
436 }
437 send_stream_[stream_idx] = send_stream;
438 }
439
SetFecControllerOverride(FecControllerOverride * fec_controller_override)440 void LibvpxVp8Encoder::SetFecControllerOverride(
441 FecControllerOverride* fec_controller_override) {
442 // TODO(bugs.webrtc.org/10769): Update downstream and remove ability to
443 // pass nullptr.
444 // RTC_DCHECK(fec_controller_override);
445 RTC_DCHECK(!fec_controller_override_);
446 fec_controller_override_ = fec_controller_override;
447 }
448
449 // TODO(eladalon): s/inst/codec_settings/g.
InitEncode(const VideoCodec * inst,const VideoEncoder::Settings & settings)450 int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst,
451 const VideoEncoder::Settings& settings) {
452 if (inst == NULL) {
453 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
454 }
455 if (inst->maxFramerate < 1) {
456 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
457 }
458 // allow zero to represent an unspecified maxBitRate
459 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
460 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
461 }
462 if (inst->width < 1 || inst->height < 1) {
463 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
464 }
465 if (settings.number_of_cores < 1) {
466 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
467 }
468
469 num_active_streams_ = 0;
470 for (int i = 0; i < inst->numberOfSimulcastStreams; ++i) {
471 if (inst->simulcastStream[i].active) {
472 ++num_active_streams_;
473 }
474 }
475 if (inst->numberOfSimulcastStreams == 0 && inst->active) {
476 num_active_streams_ = 1;
477 }
478
479 if (inst->VP8().automaticResizeOn && num_active_streams_ > 1) {
480 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
481 }
482
483 int retVal = Release();
484 if (retVal < 0) {
485 return retVal;
486 }
487
488 int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
489 if (number_of_streams > 1 &&
490 !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
491 return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
492 }
493
494 RTC_DCHECK(!frame_buffer_controller_);
495 if (frame_buffer_controller_factory_) {
496 frame_buffer_controller_ = frame_buffer_controller_factory_->Create(
497 *inst, settings, fec_controller_override_);
498 } else {
499 Vp8TemporalLayersFactory factory;
500 frame_buffer_controller_ =
501 factory.Create(*inst, settings, fec_controller_override_);
502 }
503 RTC_DCHECK(frame_buffer_controller_);
504
505 number_of_cores_ = settings.number_of_cores;
506 timestamp_ = 0;
507 codec_ = *inst;
508
509 // Code expects simulcastStream resolutions to be correct, make sure they are
510 // filled even when there are no simulcast layers.
511 if (codec_.numberOfSimulcastStreams == 0) {
512 codec_.simulcastStream[0].width = codec_.width;
513 codec_.simulcastStream[0].height = codec_.height;
514 }
515
516 encoded_images_.resize(number_of_streams);
517 encoders_.resize(number_of_streams);
518 vpx_configs_.resize(number_of_streams);
519 config_overrides_.resize(number_of_streams);
520 downsampling_factors_.resize(number_of_streams);
521 raw_images_.resize(number_of_streams);
522 send_stream_.resize(number_of_streams);
523 send_stream_[0] = true; // For non-simulcast case.
524 cpu_speed_.resize(number_of_streams);
525 std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
526
527 int idx = number_of_streams - 1;
528 for (int i = 0; i < (number_of_streams - 1); ++i, --idx) {
529 int gcd = GCD(inst->simulcastStream[idx].width,
530 inst->simulcastStream[idx - 1].width);
531 downsampling_factors_[i].num = inst->simulcastStream[idx].width / gcd;
532 downsampling_factors_[i].den = inst->simulcastStream[idx - 1].width / gcd;
533 send_stream_[i] = false;
534 }
535 if (number_of_streams > 1) {
536 send_stream_[number_of_streams - 1] = false;
537 downsampling_factors_[number_of_streams - 1].num = 1;
538 downsampling_factors_[number_of_streams - 1].den = 1;
539 }
540 for (int i = 0; i < number_of_streams; ++i) {
541 encoded_images_[i]._completeFrame = true;
542 }
543 // populate encoder configuration with default values
544 if (libvpx_->codec_enc_config_default(vpx_codec_vp8_cx(), &vpx_configs_[0],
545 0)) {
546 return WEBRTC_VIDEO_CODEC_ERROR;
547 }
548 // setting the time base of the codec
549 vpx_configs_[0].g_timebase.num = 1;
550 vpx_configs_[0].g_timebase.den = kRtpTicksPerSecond;
551 vpx_configs_[0].g_lag_in_frames = 0; // 0- no frame lagging
552
553 // Set the error resilience mode for temporal layers (but not simulcast).
554 vpx_configs_[0].g_error_resilient =
555 (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1)
556 ? VPX_ERROR_RESILIENT_DEFAULT
557 : 0;
558
559 // Override the error resilience mode if this is not simulcast, but we are
560 // using temporal layers.
561 if (field_trial::IsEnabled(kVp8ForcePartitionResilience) &&
562 (number_of_streams == 1) &&
563 (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1)) {
564 RTC_LOG(LS_INFO) << "Overriding g_error_resilient from "
565 << vpx_configs_[0].g_error_resilient << " to "
566 << VPX_ERROR_RESILIENT_PARTITIONS;
567 vpx_configs_[0].g_error_resilient = VPX_ERROR_RESILIENT_PARTITIONS;
568 }
569
570 // rate control settings
571 vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(0);
572 vpx_configs_[0].rc_end_usage = VPX_CBR;
573 vpx_configs_[0].g_pass = VPX_RC_ONE_PASS;
574 // Handle resizing outside of libvpx.
575 vpx_configs_[0].rc_resize_allowed = 0;
576 vpx_configs_[0].rc_min_quantizer =
577 codec_.mode == VideoCodecMode::kScreensharing ? 12 : 2;
578 if (inst->qpMax >= vpx_configs_[0].rc_min_quantizer) {
579 qp_max_ = inst->qpMax;
580 }
581 if (rate_control_settings_.LibvpxVp8QpMax()) {
582 qp_max_ = std::max(rate_control_settings_.LibvpxVp8QpMax().value(),
583 static_cast<int>(vpx_configs_[0].rc_min_quantizer));
584 }
585 vpx_configs_[0].rc_max_quantizer = qp_max_;
586 vpx_configs_[0].rc_undershoot_pct = 100;
587 vpx_configs_[0].rc_overshoot_pct = 15;
588 vpx_configs_[0].rc_buf_initial_sz = 500;
589 vpx_configs_[0].rc_buf_optimal_sz = 600;
590 vpx_configs_[0].rc_buf_sz = 1000;
591
592 // Set the maximum target size of any key-frame.
593 rc_max_intra_target_ = MaxIntraTarget(vpx_configs_[0].rc_buf_optimal_sz);
594
595 if (inst->VP8().keyFrameInterval > 0) {
596 vpx_configs_[0].kf_mode = VPX_KF_AUTO;
597 vpx_configs_[0].kf_max_dist = inst->VP8().keyFrameInterval;
598 } else {
599 vpx_configs_[0].kf_mode = VPX_KF_DISABLED;
600 }
601
602 // Allow the user to set the complexity for the base stream.
603 switch (inst->VP8().complexity) {
604 case VideoCodecComplexity::kComplexityHigh:
605 cpu_speed_[0] = -5;
606 break;
607 case VideoCodecComplexity::kComplexityHigher:
608 cpu_speed_[0] = -4;
609 break;
610 case VideoCodecComplexity::kComplexityMax:
611 cpu_speed_[0] = -3;
612 break;
613 default:
614 cpu_speed_[0] = -6;
615 break;
616 }
617 cpu_speed_default_ = cpu_speed_[0];
618 // Set encoding complexity (cpu_speed) based on resolution and/or platform.
619 cpu_speed_[0] = GetCpuSpeed(inst->width, inst->height);
620 for (int i = 1; i < number_of_streams; ++i) {
621 cpu_speed_[i] =
622 GetCpuSpeed(inst->simulcastStream[number_of_streams - 1 - i].width,
623 inst->simulcastStream[number_of_streams - 1 - i].height);
624 }
625 vpx_configs_[0].g_w = inst->width;
626 vpx_configs_[0].g_h = inst->height;
627
628 // Determine number of threads based on the image size and #cores.
629 // TODO(fbarchard): Consider number of Simulcast layers.
630 vpx_configs_[0].g_threads = NumberOfThreads(
631 vpx_configs_[0].g_w, vpx_configs_[0].g_h, settings.number_of_cores);
632
633 // Creating a wrapper to the image - setting image data to NULL.
634 // Actual pointer will be set in encode. Setting align to 1, as it
635 // is meaningless (no memory allocation is done here).
636 libvpx_->img_wrap(&raw_images_[0], VPX_IMG_FMT_I420, inst->width,
637 inst->height, 1, NULL);
638
639 // Note the order we use is different from webm, we have lowest resolution
640 // at position 0 and they have highest resolution at position 0.
641 const size_t stream_idx_cfg_0 = encoders_.size() - 1;
642 SimulcastRateAllocator init_allocator(codec_);
643 VideoBitrateAllocation allocation =
644 init_allocator.Allocate(VideoBitrateAllocationParameters(
645 inst->startBitrate * 1000, inst->maxFramerate));
646 std::vector<uint32_t> stream_bitrates;
647 for (int i = 0; i == 0 || i < inst->numberOfSimulcastStreams; ++i) {
648 uint32_t bitrate = allocation.GetSpatialLayerSum(i) / 1000;
649 stream_bitrates.push_back(bitrate);
650 }
651
652 vpx_configs_[0].rc_target_bitrate = stream_bitrates[stream_idx_cfg_0];
653 if (stream_bitrates[stream_idx_cfg_0] > 0) {
654 uint32_t maxFramerate =
655 inst->simulcastStream[stream_idx_cfg_0].maxFramerate;
656 if (!maxFramerate) {
657 maxFramerate = inst->maxFramerate;
658 }
659
660 frame_buffer_controller_->OnRatesUpdated(
661 stream_idx_cfg_0,
662 allocation.GetTemporalLayerAllocation(stream_idx_cfg_0), maxFramerate);
663 }
664 frame_buffer_controller_->SetQpLimits(stream_idx_cfg_0,
665 vpx_configs_[0].rc_min_quantizer,
666 vpx_configs_[0].rc_max_quantizer);
667 UpdateVpxConfiguration(stream_idx_cfg_0);
668 vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(stream_idx_cfg_0);
669
670 for (size_t i = 1; i < encoders_.size(); ++i) {
671 const size_t stream_idx = encoders_.size() - 1 - i;
672 memcpy(&vpx_configs_[i], &vpx_configs_[0], sizeof(vpx_configs_[0]));
673
674 vpx_configs_[i].g_w = inst->simulcastStream[stream_idx].width;
675 vpx_configs_[i].g_h = inst->simulcastStream[stream_idx].height;
676
677 // Use 1 thread for lower resolutions.
678 vpx_configs_[i].g_threads = 1;
679
680 vpx_configs_[i].rc_dropframe_thresh = FrameDropThreshold(stream_idx);
681
682 // Setting alignment to 32 - as that ensures at least 16 for all
683 // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for
684 // the y plane, but only half of it to the u and v planes.
685 libvpx_->img_alloc(&raw_images_[i], VPX_IMG_FMT_I420,
686 inst->simulcastStream[stream_idx].width,
687 inst->simulcastStream[stream_idx].height,
688 kVp832ByteAlign);
689 SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx);
690 vpx_configs_[i].rc_target_bitrate = stream_bitrates[stream_idx];
691 if (stream_bitrates[stream_idx] > 0) {
692 uint32_t maxFramerate = inst->simulcastStream[stream_idx].maxFramerate;
693 if (!maxFramerate) {
694 maxFramerate = inst->maxFramerate;
695 }
696 frame_buffer_controller_->OnRatesUpdated(
697 stream_idx, allocation.GetTemporalLayerAllocation(stream_idx),
698 maxFramerate);
699 }
700 frame_buffer_controller_->SetQpLimits(stream_idx,
701 vpx_configs_[i].rc_min_quantizer,
702 vpx_configs_[i].rc_max_quantizer);
703 UpdateVpxConfiguration(stream_idx);
704 }
705
706 return InitAndSetControlSettings();
707 }
708
GetCpuSpeed(int width,int height)709 int LibvpxVp8Encoder::GetCpuSpeed(int width, int height) {
710 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
711 defined(WEBRTC_ANDROID)
712 // On mobile platform, use a lower speed setting for lower resolutions for
713 // CPUs with 4 or more cores.
714 RTC_DCHECK_GT(number_of_cores_, 0);
715 if (number_of_cores_ <= 3)
716 return -12;
717
718 if (experimental_cpu_speed_config_arm_) {
719 return CpuSpeedExperiment::GetValue(width * height,
720 *experimental_cpu_speed_config_arm_);
721 }
722
723 if (width * height <= 352 * 288)
724 return -8;
725 else if (width * height <= 640 * 480)
726 return -10;
727 else
728 return -12;
729 #else
730 // For non-ARM, increase encoding complexity (i.e., use lower speed setting)
731 // if resolution is below CIF. Otherwise, keep the default/user setting
732 // (|cpu_speed_default_|) set on InitEncode via VP8().complexity.
733 if (width * height < 352 * 288)
734 return (cpu_speed_default_ < -4) ? -4 : cpu_speed_default_;
735 else
736 return cpu_speed_default_;
737 #endif
738 }
739
NumberOfThreads(int width,int height,int cpus)740 int LibvpxVp8Encoder::NumberOfThreads(int width, int height, int cpus) {
741 #if defined(WEBRTC_ANDROID)
742 if (width * height >= 320 * 180) {
743 if (cpus >= 4) {
744 // 3 threads for CPUs with 4 and more cores since most of times only 4
745 // cores will be active.
746 return 3;
747 } else if (cpus == 3 || cpus == 2) {
748 return 2;
749 } else {
750 return 1;
751 }
752 }
753 return 1;
754 #else
755 #if defined(WEBRTC_IOS)
756 std::string trial_string =
757 field_trial::FindFullName(kVP8IosMaxNumberOfThreadFieldTrial);
758 FieldTrialParameter<int> max_thread_number(
759 kVP8IosMaxNumberOfThreadFieldTrialParameter, 0);
760 ParseFieldTrial({&max_thread_number}, trial_string);
761 if (max_thread_number.Get() > 0) {
762 if (width * height < 320 * 180) {
763 return 1; // Use single thread for small screens
764 }
765 // thread number must be less than or equal to the number of CPUs.
766 return std::min(cpus, max_thread_number.Get());
767 }
768 #endif // defined(WEBRTC_IOS)
769 if (width * height >= 1920 * 1080 && cpus > 8) {
770 return 8; // 8 threads for 1080p on high perf machines.
771 } else if (width * height > 1280 * 960 && cpus >= 6) {
772 // 3 threads for 1080p.
773 return 3;
774 } else if (width * height > 640 * 480 && cpus >= 3) {
775 // Default 2 threads for qHD/HD, but allow 3 if core count is high enough,
776 // as this will allow more margin for high-core/low clock machines or if
777 // not built with highest optimization.
778 if (cpus >= 6) {
779 return 3;
780 }
781 return 2;
782 } else {
783 // 1 thread for VGA or less.
784 return 1;
785 }
786 #endif
787 }
788
InitAndSetControlSettings()789 int LibvpxVp8Encoder::InitAndSetControlSettings() {
790 vpx_codec_flags_t flags = 0;
791 flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
792
793 if (encoders_.size() > 1) {
794 int error = libvpx_->codec_enc_init_multi(
795 &encoders_[0], vpx_codec_vp8_cx(), &vpx_configs_[0], encoders_.size(),
796 flags, &downsampling_factors_[0]);
797 if (error) {
798 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
799 }
800 } else {
801 if (libvpx_->codec_enc_init(&encoders_[0], vpx_codec_vp8_cx(),
802 &vpx_configs_[0], flags)) {
803 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
804 }
805 }
806 // Enable denoising for the highest resolution stream, and for
807 // the second highest resolution if we are doing more than 2
808 // spatial layers/streams.
809 // TODO(holmer): Investigate possibility of adding a libvpx API
810 // for getting the denoised frame from the encoder and using that
811 // when encoding lower resolution streams. Would it work with the
812 // multi-res encoding feature?
813 denoiserState denoiser_state = kDenoiserOnYOnly;
814 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
815 defined(WEBRTC_ANDROID)
816 denoiser_state = kDenoiserOnYOnly;
817 #else
818 denoiser_state = kDenoiserOnAdaptive;
819 #endif
820 libvpx_->codec_control(
821 &encoders_[0], VP8E_SET_NOISE_SENSITIVITY,
822 codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
823 if (encoders_.size() > 2) {
824 libvpx_->codec_control(
825 &encoders_[1], VP8E_SET_NOISE_SENSITIVITY,
826 codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
827 }
828 for (size_t i = 0; i < encoders_.size(); ++i) {
829 // Allow more screen content to be detected as static.
830 libvpx_->codec_control(
831 &(encoders_[i]), VP8E_SET_STATIC_THRESHOLD,
832 codec_.mode == VideoCodecMode::kScreensharing ? 100u : 1u);
833 libvpx_->codec_control(&(encoders_[i]), VP8E_SET_CPUUSED, cpu_speed_[i]);
834 libvpx_->codec_control(
835 &(encoders_[i]), VP8E_SET_TOKEN_PARTITIONS,
836 static_cast<vp8e_token_partitions>(kTokenPartitions));
837 libvpx_->codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
838 rc_max_intra_target_);
839 // VP8E_SET_SCREEN_CONTENT_MODE 2 = screen content with more aggressive
840 // rate control (drop frames on large target bitrate overshoot)
841 libvpx_->codec_control(
842 &(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE,
843 codec_.mode == VideoCodecMode::kScreensharing ? 2u : 0u);
844 }
845 inited_ = true;
846 return WEBRTC_VIDEO_CODEC_OK;
847 }
848
MaxIntraTarget(uint32_t optimalBuffersize)849 uint32_t LibvpxVp8Encoder::MaxIntraTarget(uint32_t optimalBuffersize) {
850 // Set max to the optimal buffer level (normalized by target BR),
851 // and scaled by a scalePar.
852 // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
853 // This values is presented in percentage of perFrameBw:
854 // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
855 // The target in % is as follows:
856
857 float scalePar = 0.5;
858 uint32_t targetPct = optimalBuffersize * scalePar * codec_.maxFramerate / 10;
859
860 // Don't go below 3 times the per frame bandwidth.
861 const uint32_t minIntraTh = 300;
862 return (targetPct < minIntraTh) ? minIntraTh : targetPct;
863 }
864
FrameDropThreshold(size_t spatial_idx) const865 uint32_t LibvpxVp8Encoder::FrameDropThreshold(size_t spatial_idx) const {
866 bool enable_frame_dropping = codec_.VP8().frameDroppingOn;
867 // If temporal layers are used, they get to override the frame dropping
868 // setting, as eg. ScreenshareLayers does not work as intended with frame
869 // dropping on and DefaultTemporalLayers will have performance issues with
870 // frame dropping off.
871 RTC_DCHECK(frame_buffer_controller_);
872 RTC_DCHECK_LT(spatial_idx, frame_buffer_controller_->StreamCount());
873 enable_frame_dropping =
874 frame_buffer_controller_->SupportsEncoderFrameDropping(spatial_idx);
875 return enable_frame_dropping ? 30 : 0;
876 }
877
SteadyStateSize(int sid,int tid)878 size_t LibvpxVp8Encoder::SteadyStateSize(int sid, int tid) {
879 const int encoder_id = encoders_.size() - 1 - sid;
880 size_t bitrate_bps;
881 float fps;
882 if (SimulcastUtility::IsConferenceModeScreenshare(codec_) ||
883 vpx_configs_[encoder_id].ts_number_layers <= 1) {
884 // In conference screenshare there's no defined per temporal layer bitrate
885 // and framerate.
886 bitrate_bps = vpx_configs_[encoder_id].rc_target_bitrate * 1000;
887 fps = codec_.maxFramerate;
888 } else {
889 bitrate_bps = vpx_configs_[encoder_id].ts_target_bitrate[tid] * 1000;
890 fps = codec_.maxFramerate /
891 fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid], 1.0);
892 if (tid > 0) {
893 // Layer bitrate and fps are counted as a partial sums.
894 bitrate_bps -= vpx_configs_[encoder_id].ts_target_bitrate[tid - 1] * 1000;
895 fps = codec_.maxFramerate /
896 fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid - 1], 1.0);
897 }
898 }
899
900 if (fps < 1e-9)
901 return 0;
902 return static_cast<size_t>(
903 bitrate_bps / (8 * fps) *
904 (100 -
905 variable_framerate_experiment_.steady_state_undershoot_percentage) /
906 100 +
907 0.5);
908 }
909
UpdateVpxConfiguration(size_t stream_index)910 bool LibvpxVp8Encoder::UpdateVpxConfiguration(size_t stream_index) {
911 RTC_DCHECK(frame_buffer_controller_);
912
913 const size_t config_index = vpx_configs_.size() - 1 - stream_index;
914
915 RTC_DCHECK_LT(config_index, config_overrides_.size());
916 Vp8EncoderConfig* config = &config_overrides_[config_index];
917
918 const Vp8EncoderConfig new_config =
919 frame_buffer_controller_->UpdateConfiguration(stream_index);
920
921 if (new_config.reset_previous_configuration_overrides) {
922 *config = new_config;
923 return true;
924 }
925
926 const bool changes_made = MaybeExtendVp8EncoderConfig(new_config, config);
927
928 // Note that overrides must be applied even if they haven't changed.
929 RTC_DCHECK_LT(config_index, vpx_configs_.size());
930 vpx_codec_enc_cfg_t* vpx_config = &vpx_configs_[config_index];
931 ApplyVp8EncoderConfigToVpxConfig(*config, vpx_config);
932
933 return changes_made;
934 }
935
Encode(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)936 int LibvpxVp8Encoder::Encode(const VideoFrame& frame,
937 const std::vector<VideoFrameType>* frame_types) {
938 RTC_DCHECK_EQ(frame.width(), codec_.width);
939 RTC_DCHECK_EQ(frame.height(), codec_.height);
940
941 if (!inited_)
942 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
943 if (encoded_complete_callback_ == NULL)
944 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
945
946 bool key_frame_requested = false;
947 for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size();
948 ++i) {
949 if (key_frame_request_[i] && send_stream_[i]) {
950 key_frame_requested = true;
951 break;
952 }
953 }
954 if (!key_frame_requested && frame_types) {
955 for (size_t i = 0; i < frame_types->size() && i < send_stream_.size();
956 ++i) {
957 if ((*frame_types)[i] == VideoFrameType::kVideoFrameKey &&
958 send_stream_[i]) {
959 key_frame_requested = true;
960 break;
961 }
962 }
963 }
964
965 if (frame.update_rect().IsEmpty() && num_steady_state_frames_ >= 3 &&
966 !key_frame_requested) {
967 if (variable_framerate_experiment_.enabled &&
968 framerate_controller_.DropFrame(frame.timestamp() / kRtpTicksPerMs)) {
969 return WEBRTC_VIDEO_CODEC_OK;
970 }
971 framerate_controller_.AddFrame(frame.timestamp() / kRtpTicksPerMs);
972 }
973
974 bool send_key_frame = key_frame_requested;
975 bool drop_frame = false;
976 bool retransmission_allowed = true;
977 Vp8FrameConfig tl_configs[kMaxSimulcastStreams];
978 for (size_t i = 0; i < encoders_.size(); ++i) {
979 tl_configs[i] =
980 frame_buffer_controller_->NextFrameConfig(i, frame.timestamp());
981 send_key_frame |= tl_configs[i].IntraFrame();
982 drop_frame |= tl_configs[i].drop_frame;
983 RTC_DCHECK(i == 0 ||
984 retransmission_allowed == tl_configs[i].retransmission_allowed);
985 retransmission_allowed = tl_configs[i].retransmission_allowed;
986 }
987
988 if (drop_frame && !send_key_frame) {
989 return WEBRTC_VIDEO_CODEC_OK;
990 }
991
992 vpx_enc_frame_flags_t flags[kMaxSimulcastStreams];
993 for (size_t i = 0; i < encoders_.size(); ++i) {
994 flags[i] = send_key_frame ? VPX_EFLAG_FORCE_KF : EncodeFlags(tl_configs[i]);
995 }
996
997 rtc::scoped_refptr<I420BufferInterface> input_image =
998 frame.video_frame_buffer()->ToI420();
999 // Since we are extracting raw pointers from |input_image| to
1000 // |raw_images_[0]|, the resolution of these frames must match.
1001 RTC_DCHECK_EQ(input_image->width(), raw_images_[0].d_w);
1002 RTC_DCHECK_EQ(input_image->height(), raw_images_[0].d_h);
1003
1004 // Image in vpx_image_t format.
1005 // Input image is const. VP8's raw image is not defined as const.
1006 raw_images_[0].planes[VPX_PLANE_Y] =
1007 const_cast<uint8_t*>(input_image->DataY());
1008 raw_images_[0].planes[VPX_PLANE_U] =
1009 const_cast<uint8_t*>(input_image->DataU());
1010 raw_images_[0].planes[VPX_PLANE_V] =
1011 const_cast<uint8_t*>(input_image->DataV());
1012
1013 raw_images_[0].stride[VPX_PLANE_Y] = input_image->StrideY();
1014 raw_images_[0].stride[VPX_PLANE_U] = input_image->StrideU();
1015 raw_images_[0].stride[VPX_PLANE_V] = input_image->StrideV();
1016
1017 struct CleanUpOnExit {
1018 explicit CleanUpOnExit(vpx_image_t& raw_image) : raw_image_(raw_image) {}
1019 ~CleanUpOnExit() {
1020 raw_image_.planes[VPX_PLANE_Y] = nullptr;
1021 raw_image_.planes[VPX_PLANE_U] = nullptr;
1022 raw_image_.planes[VPX_PLANE_V] = nullptr;
1023 }
1024 vpx_image_t& raw_image_;
1025 } clean_up_on_exit(raw_images_[0]);
1026
1027 for (size_t i = 1; i < encoders_.size(); ++i) {
1028 // Scale the image down a number of times by downsampling factor
1029 libyuv::I420Scale(
1030 raw_images_[i - 1].planes[VPX_PLANE_Y],
1031 raw_images_[i - 1].stride[VPX_PLANE_Y],
1032 raw_images_[i - 1].planes[VPX_PLANE_U],
1033 raw_images_[i - 1].stride[VPX_PLANE_U],
1034 raw_images_[i - 1].planes[VPX_PLANE_V],
1035 raw_images_[i - 1].stride[VPX_PLANE_V], raw_images_[i - 1].d_w,
1036 raw_images_[i - 1].d_h, raw_images_[i].planes[VPX_PLANE_Y],
1037 raw_images_[i].stride[VPX_PLANE_Y], raw_images_[i].planes[VPX_PLANE_U],
1038 raw_images_[i].stride[VPX_PLANE_U], raw_images_[i].planes[VPX_PLANE_V],
1039 raw_images_[i].stride[VPX_PLANE_V], raw_images_[i].d_w,
1040 raw_images_[i].d_h, libyuv::kFilterBilinear);
1041 }
1042
1043 if (send_key_frame) {
1044 // Adapt the size of the key frame when in screenshare with 1 temporal
1045 // layer.
1046 if (encoders_.size() == 1 &&
1047 codec_.mode == VideoCodecMode::kScreensharing &&
1048 codec_.VP8()->numberOfTemporalLayers <= 1) {
1049 const uint32_t forceKeyFrameIntraTh = 100;
1050 libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
1051 forceKeyFrameIntraTh);
1052 }
1053
1054 std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
1055 }
1056
1057 // Set the encoder frame flags and temporal layer_id for each spatial stream.
1058 // Note that streams are defined starting from lowest resolution at
1059 // position 0 to highest resolution at position |encoders_.size() - 1|,
1060 // whereas |encoder_| is from highest to lowest resolution.
1061 for (size_t i = 0; i < encoders_.size(); ++i) {
1062 const size_t stream_idx = encoders_.size() - 1 - i;
1063
1064 if (UpdateVpxConfiguration(stream_idx)) {
1065 if (libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i]))
1066 return WEBRTC_VIDEO_CODEC_ERROR;
1067 }
1068
1069 libvpx_->codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS,
1070 static_cast<int>(flags[stream_idx]));
1071 libvpx_->codec_control(&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID,
1072 tl_configs[i].encoder_layer_id);
1073 }
1074 // TODO(holmer): Ideally the duration should be the timestamp diff of this
1075 // frame and the next frame to be encoded, which we don't have. Instead we
1076 // would like to use the duration of the previous frame. Unfortunately the
1077 // rate control seems to be off with that setup. Using the average input
1078 // frame rate to calculate an average duration for now.
1079 assert(codec_.maxFramerate > 0);
1080 uint32_t duration = kRtpTicksPerSecond / codec_.maxFramerate;
1081
1082 int error = WEBRTC_VIDEO_CODEC_OK;
1083 int num_tries = 0;
1084 // If the first try returns WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT
1085 // the frame must be reencoded with the same parameters again because
1086 // target bitrate is exceeded and encoder state has been reset.
1087 while (num_tries == 0 ||
1088 (num_tries == 1 &&
1089 error == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT)) {
1090 ++num_tries;
1091 // Note we must pass 0 for |flags| field in encode call below since they are
1092 // set above in |libvpx_interface_->vpx_codec_control_| function for each
1093 // encoder/spatial layer.
1094 error = libvpx_->codec_encode(&encoders_[0], &raw_images_[0], timestamp_,
1095 duration, 0, VPX_DL_REALTIME);
1096 // Reset specific intra frame thresholds, following the key frame.
1097 if (send_key_frame) {
1098 libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
1099 rc_max_intra_target_);
1100 }
1101 if (error)
1102 return WEBRTC_VIDEO_CODEC_ERROR;
1103 // Examines frame timestamps only.
1104 error = GetEncodedPartitions(frame, retransmission_allowed);
1105 }
1106 // TODO(sprang): Shouldn't we use the frame timestamp instead?
1107 timestamp_ += duration;
1108 return error;
1109 }
1110
PopulateCodecSpecific(CodecSpecificInfo * codec_specific,const vpx_codec_cx_pkt_t & pkt,int stream_idx,int encoder_idx,uint32_t timestamp)1111 void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
1112 const vpx_codec_cx_pkt_t& pkt,
1113 int stream_idx,
1114 int encoder_idx,
1115 uint32_t timestamp) {
1116 assert(codec_specific != NULL);
1117 codec_specific->codecType = kVideoCodecVP8;
1118 codec_specific->codecSpecific.VP8.keyIdx =
1119 kNoKeyIdx; // TODO(hlundin) populate this
1120 codec_specific->codecSpecific.VP8.nonReference =
1121 (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
1122
1123 int qp = 0;
1124 vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
1125 frame_buffer_controller_->OnEncodeDone(
1126 stream_idx, timestamp, encoded_images_[encoder_idx].size(),
1127 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, codec_specific);
1128 }
1129
GetEncodedPartitions(const VideoFrame & input_image,bool retransmission_allowed)1130 int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image,
1131 bool retransmission_allowed) {
1132 int stream_idx = static_cast<int>(encoders_.size()) - 1;
1133 int result = WEBRTC_VIDEO_CODEC_OK;
1134 for (size_t encoder_idx = 0; encoder_idx < encoders_.size();
1135 ++encoder_idx, --stream_idx) {
1136 vpx_codec_iter_t iter = NULL;
1137 encoded_images_[encoder_idx].set_size(0);
1138 encoded_images_[encoder_idx]._frameType = VideoFrameType::kVideoFrameDelta;
1139 CodecSpecificInfo codec_specific;
1140 const vpx_codec_cx_pkt_t* pkt = NULL;
1141
1142 size_t encoded_size = 0;
1143 while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) !=
1144 NULL) {
1145 if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) {
1146 encoded_size += pkt->data.frame.sz;
1147 }
1148 }
1149
1150 // TODO(nisse): Introduce some buffer cache or buffer pool, to reduce
1151 // allocations and/or copy operations.
1152 auto buffer = EncodedImageBuffer::Create(encoded_size);
1153
1154 iter = NULL;
1155 size_t encoded_pos = 0;
1156 while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) !=
1157 NULL) {
1158 switch (pkt->kind) {
1159 case VPX_CODEC_CX_FRAME_PKT: {
1160 RTC_CHECK_LE(encoded_pos + pkt->data.frame.sz, buffer->size());
1161 memcpy(&buffer->data()[encoded_pos], pkt->data.frame.buf,
1162 pkt->data.frame.sz);
1163 encoded_pos += pkt->data.frame.sz;
1164 break;
1165 }
1166 default:
1167 break;
1168 }
1169 // End of frame
1170 if ((pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT) == 0) {
1171 // check if encoded frame is a key frame
1172 if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
1173 encoded_images_[encoder_idx]._frameType =
1174 VideoFrameType::kVideoFrameKey;
1175 }
1176 encoded_images_[encoder_idx].SetEncodedData(buffer);
1177 encoded_images_[encoder_idx].set_size(encoded_pos);
1178 encoded_images_[encoder_idx].SetSpatialIndex(stream_idx);
1179 PopulateCodecSpecific(&codec_specific, *pkt, stream_idx, encoder_idx,
1180 input_image.timestamp());
1181 break;
1182 }
1183 }
1184 encoded_images_[encoder_idx].SetTimestamp(input_image.timestamp());
1185 encoded_images_[encoder_idx].SetRetransmissionAllowed(
1186 retransmission_allowed);
1187
1188 if (send_stream_[stream_idx]) {
1189 if (encoded_images_[encoder_idx].size() > 0) {
1190 TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,
1191 encoded_images_[encoder_idx].size());
1192 encoded_images_[encoder_idx]._encodedHeight =
1193 codec_.simulcastStream[stream_idx].height;
1194 encoded_images_[encoder_idx]._encodedWidth =
1195 codec_.simulcastStream[stream_idx].width;
1196 int qp_128 = -1;
1197 libvpx_->codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER,
1198 &qp_128);
1199 encoded_images_[encoder_idx].qp_ = qp_128;
1200 encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx],
1201 &codec_specific, nullptr);
1202 const size_t steady_state_size = SteadyStateSize(
1203 stream_idx, codec_specific.codecSpecific.VP8.temporalIdx);
1204 if (qp_128 > variable_framerate_experiment_.steady_state_qp ||
1205 encoded_images_[encoder_idx].size() > steady_state_size) {
1206 num_steady_state_frames_ = 0;
1207 } else {
1208 ++num_steady_state_frames_;
1209 }
1210 } else if (!frame_buffer_controller_->SupportsEncoderFrameDropping(
1211 stream_idx)) {
1212 result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
1213 if (encoded_images_[encoder_idx].size() == 0) {
1214 // Dropped frame that will be re-encoded.
1215 frame_buffer_controller_->OnFrameDropped(stream_idx,
1216 input_image.timestamp());
1217 }
1218 }
1219 }
1220 }
1221 return result;
1222 }
1223
GetEncoderInfo() const1224 VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const {
1225 EncoderInfo info;
1226 info.supports_native_handle = false;
1227 info.implementation_name = "libvpx";
1228 info.has_trusted_rate_controller =
1229 rate_control_settings_.LibvpxVp8TrustedRateController();
1230 info.is_hardware_accelerated = false;
1231 info.has_internal_source = false;
1232 info.supports_simulcast = true;
1233 if (!resolution_bitrate_limits_.empty()) {
1234 info.resolution_bitrate_limits = resolution_bitrate_limits_;
1235 }
1236
1237 const bool enable_scaling =
1238 num_active_streams_ == 1 &&
1239 (vpx_configs_.empty() || vpx_configs_[0].rc_dropframe_thresh > 0) &&
1240 codec_.VP8().automaticResizeOn;
1241
1242 info.scaling_settings = enable_scaling
1243 ? VideoEncoder::ScalingSettings(
1244 kLowVp8QpThreshold, kHighVp8QpThreshold)
1245 : VideoEncoder::ScalingSettings::kOff;
1246 if (rate_control_settings_.LibvpxVp8MinPixels()) {
1247 info.scaling_settings.min_pixels_per_frame =
1248 rate_control_settings_.LibvpxVp8MinPixels().value();
1249 }
1250
1251 if (inited_) {
1252 // |encoder_idx| is libvpx index where 0 is highest resolution.
1253 // |si| is simulcast index, where 0 is lowest resolution.
1254 for (size_t si = 0, encoder_idx = encoders_.size() - 1;
1255 si < encoders_.size(); ++si, --encoder_idx) {
1256 info.fps_allocation[si].clear();
1257 if ((codec_.numberOfSimulcastStreams > si &&
1258 !codec_.simulcastStream[si].active) ||
1259 (si == 0 && SimulcastUtility::IsConferenceModeScreenshare(codec_))) {
1260 // No defined frame rate fractions if not active or if using
1261 // ScreenshareLayers, leave vector empty and continue;
1262 continue;
1263 }
1264 if (vpx_configs_[encoder_idx].ts_number_layers <= 1) {
1265 info.fps_allocation[si].push_back(EncoderInfo::kMaxFramerateFraction);
1266 } else {
1267 for (size_t ti = 0; ti < vpx_configs_[encoder_idx].ts_number_layers;
1268 ++ti) {
1269 RTC_DCHECK_GT(vpx_configs_[encoder_idx].ts_rate_decimator[ti], 0);
1270 info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>(
1271 EncoderInfo::kMaxFramerateFraction /
1272 vpx_configs_[encoder_idx].ts_rate_decimator[ti] +
1273 0.5));
1274 }
1275 }
1276 }
1277 }
1278
1279 return info;
1280 }
1281
RegisterEncodeCompleteCallback(EncodedImageCallback * callback)1282 int LibvpxVp8Encoder::RegisterEncodeCompleteCallback(
1283 EncodedImageCallback* callback) {
1284 encoded_complete_callback_ = callback;
1285 return WEBRTC_VIDEO_CODEC_OK;
1286 }
1287
1288 // static
1289 LibvpxVp8Encoder::VariableFramerateExperiment
ParseVariableFramerateConfig(std::string group_name)1290 LibvpxVp8Encoder::ParseVariableFramerateConfig(std::string group_name) {
1291 FieldTrialFlag enabled = FieldTrialFlag("Enabled");
1292 FieldTrialParameter<double> framerate_limit("min_fps", 5.0);
1293 FieldTrialParameter<int> qp("min_qp", 15);
1294 FieldTrialParameter<int> undershoot_percentage("undershoot", 30);
1295 ParseFieldTrial({&enabled, &framerate_limit, &qp, &undershoot_percentage},
1296 field_trial::FindFullName(group_name));
1297 VariableFramerateExperiment config;
1298 config.enabled = enabled.Get();
1299 config.framerate_limit = framerate_limit.Get();
1300 config.steady_state_qp = qp.Get();
1301 config.steady_state_undershoot_percentage = undershoot_percentage.Get();
1302
1303 return config;
1304 }
1305
1306 } // namespace webrtc
1307