1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h"
12
13 #include <stdio.h>
14 #include <string.h>
15
16 #include <algorithm>
17 #include <memory>
18 #include <string>
19
20 #include "absl/types/optional.h"
21 #include "api/scoped_refptr.h"
22 #include "api/video/i420_buffer.h"
23 #include "api/video/video_frame.h"
24 #include "api/video/video_frame_buffer.h"
25 #include "api/video/video_rotation.h"
26 #include "modules/video_coding/codecs/vp8/include/vp8.h"
27 #include "modules/video_coding/include/video_error_codes.h"
28 #include "rtc_base/checks.h"
29 #include "rtc_base/numerics/exp_filter.h"
30 #include "rtc_base/time_utils.h"
31 #include "system_wrappers/include/field_trial.h"
32 #include "system_wrappers/include/metrics.h"
33 #include "third_party/libyuv/include/libyuv/convert.h"
34 #include "vpx/vp8.h"
35 #include "vpx/vp8dx.h"
36 #include "vpx/vpx_decoder.h"
37
38 namespace webrtc {
39 namespace {
40 constexpr int kVp8ErrorPropagationTh = 30;
41 // vpx_decoder.h documentation indicates decode deadline is time in us, with
42 // "Set to zero for unlimited.", but actual implementation requires this to be
43 // a mode with 0 meaning allow delay and 1 not allowing it.
44 constexpr long kDecodeDeadlineRealtime = 1; // NOLINT
45
46 const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm";
47 const char kVp8PostProcFieldTrial[] = "WebRTC-VP8-Postproc-Config";
48
49 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
50 defined(WEBRTC_ANDROID)
51 constexpr bool kIsArm = true;
52 #else
53 constexpr bool kIsArm = false;
54 #endif
55
DefaultDeblockParams()56 absl::optional<LibvpxVp8Decoder::DeblockParams> DefaultDeblockParams() {
57 if (kIsArm) {
58 // For ARM, this is only called when deblocking is explicitly enabled, and
59 // the default strength is set by the ctor.
60 return LibvpxVp8Decoder::DeblockParams();
61 }
62 // For non-arm, don't use the explicit deblocking settings by default.
63 return absl::nullopt;
64 }
65
66 absl::optional<LibvpxVp8Decoder::DeblockParams>
GetPostProcParamsFromFieldTrialGroup()67 GetPostProcParamsFromFieldTrialGroup() {
68 std::string group = webrtc::field_trial::FindFullName(
69 kIsArm ? kVp8PostProcArmFieldTrial : kVp8PostProcFieldTrial);
70 if (group.empty()) {
71 return DefaultDeblockParams();
72 }
73
74 LibvpxVp8Decoder::DeblockParams params;
75 if (sscanf(group.c_str(), "Enabled-%d,%d,%d", ¶ms.max_level,
76 ¶ms.min_qp, ¶ms.degrade_qp) != 3) {
77 return DefaultDeblockParams();
78 }
79
80 if (params.max_level < 0 || params.max_level > 16) {
81 return DefaultDeblockParams();
82 }
83
84 if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) {
85 return DefaultDeblockParams();
86 }
87
88 return params;
89 }
90
91 } // namespace
92
Create()93 std::unique_ptr<VideoDecoder> VP8Decoder::Create() {
94 return std::make_unique<LibvpxVp8Decoder>();
95 }
96
97 class LibvpxVp8Decoder::QpSmoother {
98 public:
QpSmoother()99 QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {}
100
GetAvg() const101 int GetAvg() const {
102 float value = smoother_.filtered();
103 return (value == rtc::ExpFilter::kValueUndefined) ? 0
104 : static_cast<int>(value);
105 }
106
Add(float sample)107 void Add(float sample) {
108 int64_t now_ms = rtc::TimeMillis();
109 smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
110 last_sample_ms_ = now_ms;
111 }
112
Reset()113 void Reset() { smoother_.Reset(kAlpha); }
114
115 private:
116 const float kAlpha = 0.95f;
117 int64_t last_sample_ms_;
118 rtc::ExpFilter smoother_;
119 };
120
LibvpxVp8Decoder()121 LibvpxVp8Decoder::LibvpxVp8Decoder()
122 : use_postproc_(
123 kIsArm ? webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)
124 : true),
125 buffer_pool_(false, 300 /* max_number_of_buffers*/),
126 decode_complete_callback_(NULL),
127 inited_(false),
128 decoder_(NULL),
129 propagation_cnt_(-1),
130 last_frame_width_(0),
131 last_frame_height_(0),
132 key_frame_required_(true),
133 deblock_params_(use_postproc_ ? GetPostProcParamsFromFieldTrialGroup()
134 : absl::nullopt),
135 qp_smoother_(use_postproc_ ? new QpSmoother() : nullptr) {}
136
~LibvpxVp8Decoder()137 LibvpxVp8Decoder::~LibvpxVp8Decoder() {
138 inited_ = true; // in order to do the actual release
139 Release();
140 }
141
InitDecode(const VideoCodec * inst,int number_of_cores)142 int LibvpxVp8Decoder::InitDecode(const VideoCodec* inst, int number_of_cores) {
143 int ret_val = Release();
144 if (ret_val < 0) {
145 return ret_val;
146 }
147 if (decoder_ == NULL) {
148 decoder_ = new vpx_codec_ctx_t;
149 memset(decoder_, 0, sizeof(*decoder_));
150 }
151 vpx_codec_dec_cfg_t cfg;
152 // Setting number of threads to a constant value (1)
153 cfg.threads = 1;
154 cfg.h = cfg.w = 0; // set after decode
155
156 vpx_codec_flags_t flags = use_postproc_ ? VPX_CODEC_USE_POSTPROC : 0;
157
158 if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) {
159 delete decoder_;
160 decoder_ = nullptr;
161 return WEBRTC_VIDEO_CODEC_MEMORY;
162 }
163
164 propagation_cnt_ = -1;
165 inited_ = true;
166
167 // Always start with a complete key frame.
168 key_frame_required_ = true;
169 if (inst && inst->buffer_pool_size) {
170 if (!buffer_pool_.Resize(*inst->buffer_pool_size)) {
171 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
172 }
173 }
174 return WEBRTC_VIDEO_CODEC_OK;
175 }
176
Decode(const EncodedImage & input_image,bool missing_frames,int64_t)177 int LibvpxVp8Decoder::Decode(const EncodedImage& input_image,
178 bool missing_frames,
179 int64_t /*render_time_ms*/) {
180 if (!inited_) {
181 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
182 }
183 if (decode_complete_callback_ == NULL) {
184 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
185 }
186 if (input_image.data() == NULL && input_image.size() > 0) {
187 // Reset to avoid requesting key frames too often.
188 if (propagation_cnt_ > 0)
189 propagation_cnt_ = 0;
190 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
191 }
192
193 // Post process configurations.
194 if (use_postproc_) {
195 vp8_postproc_cfg_t ppcfg;
196 // MFQE enabled to reduce key frame popping.
197 ppcfg.post_proc_flag = VP8_MFQE;
198
199 if (kIsArm) {
200 RTC_DCHECK(deblock_params_.has_value());
201 }
202 if (deblock_params_.has_value()) {
203 // For low resolutions, use stronger deblocking filter.
204 int last_width_x_height = last_frame_width_ * last_frame_height_;
205 if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) {
206 // Enable the deblock and demacroblocker based on qp thresholds.
207 RTC_DCHECK(qp_smoother_);
208 int qp = qp_smoother_->GetAvg();
209 if (qp > deblock_params_->min_qp) {
210 int level = deblock_params_->max_level;
211 if (qp < deblock_params_->degrade_qp) {
212 // Use lower level.
213 level = deblock_params_->max_level *
214 (qp - deblock_params_->min_qp) /
215 (deblock_params_->degrade_qp - deblock_params_->min_qp);
216 }
217 // Deblocking level only affects VP8_DEMACROBLOCK.
218 ppcfg.deblocking_level = std::max(level, 1);
219 ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK;
220 }
221 }
222 } else {
223 // Non-arm with no explicit deblock params set.
224 ppcfg.post_proc_flag |= VP8_DEBLOCK;
225 // For VGA resolutions and lower, enable the demacroblocker postproc.
226 if (last_frame_width_ * last_frame_height_ <= 640 * 360) {
227 ppcfg.post_proc_flag |= VP8_DEMACROBLOCK;
228 }
229 // Strength of deblocking filter. Valid range:[0,16]
230 ppcfg.deblocking_level = 3;
231 }
232
233 vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
234 }
235
236 // Always start with a complete key frame.
237 if (key_frame_required_) {
238 if (input_image._frameType != VideoFrameType::kVideoFrameKey)
239 return WEBRTC_VIDEO_CODEC_ERROR;
240 // We have a key frame - is it complete?
241 if (input_image._completeFrame) {
242 key_frame_required_ = false;
243 } else {
244 return WEBRTC_VIDEO_CODEC_ERROR;
245 }
246 }
247 // Restrict error propagation using key frame requests.
248 // Reset on a key frame refresh.
249 if (input_image._frameType == VideoFrameType::kVideoFrameKey &&
250 input_image._completeFrame) {
251 propagation_cnt_ = -1;
252 // Start count on first loss.
253 } else if ((!input_image._completeFrame || missing_frames) &&
254 propagation_cnt_ == -1) {
255 propagation_cnt_ = 0;
256 }
257 if (propagation_cnt_ >= 0) {
258 propagation_cnt_++;
259 }
260
261 vpx_codec_iter_t iter = NULL;
262 vpx_image_t* img;
263 int ret;
264
265 // Check for missing frames.
266 if (missing_frames) {
267 // Call decoder with zero data length to signal missing frames.
268 if (vpx_codec_decode(decoder_, NULL, 0, 0, kDecodeDeadlineRealtime)) {
269 // Reset to avoid requesting key frames too often.
270 if (propagation_cnt_ > 0)
271 propagation_cnt_ = 0;
272 return WEBRTC_VIDEO_CODEC_ERROR;
273 }
274 img = vpx_codec_get_frame(decoder_, &iter);
275 iter = NULL;
276 }
277
278 const uint8_t* buffer = input_image.data();
279 if (input_image.size() == 0) {
280 buffer = NULL; // Triggers full frame concealment.
281 }
282 if (vpx_codec_decode(decoder_, buffer, input_image.size(), 0,
283 kDecodeDeadlineRealtime)) {
284 // Reset to avoid requesting key frames too often.
285 if (propagation_cnt_ > 0) {
286 propagation_cnt_ = 0;
287 }
288 return WEBRTC_VIDEO_CODEC_ERROR;
289 }
290
291 img = vpx_codec_get_frame(decoder_, &iter);
292 int qp;
293 vpx_codec_err_t vpx_ret =
294 vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
295 RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
296 ret = ReturnFrame(img, input_image.Timestamp(), qp, input_image.ColorSpace());
297 if (ret != 0) {
298 // Reset to avoid requesting key frames too often.
299 if (ret < 0 && propagation_cnt_ > 0)
300 propagation_cnt_ = 0;
301 return ret;
302 }
303 // Check Vs. threshold
304 if (propagation_cnt_ > kVp8ErrorPropagationTh) {
305 // Reset to avoid requesting key frames too often.
306 propagation_cnt_ = 0;
307 return WEBRTC_VIDEO_CODEC_ERROR;
308 }
309 return WEBRTC_VIDEO_CODEC_OK;
310 }
311
ReturnFrame(const vpx_image_t * img,uint32_t timestamp,int qp,const webrtc::ColorSpace * explicit_color_space)312 int LibvpxVp8Decoder::ReturnFrame(
313 const vpx_image_t* img,
314 uint32_t timestamp,
315 int qp,
316 const webrtc::ColorSpace* explicit_color_space) {
317 if (img == NULL) {
318 // Decoder OK and NULL image => No show frame
319 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
320 }
321 if (qp_smoother_) {
322 if (last_frame_width_ != static_cast<int>(img->d_w) ||
323 last_frame_height_ != static_cast<int>(img->d_h)) {
324 qp_smoother_->Reset();
325 }
326 qp_smoother_->Add(qp);
327 }
328 last_frame_width_ = img->d_w;
329 last_frame_height_ = img->d_h;
330 // Allocate memory for decoded image.
331 rtc::scoped_refptr<I420Buffer> buffer =
332 buffer_pool_.CreateBuffer(img->d_w, img->d_h);
333 if (!buffer.get()) {
334 // Pool has too many pending frames.
335 RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames",
336 1);
337 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
338 }
339
340 libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y],
341 img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U],
342 img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V],
343 buffer->MutableDataY(), buffer->StrideY(),
344 buffer->MutableDataU(), buffer->StrideU(),
345 buffer->MutableDataV(), buffer->StrideV(), img->d_w,
346 img->d_h);
347
348 VideoFrame decoded_image = VideoFrame::Builder()
349 .set_video_frame_buffer(buffer)
350 .set_timestamp_rtp(timestamp)
351 .set_color_space(explicit_color_space)
352 .build();
353 decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
354
355 return WEBRTC_VIDEO_CODEC_OK;
356 }
357
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)358 int LibvpxVp8Decoder::RegisterDecodeCompleteCallback(
359 DecodedImageCallback* callback) {
360 decode_complete_callback_ = callback;
361 return WEBRTC_VIDEO_CODEC_OK;
362 }
363
Release()364 int LibvpxVp8Decoder::Release() {
365 int ret_val = WEBRTC_VIDEO_CODEC_OK;
366
367 if (decoder_ != NULL) {
368 if (inited_) {
369 if (vpx_codec_destroy(decoder_)) {
370 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
371 }
372 }
373 delete decoder_;
374 decoder_ = NULL;
375 }
376 buffer_pool_.Release();
377 inited_ = false;
378 return ret_val;
379 }
380
ImplementationName() const381 const char* LibvpxVp8Decoder::ImplementationName() const {
382 return "libvpx";
383 }
384 } // namespace webrtc
385