1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/test/videoprocessor.h"
12
13 #include <string.h>
14
15 #include <algorithm>
16 #include <cstddef>
17 #include <limits>
18 #include <memory>
19 #include <utility>
20
21 #include "api/scoped_refptr.h"
22 #include "api/video/builtin_video_bitrate_allocator_factory.h"
23 #include "api/video/i420_buffer.h"
24 #include "api/video/video_bitrate_allocator_factory.h"
25 #include "api/video/video_frame_buffer.h"
26 #include "api/video/video_rotation.h"
27 #include "api/video_codecs/video_codec.h"
28 #include "api/video_codecs/video_encoder.h"
29 #include "common_video/h264/h264_common.h"
30 #include "common_video/libyuv/include/webrtc_libyuv.h"
31 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
32 #include "modules/video_coding/codecs/interface/common_constants.h"
33 #include "modules/video_coding/include/video_error_codes.h"
34 #include "rtc_base/checks.h"
35 #include "rtc_base/task_utils/to_queued_task.h"
36 #include "rtc_base/time_utils.h"
37 #include "test/gtest.h"
38 #include "third_party/libyuv/include/libyuv/compare.h"
39 #include "third_party/libyuv/include/libyuv/scale.h"
40
41 namespace webrtc {
42 namespace test {
43
44 using FrameStatistics = VideoCodecTestStats::FrameStatistics;
45
46 namespace {
47 const int kMsToRtpTimestamp = kVideoPayloadTypeFrequency / 1000;
48 const int kMaxBufferedInputFrames = 20;
49
50 const VideoEncoder::Capabilities kCapabilities(false);
51
GetMaxNaluSizeBytes(const EncodedImage & encoded_frame,const VideoCodecTestFixture::Config & config)52 size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame,
53 const VideoCodecTestFixture::Config& config) {
54 if (config.codec_settings.codecType != kVideoCodecH264)
55 return 0;
56
57 std::vector<webrtc::H264::NaluIndex> nalu_indices =
58 webrtc::H264::FindNaluIndices(encoded_frame.data(), encoded_frame.size());
59
60 RTC_CHECK(!nalu_indices.empty());
61
62 size_t max_size = 0;
63 for (const webrtc::H264::NaluIndex& index : nalu_indices)
64 max_size = std::max(max_size, index.payload_size);
65
66 return max_size;
67 }
68
GetTemporalLayerIndex(const CodecSpecificInfo & codec_specific)69 size_t GetTemporalLayerIndex(const CodecSpecificInfo& codec_specific) {
70 size_t temporal_idx = 0;
71 if (codec_specific.codecType == kVideoCodecVP8) {
72 temporal_idx = codec_specific.codecSpecific.VP8.temporalIdx;
73 } else if (codec_specific.codecType == kVideoCodecVP9) {
74 temporal_idx = codec_specific.codecSpecific.VP9.temporal_idx;
75 }
76 if (temporal_idx == kNoTemporalIdx) {
77 temporal_idx = 0;
78 }
79 return temporal_idx;
80 }
81
GetElapsedTimeMicroseconds(int64_t start_ns,int64_t stop_ns)82 int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) {
83 int64_t diff_us = (stop_ns - start_ns) / rtc::kNumNanosecsPerMicrosec;
84 RTC_DCHECK_GE(diff_us, std::numeric_limits<int>::min());
85 RTC_DCHECK_LE(diff_us, std::numeric_limits<int>::max());
86 return static_cast<int>(diff_us);
87 }
88
ExtractI420BufferWithSize(const VideoFrame & image,int width,int height,rtc::Buffer * buffer)89 void ExtractI420BufferWithSize(const VideoFrame& image,
90 int width,
91 int height,
92 rtc::Buffer* buffer) {
93 if (image.width() != width || image.height() != height) {
94 EXPECT_DOUBLE_EQ(static_cast<double>(width) / height,
95 static_cast<double>(image.width()) / image.height());
96 // Same aspect ratio, no cropping needed.
97 rtc::scoped_refptr<I420Buffer> scaled(I420Buffer::Create(width, height));
98 scaled->ScaleFrom(*image.video_frame_buffer()->ToI420());
99
100 size_t length =
101 CalcBufferSize(VideoType::kI420, scaled->width(), scaled->height());
102 buffer->SetSize(length);
103 RTC_CHECK_NE(ExtractBuffer(scaled, length, buffer->data()), -1);
104 return;
105 }
106
107 // No resize.
108 size_t length =
109 CalcBufferSize(VideoType::kI420, image.width(), image.height());
110 buffer->SetSize(length);
111 RTC_CHECK_NE(ExtractBuffer(image, length, buffer->data()), -1);
112 }
113
CalculateFrameQuality(const I420BufferInterface & ref_buffer,const I420BufferInterface & dec_buffer,FrameStatistics * frame_stat,bool calc_ssim)114 void CalculateFrameQuality(const I420BufferInterface& ref_buffer,
115 const I420BufferInterface& dec_buffer,
116 FrameStatistics* frame_stat,
117 bool calc_ssim) {
118 if (ref_buffer.width() != dec_buffer.width() ||
119 ref_buffer.height() != dec_buffer.height()) {
120 RTC_CHECK_GE(ref_buffer.width(), dec_buffer.width());
121 RTC_CHECK_GE(ref_buffer.height(), dec_buffer.height());
122 // Downscale reference frame.
123 rtc::scoped_refptr<I420Buffer> scaled_buffer =
124 I420Buffer::Create(dec_buffer.width(), dec_buffer.height());
125 I420Scale(ref_buffer.DataY(), ref_buffer.StrideY(), ref_buffer.DataU(),
126 ref_buffer.StrideU(), ref_buffer.DataV(), ref_buffer.StrideV(),
127 ref_buffer.width(), ref_buffer.height(),
128 scaled_buffer->MutableDataY(), scaled_buffer->StrideY(),
129 scaled_buffer->MutableDataU(), scaled_buffer->StrideU(),
130 scaled_buffer->MutableDataV(), scaled_buffer->StrideV(),
131 scaled_buffer->width(), scaled_buffer->height(),
132 libyuv::kFilterBox);
133
134 CalculateFrameQuality(*scaled_buffer, dec_buffer, frame_stat, calc_ssim);
135 } else {
136 const uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane(
137 dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(),
138 ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height());
139
140 const uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane(
141 dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(),
142 ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2);
143
144 const uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane(
145 dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(),
146 ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2);
147
148 const size_t num_y_samples = dec_buffer.width() * dec_buffer.height();
149 const size_t num_u_samples =
150 dec_buffer.width() / 2 * dec_buffer.height() / 2;
151
152 frame_stat->psnr_y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples);
153 frame_stat->psnr_u = libyuv::SumSquareErrorToPsnr(sse_u, num_u_samples);
154 frame_stat->psnr_v = libyuv::SumSquareErrorToPsnr(sse_v, num_u_samples);
155 frame_stat->psnr = libyuv::SumSquareErrorToPsnr(
156 sse_y + sse_u + sse_v, num_y_samples + 2 * num_u_samples);
157
158 if (calc_ssim) {
159 frame_stat->ssim = I420SSIM(ref_buffer, dec_buffer);
160 }
161 }
162 }
163
164 } // namespace
165
VideoProcessor(webrtc::VideoEncoder * encoder,VideoDecoderList * decoders,FrameReader * input_frame_reader,const VideoCodecTestFixture::Config & config,VideoCodecTestStatsImpl * stats,IvfFileWriterMap * encoded_frame_writers,FrameWriterList * decoded_frame_writers)166 VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
167 VideoDecoderList* decoders,
168 FrameReader* input_frame_reader,
169 const VideoCodecTestFixture::Config& config,
170 VideoCodecTestStatsImpl* stats,
171 IvfFileWriterMap* encoded_frame_writers,
172 FrameWriterList* decoded_frame_writers)
173 : config_(config),
174 num_simulcast_or_spatial_layers_(
175 std::max(config_.NumberOfSimulcastStreams(),
176 config_.NumberOfSpatialLayers())),
177 stats_(stats),
178 encoder_(encoder),
179 decoders_(decoders),
180 bitrate_allocator_(
181 CreateBuiltinVideoBitrateAllocatorFactory()
182 ->CreateVideoBitrateAllocator(config_.codec_settings)),
183 framerate_fps_(0),
184 encode_callback_(this),
185 input_frame_reader_(input_frame_reader),
186 merged_encoded_frames_(num_simulcast_or_spatial_layers_),
187 encoded_frame_writers_(encoded_frame_writers),
188 decoded_frame_writers_(decoded_frame_writers),
189 last_inputed_frame_num_(0),
190 last_inputed_timestamp_(0),
191 first_encoded_frame_(num_simulcast_or_spatial_layers_, true),
192 last_encoded_frame_num_(num_simulcast_or_spatial_layers_),
193 first_decoded_frame_(num_simulcast_or_spatial_layers_, true),
194 last_decoded_frame_num_(num_simulcast_or_spatial_layers_),
195 decoded_frame_buffer_(num_simulcast_or_spatial_layers_),
196 post_encode_time_ns_(0) {
197 // Sanity checks.
198 RTC_CHECK(TaskQueueBase::Current())
199 << "VideoProcessor must be run on a task queue.";
200 RTC_CHECK(stats_);
201 RTC_CHECK(encoder_);
202 RTC_CHECK(decoders_);
203 RTC_CHECK_EQ(decoders_->size(), num_simulcast_or_spatial_layers_);
204 RTC_CHECK(input_frame_reader_);
205 RTC_CHECK(encoded_frame_writers_);
206 RTC_CHECK(!decoded_frame_writers ||
207 decoded_frame_writers->size() == num_simulcast_or_spatial_layers_);
208
209 // Setup required callbacks for the encoder and decoder and initialize them.
210 RTC_CHECK_EQ(encoder_->RegisterEncodeCompleteCallback(&encode_callback_),
211 WEBRTC_VIDEO_CODEC_OK);
212
213 // Initialize codecs so that they are ready to receive frames.
214 RTC_CHECK_EQ(encoder_->InitEncode(
215 &config_.codec_settings,
216 VideoEncoder::Settings(
217 kCapabilities, static_cast<int>(config_.NumberOfCores()),
218 config_.max_payload_size_bytes)),
219 WEBRTC_VIDEO_CODEC_OK);
220
221 for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
222 decode_callback_.push_back(
223 std::make_unique<VideoProcessorDecodeCompleteCallback>(this, i));
224 RTC_CHECK_EQ(
225 decoders_->at(i)->InitDecode(&config_.codec_settings,
226 static_cast<int>(config_.NumberOfCores())),
227 WEBRTC_VIDEO_CODEC_OK);
228 RTC_CHECK_EQ(decoders_->at(i)->RegisterDecodeCompleteCallback(
229 decode_callback_.at(i).get()),
230 WEBRTC_VIDEO_CODEC_OK);
231 }
232 }
233
~VideoProcessor()234 VideoProcessor::~VideoProcessor() {
235 RTC_DCHECK_RUN_ON(&sequence_checker_);
236
237 // Explicitly reset codecs, in case they don't do that themselves when they
238 // go out of scope.
239 RTC_CHECK_EQ(encoder_->Release(), WEBRTC_VIDEO_CODEC_OK);
240 encoder_->RegisterEncodeCompleteCallback(nullptr);
241 for (auto& decoder : *decoders_) {
242 RTC_CHECK_EQ(decoder->Release(), WEBRTC_VIDEO_CODEC_OK);
243 decoder->RegisterDecodeCompleteCallback(nullptr);
244 }
245
246 // Sanity check.
247 RTC_CHECK_LE(input_frames_.size(), kMaxBufferedInputFrames);
248 }
249
ProcessFrame()250 void VideoProcessor::ProcessFrame() {
251 RTC_DCHECK_RUN_ON(&sequence_checker_);
252 const size_t frame_number = last_inputed_frame_num_++;
253
254 // Get input frame and store for future quality calculation.
255 rtc::scoped_refptr<I420BufferInterface> buffer =
256 input_frame_reader_->ReadFrame();
257 RTC_CHECK(buffer) << "Tried to read too many frames from the file.";
258 const size_t timestamp =
259 last_inputed_timestamp_ +
260 static_cast<size_t>(kVideoPayloadTypeFrequency / framerate_fps_);
261 VideoFrame input_frame =
262 VideoFrame::Builder()
263 .set_video_frame_buffer(buffer)
264 .set_timestamp_rtp(static_cast<uint32_t>(timestamp))
265 .set_timestamp_ms(static_cast<int64_t>(timestamp / kMsToRtpTimestamp))
266 .set_rotation(webrtc::kVideoRotation_0)
267 .build();
268 // Store input frame as a reference for quality calculations.
269 if (config_.decode && !config_.measure_cpu) {
270 if (input_frames_.size() == kMaxBufferedInputFrames) {
271 input_frames_.erase(input_frames_.begin());
272 }
273 input_frames_.emplace(frame_number, input_frame);
274 }
275 last_inputed_timestamp_ = timestamp;
276
277 post_encode_time_ns_ = 0;
278
279 // Create frame statistics object for all simulcast/spatial layers.
280 for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
281 FrameStatistics frame_stat(frame_number, timestamp, i);
282 stats_->AddFrame(frame_stat);
283 }
284
285 // For the highest measurement accuracy of the encode time, the start/stop
286 // time recordings should wrap the Encode call as tightly as possible.
287 const int64_t encode_start_ns = rtc::TimeNanos();
288 for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
289 FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i);
290 frame_stat->encode_start_ns = encode_start_ns;
291 }
292
293 // Encode.
294 const std::vector<VideoFrameType> frame_types =
295 (frame_number == 0)
296 ? std::vector<VideoFrameType>{VideoFrameType::kVideoFrameKey}
297 : std::vector<VideoFrameType>{VideoFrameType::kVideoFrameDelta};
298 const int encode_return_code = encoder_->Encode(input_frame, &frame_types);
299 for (size_t i = 0; i < num_simulcast_or_spatial_layers_; ++i) {
300 FrameStatistics* frame_stat = stats_->GetFrame(frame_number, i);
301 frame_stat->encode_return_code = encode_return_code;
302 }
303 }
304
SetRates(size_t bitrate_kbps,double framerate_fps)305 void VideoProcessor::SetRates(size_t bitrate_kbps, double framerate_fps) {
306 RTC_DCHECK_RUN_ON(&sequence_checker_);
307 framerate_fps_ = framerate_fps;
308 bitrate_allocation_ =
309 bitrate_allocator_->Allocate(VideoBitrateAllocationParameters(
310 static_cast<uint32_t>(bitrate_kbps * 1000), framerate_fps_));
311 encoder_->SetRates(
312 VideoEncoder::RateControlParameters(bitrate_allocation_, framerate_fps_));
313 }
314
Decoded(VideoFrame & image)315 int32_t VideoProcessor::VideoProcessorDecodeCompleteCallback::Decoded(
316 VideoFrame& image) {
317 // Post the callback to the right task queue, if needed.
318 if (!task_queue_->IsCurrent()) {
319 // There might be a limited amount of output buffers, make a copy to make
320 // sure we don't block the decoder.
321 VideoFrame copy = VideoFrame::Builder()
322 .set_video_frame_buffer(I420Buffer::Copy(
323 *image.video_frame_buffer()->ToI420()))
324 .set_rotation(image.rotation())
325 .set_timestamp_us(image.timestamp_us())
326 .set_id(image.id())
327 .build();
328 copy.set_timestamp(image.timestamp());
329
330 task_queue_->PostTask(ToQueuedTask([this, copy]() {
331 video_processor_->FrameDecoded(copy, simulcast_svc_idx_);
332 }));
333 return 0;
334 }
335 video_processor_->FrameDecoded(image, simulcast_svc_idx_);
336 return 0;
337 }
338
FrameEncoded(const webrtc::EncodedImage & encoded_image,const webrtc::CodecSpecificInfo & codec_specific)339 void VideoProcessor::FrameEncoded(
340 const webrtc::EncodedImage& encoded_image,
341 const webrtc::CodecSpecificInfo& codec_specific) {
342 RTC_DCHECK_RUN_ON(&sequence_checker_);
343
344 // For the highest measurement accuracy of the encode time, the start/stop
345 // time recordings should wrap the Encode call as tightly as possible.
346 const int64_t encode_stop_ns = rtc::TimeNanos();
347
348 const VideoCodecType codec_type = codec_specific.codecType;
349 if (config_.encoded_frame_checker) {
350 config_.encoded_frame_checker->CheckEncodedFrame(codec_type, encoded_image);
351 }
352
353 // Layer metadata.
354 size_t spatial_idx = encoded_image.SpatialIndex().value_or(0);
355 size_t temporal_idx = GetTemporalLayerIndex(codec_specific);
356
357 FrameStatistics* frame_stat =
358 stats_->GetFrameWithTimestamp(encoded_image.Timestamp(), spatial_idx);
359 const size_t frame_number = frame_stat->frame_number;
360
361 // Ensure that the encode order is monotonically increasing, within this
362 // simulcast/spatial layer.
363 RTC_CHECK(first_encoded_frame_[spatial_idx] ||
364 last_encoded_frame_num_[spatial_idx] < frame_number);
365
366 // Ensure SVC spatial layers are delivered in ascending order.
367 const size_t num_spatial_layers = config_.NumberOfSpatialLayers();
368 if (!first_encoded_frame_[spatial_idx] && num_spatial_layers > 1) {
369 for (size_t i = 0; i < spatial_idx; ++i) {
370 RTC_CHECK_LE(last_encoded_frame_num_[i], frame_number);
371 }
372 for (size_t i = spatial_idx + 1; i < num_simulcast_or_spatial_layers_;
373 ++i) {
374 RTC_CHECK_GT(frame_number, last_encoded_frame_num_[i]);
375 }
376 }
377 first_encoded_frame_[spatial_idx] = false;
378 last_encoded_frame_num_[spatial_idx] = frame_number;
379
380 // Update frame statistics.
381 frame_stat->encoding_successful = true;
382 frame_stat->encode_time_us = GetElapsedTimeMicroseconds(
383 frame_stat->encode_start_ns, encode_stop_ns - post_encode_time_ns_);
384 frame_stat->target_bitrate_kbps =
385 bitrate_allocation_.GetTemporalLayerSum(spatial_idx, temporal_idx) / 1000;
386 frame_stat->target_framerate_fps = framerate_fps_;
387 frame_stat->length_bytes = encoded_image.size();
388 frame_stat->frame_type = encoded_image._frameType;
389 frame_stat->temporal_idx = temporal_idx;
390 frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_);
391 frame_stat->qp = encoded_image.qp_;
392
393 bool end_of_picture = false;
394 if (codec_type == kVideoCodecVP9) {
395 const CodecSpecificInfoVP9& vp9_info = codec_specific.codecSpecific.VP9;
396 frame_stat->inter_layer_predicted = vp9_info.inter_layer_predicted;
397 frame_stat->non_ref_for_inter_layer_pred =
398 vp9_info.non_ref_for_inter_layer_pred;
399 end_of_picture = vp9_info.end_of_picture;
400 } else {
401 frame_stat->inter_layer_predicted = false;
402 frame_stat->non_ref_for_inter_layer_pred = true;
403 }
404
405 const webrtc::EncodedImage* encoded_image_for_decode = &encoded_image;
406 if (config_.decode || !encoded_frame_writers_->empty()) {
407 if (num_spatial_layers > 1) {
408 encoded_image_for_decode = BuildAndStoreSuperframe(
409 encoded_image, codec_type, frame_number, spatial_idx,
410 frame_stat->inter_layer_predicted);
411 }
412 }
413
414 if (config_.decode) {
415 DecodeFrame(*encoded_image_for_decode, spatial_idx);
416
417 if (end_of_picture && num_spatial_layers > 1) {
418 // If inter-layer prediction is enabled and upper layer was dropped then
419 // base layer should be passed to upper layer decoder. Otherwise decoder
420 // won't be able to decode next superframe.
421 const EncodedImage* base_image = nullptr;
422 const FrameStatistics* base_stat = nullptr;
423 for (size_t i = 0; i < num_spatial_layers; ++i) {
424 const bool layer_dropped = (first_decoded_frame_[i] ||
425 last_decoded_frame_num_[i] < frame_number);
426
427 // Ensure current layer was decoded.
428 RTC_CHECK(layer_dropped == false || i != spatial_idx);
429
430 if (!layer_dropped) {
431 base_image = &merged_encoded_frames_[i];
432 base_stat =
433 stats_->GetFrameWithTimestamp(encoded_image.Timestamp(), i);
434 } else if (base_image && !base_stat->non_ref_for_inter_layer_pred) {
435 DecodeFrame(*base_image, i);
436 }
437 }
438 }
439 } else {
440 frame_stat->decode_return_code = WEBRTC_VIDEO_CODEC_NO_OUTPUT;
441 }
442
443 // Since frames in higher TLs typically depend on frames in lower TLs,
444 // write out frames in lower TLs to bitstream dumps of higher TLs.
445 for (size_t write_temporal_idx = temporal_idx;
446 write_temporal_idx < config_.NumberOfTemporalLayers();
447 ++write_temporal_idx) {
448 const VideoProcessor::LayerKey layer_key(spatial_idx, write_temporal_idx);
449 auto it = encoded_frame_writers_->find(layer_key);
450 if (it != encoded_frame_writers_->cend()) {
451 RTC_CHECK(it->second->WriteFrame(*encoded_image_for_decode,
452 config_.codec_settings.codecType));
453 }
454 }
455
456 if (!config_.encode_in_real_time) {
457 // To get pure encode time for next layers, measure time spent in encode
458 // callback and subtract it from encode time of next layers.
459 post_encode_time_ns_ += rtc::TimeNanos() - encode_stop_ns;
460 }
461 }
462
FrameDecoded(const VideoFrame & decoded_frame,size_t spatial_idx)463 void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame,
464 size_t spatial_idx) {
465 RTC_DCHECK_RUN_ON(&sequence_checker_);
466
467 // For the highest measurement accuracy of the decode time, the start/stop
468 // time recordings should wrap the Decode call as tightly as possible.
469 const int64_t decode_stop_ns = rtc::TimeNanos();
470
471 FrameStatistics* frame_stat =
472 stats_->GetFrameWithTimestamp(decoded_frame.timestamp(), spatial_idx);
473 const size_t frame_number = frame_stat->frame_number;
474
475 if (decoded_frame_writers_ && !first_decoded_frame_[spatial_idx]) {
476 // Fill drops with last decoded frame to make them look like freeze at
477 // playback and to keep decoded layers in sync.
478 for (size_t i = last_decoded_frame_num_[spatial_idx] + 1; i < frame_number;
479 ++i) {
480 RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
481 ->WriteFrame(decoded_frame_buffer_[spatial_idx].data()));
482 }
483 }
484
485 // Ensure that the decode order is monotonically increasing, within this
486 // simulcast/spatial layer.
487 RTC_CHECK(first_decoded_frame_[spatial_idx] ||
488 last_decoded_frame_num_[spatial_idx] < frame_number);
489 first_decoded_frame_[spatial_idx] = false;
490 last_decoded_frame_num_[spatial_idx] = frame_number;
491
492 // Update frame statistics.
493 frame_stat->decoding_successful = true;
494 frame_stat->decode_time_us =
495 GetElapsedTimeMicroseconds(frame_stat->decode_start_ns, decode_stop_ns);
496 frame_stat->decoded_width = decoded_frame.width();
497 frame_stat->decoded_height = decoded_frame.height();
498
499 // Skip quality metrics calculation to not affect CPU usage.
500 if (!config_.measure_cpu) {
501 const auto reference_frame = input_frames_.find(frame_number);
502 RTC_CHECK(reference_frame != input_frames_.cend())
503 << "The codecs are either buffering too much, dropping too much, or "
504 "being too slow relative the input frame rate.";
505
506 // SSIM calculation is not optimized. Skip it in real-time mode.
507 const bool calc_ssim = !config_.encode_in_real_time;
508 CalculateFrameQuality(
509 *reference_frame->second.video_frame_buffer()->ToI420(),
510 *decoded_frame.video_frame_buffer()->ToI420(), frame_stat, calc_ssim);
511
512 // Erase all buffered input frames that we have moved past for all
513 // simulcast/spatial layers. Never buffer more than
514 // |kMaxBufferedInputFrames| frames, to protect against long runs of
515 // consecutive frame drops for a particular layer.
516 const auto min_last_decoded_frame_num = std::min_element(
517 last_decoded_frame_num_.cbegin(), last_decoded_frame_num_.cend());
518 const size_t min_buffered_frame_num = std::max(
519 0, static_cast<int>(frame_number) - kMaxBufferedInputFrames + 1);
520 RTC_CHECK(min_last_decoded_frame_num != last_decoded_frame_num_.cend());
521 const auto input_frames_erase_before = input_frames_.lower_bound(
522 std::max(*min_last_decoded_frame_num, min_buffered_frame_num));
523 input_frames_.erase(input_frames_.cbegin(), input_frames_erase_before);
524 }
525
526 if (decoded_frame_writers_) {
527 ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width,
528 config_.codec_settings.height,
529 &decoded_frame_buffer_[spatial_idx]);
530 RTC_CHECK_EQ(decoded_frame_buffer_[spatial_idx].size(),
531 decoded_frame_writers_->at(spatial_idx)->FrameLength());
532 RTC_CHECK(decoded_frame_writers_->at(spatial_idx)
533 ->WriteFrame(decoded_frame_buffer_[spatial_idx].data()));
534 }
535 }
536
DecodeFrame(const EncodedImage & encoded_image,size_t spatial_idx)537 void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image,
538 size_t spatial_idx) {
539 RTC_DCHECK_RUN_ON(&sequence_checker_);
540 FrameStatistics* frame_stat =
541 stats_->GetFrameWithTimestamp(encoded_image.Timestamp(), spatial_idx);
542
543 frame_stat->decode_start_ns = rtc::TimeNanos();
544 frame_stat->decode_return_code =
545 decoders_->at(spatial_idx)->Decode(encoded_image, false, 0);
546 }
547
BuildAndStoreSuperframe(const EncodedImage & encoded_image,const VideoCodecType codec,size_t frame_number,size_t spatial_idx,bool inter_layer_predicted)548 const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe(
549 const EncodedImage& encoded_image,
550 const VideoCodecType codec,
551 size_t frame_number,
552 size_t spatial_idx,
553 bool inter_layer_predicted) {
554 // Should only be called for SVC.
555 RTC_CHECK_GT(config_.NumberOfSpatialLayers(), 1);
556
557 EncodedImage base_image;
558 RTC_CHECK_EQ(base_image.size(), 0);
559
560 // Each SVC layer is decoded with dedicated decoder. Find the nearest
561 // non-dropped base frame and merge it and current frame into superframe.
562 if (inter_layer_predicted) {
563 for (int base_idx = static_cast<int>(spatial_idx) - 1; base_idx >= 0;
564 --base_idx) {
565 EncodedImage lower_layer = merged_encoded_frames_.at(base_idx);
566 if (lower_layer.Timestamp() == encoded_image.Timestamp()) {
567 base_image = lower_layer;
568 break;
569 }
570 }
571 }
572 const size_t payload_size_bytes = base_image.size() + encoded_image.size();
573
574 EncodedImage copied_image = encoded_image;
575 copied_image.SetEncodedData(EncodedImageBuffer::Create(payload_size_bytes));
576 if (base_image.size()) {
577 RTC_CHECK(base_image.data());
578 memcpy(copied_image.data(), base_image.data(), base_image.size());
579 }
580 memcpy(copied_image.data() + base_image.size(), encoded_image.data(),
581 encoded_image.size());
582
583 copied_image.set_size(payload_size_bytes);
584
585 // Replace previous EncodedImage for this spatial layer.
586 merged_encoded_frames_.at(spatial_idx) = std::move(copied_image);
587
588 return &merged_encoded_frames_.at(spatial_idx);
589 }
590
591 } // namespace test
592 } // namespace webrtc
593