1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h"
12
13 #include <stdint.h>
14 #include <stdio.h>
15
16 #include <algorithm>
17 #include <cmath>
18 #include <memory>
19 #include <string>
20 #include <utility>
21 #include <vector>
22
23 #include "absl/strings/str_replace.h"
24 #include "absl/strings/string_view.h"
25 #include "absl/types/optional.h"
26 #include "api/array_view.h"
27 #include "api/test/metrics/global_metrics_logger_and_exporter.h"
28 #include "api/test/metrics/metric.h"
29 #include "api/transport/field_trial_based_config.h"
30 #include "api/video/video_bitrate_allocation.h"
31 #include "api/video_codecs/h264_profile_level_id.h"
32 #include "api/video_codecs/sdp_video_format.h"
33 #include "api/video_codecs/video_codec.h"
34 #include "api/video_codecs/video_decoder.h"
35 #include "api/video_codecs/video_decoder_factory_template.h"
36 #include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h"
37 #include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h"
38 #include "api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h"
39 #include "api/video_codecs/video_decoder_factory_template_open_h264_adapter.h"
40 #include "api/video_codecs/video_encoder_factory.h"
41 #include "api/video_codecs/video_encoder_factory_template.h"
42 #include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h"
43 #include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h"
44 #include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h"
45 #include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h"
46 #include "common_video/h264/h264_common.h"
47 #include "media/base/media_constants.h"
48 #include "modules/video_coding/codecs/h264/include/h264_globals.h"
49 #include "modules/video_coding/codecs/vp9/svc_config.h"
50 #include "modules/video_coding/utility/ivf_file_writer.h"
51 #include "rtc_base/checks.h"
52 #include "rtc_base/cpu_time.h"
53 #include "rtc_base/logging.h"
54 #include "rtc_base/strings/string_builder.h"
55 #include "rtc_base/time_utils.h"
56 #include "system_wrappers/include/cpu_info.h"
57 #include "system_wrappers/include/sleep.h"
58 #include "test/gtest.h"
59 #include "test/testsupport/file_utils.h"
60 #include "test/testsupport/frame_writer.h"
61 #include "test/video_codec_settings.h"
62 #include "video/config/simulcast.h"
63 #include "video/config/video_encoder_config.h"
64
65 namespace webrtc {
66 namespace test {
67 namespace {
68
69 using VideoStatistics = VideoCodecTestStats::VideoStatistics;
70
71 const int kBaseKeyFrameInterval = 3000;
72 const double kBitratePriority = 1.0;
73 const int kDefaultMaxFramerateFps = 30;
74 const int kMaxQp = 56;
75
ConfigureSimulcast(VideoCodec * codec_settings)76 void ConfigureSimulcast(VideoCodec* codec_settings) {
77 FieldTrialBasedConfig trials;
78 const std::vector<webrtc::VideoStream> streams = cricket::GetSimulcastConfig(
79 /*min_layer=*/1, codec_settings->numberOfSimulcastStreams,
80 codec_settings->width, codec_settings->height, kBitratePriority, kMaxQp,
81 /* is_screenshare = */ false, true, trials);
82
83 for (size_t i = 0; i < streams.size(); ++i) {
84 SimulcastStream* ss = &codec_settings->simulcastStream[i];
85 ss->width = static_cast<uint16_t>(streams[i].width);
86 ss->height = static_cast<uint16_t>(streams[i].height);
87 ss->numberOfTemporalLayers =
88 static_cast<unsigned char>(*streams[i].num_temporal_layers);
89 ss->maxBitrate = streams[i].max_bitrate_bps / 1000;
90 ss->targetBitrate = streams[i].target_bitrate_bps / 1000;
91 ss->minBitrate = streams[i].min_bitrate_bps / 1000;
92 ss->qpMax = streams[i].max_qp;
93 ss->active = true;
94 }
95 }
96
ConfigureSvc(VideoCodec * codec_settings)97 void ConfigureSvc(VideoCodec* codec_settings) {
98 RTC_CHECK_EQ(kVideoCodecVP9, codec_settings->codecType);
99
100 const std::vector<SpatialLayer> layers = GetSvcConfig(
101 codec_settings->width, codec_settings->height, kDefaultMaxFramerateFps,
102 /*first_active_layer=*/0, codec_settings->VP9()->numberOfSpatialLayers,
103 codec_settings->VP9()->numberOfTemporalLayers,
104 /* is_screen_sharing = */ false);
105 ASSERT_EQ(codec_settings->VP9()->numberOfSpatialLayers, layers.size())
106 << "GetSvcConfig returned fewer spatial layers than configured.";
107
108 for (size_t i = 0; i < layers.size(); ++i) {
109 codec_settings->spatialLayers[i] = layers[i];
110 }
111 }
112
CodecSpecificToString(const VideoCodec & codec)113 std::string CodecSpecificToString(const VideoCodec& codec) {
114 char buf[1024];
115 rtc::SimpleStringBuilder ss(buf);
116 switch (codec.codecType) {
117 case kVideoCodecVP8:
118 ss << "\nnum_temporal_layers: "
119 << static_cast<int>(codec.VP8().numberOfTemporalLayers);
120 ss << "\ndenoising: " << codec.VP8().denoisingOn;
121 ss << "\nautomatic_resize: " << codec.VP8().automaticResizeOn;
122 ss << "\nkey_frame_interval: " << codec.VP8().keyFrameInterval;
123 break;
124 case kVideoCodecVP9:
125 ss << "\nnum_temporal_layers: "
126 << static_cast<int>(codec.VP9().numberOfTemporalLayers);
127 ss << "\nnum_spatial_layers: "
128 << static_cast<int>(codec.VP9().numberOfSpatialLayers);
129 ss << "\ndenoising: " << codec.VP9().denoisingOn;
130 ss << "\nkey_frame_interval: " << codec.VP9().keyFrameInterval;
131 ss << "\nadaptive_qp_mode: " << codec.VP9().adaptiveQpMode;
132 ss << "\nautomatic_resize: " << codec.VP9().automaticResizeOn;
133 ss << "\nflexible_mode: " << codec.VP9().flexibleMode;
134 break;
135 case kVideoCodecH264:
136 ss << "\nkey_frame_interval: " << codec.H264().keyFrameInterval;
137 ss << "\nnum_temporal_layers: "
138 << static_cast<int>(codec.H264().numberOfTemporalLayers);
139 break;
140 default:
141 break;
142 }
143 return ss.str();
144 }
145
RunEncodeInRealTime(const VideoCodecTestFixtureImpl::Config & config)146 bool RunEncodeInRealTime(const VideoCodecTestFixtureImpl::Config& config) {
147 if (config.measure_cpu || config.encode_in_real_time) {
148 return true;
149 }
150 return false;
151 }
152
FilenameWithParams(const VideoCodecTestFixtureImpl::Config & config)153 std::string FilenameWithParams(
154 const VideoCodecTestFixtureImpl::Config& config) {
155 return config.filename + "_" + config.CodecName() + "_" +
156 std::to_string(config.codec_settings.startBitrate);
157 }
158
CreateSdpVideoFormat(const VideoCodecTestFixtureImpl::Config & config)159 SdpVideoFormat CreateSdpVideoFormat(
160 const VideoCodecTestFixtureImpl::Config& config) {
161 if (config.codec_settings.codecType == kVideoCodecH264) {
162 const char* packetization_mode =
163 config.h264_codec_settings.packetization_mode ==
164 H264PacketizationMode::NonInterleaved
165 ? "1"
166 : "0";
167 SdpVideoFormat::Parameters codec_params = {
168 {cricket::kH264FmtpProfileLevelId,
169 *H264ProfileLevelIdToString(H264ProfileLevelId(
170 config.h264_codec_settings.profile, H264Level::kLevel3_1))},
171 {cricket::kH264FmtpPacketizationMode, packetization_mode},
172 {cricket::kH264FmtpLevelAsymmetryAllowed, "1"}};
173
174 return SdpVideoFormat(config.codec_name, codec_params);
175 } else if (config.codec_settings.codecType == kVideoCodecVP9) {
176 return SdpVideoFormat(config.codec_name, {{"profile-id", "0"}});
177 }
178
179 return SdpVideoFormat(config.codec_name);
180 }
181
182 } // namespace
183
184 VideoCodecTestFixtureImpl::Config::Config() = default;
185
SetCodecSettings(std::string codec_name,size_t num_simulcast_streams,size_t num_spatial_layers,size_t num_temporal_layers,bool denoising_on,bool frame_dropper_on,bool spatial_resize_on,size_t width,size_t height)186 void VideoCodecTestFixtureImpl::Config::SetCodecSettings(
187 std::string codec_name,
188 size_t num_simulcast_streams,
189 size_t num_spatial_layers,
190 size_t num_temporal_layers,
191 bool denoising_on,
192 bool frame_dropper_on,
193 bool spatial_resize_on,
194 size_t width,
195 size_t height) {
196 this->codec_name = codec_name;
197 VideoCodecType codec_type = PayloadStringToCodecType(codec_name);
198 webrtc::test::CodecSettings(codec_type, &codec_settings);
199
200 // TODO(brandtr): Move the setting of `width` and `height` to the tests, and
201 // DCHECK that they are set before initializing the codec instead.
202 codec_settings.width = static_cast<uint16_t>(width);
203 codec_settings.height = static_cast<uint16_t>(height);
204
205 RTC_CHECK(num_simulcast_streams >= 1 &&
206 num_simulcast_streams <= kMaxSimulcastStreams);
207 RTC_CHECK(num_spatial_layers >= 1 && num_spatial_layers <= kMaxSpatialLayers);
208 RTC_CHECK(num_temporal_layers >= 1 &&
209 num_temporal_layers <= kMaxTemporalStreams);
210
211 // Simulcast is only available with VP8.
212 RTC_CHECK(num_simulcast_streams < 2 || codec_type == kVideoCodecVP8);
213
214 // Spatial scalability is only available with VP9.
215 RTC_CHECK(num_spatial_layers < 2 || codec_type == kVideoCodecVP9);
216
217 // Some base code requires numberOfSimulcastStreams to be set to zero
218 // when simulcast is not used.
219 codec_settings.numberOfSimulcastStreams =
220 num_simulcast_streams <= 1 ? 0
221 : static_cast<uint8_t>(num_simulcast_streams);
222
223 codec_settings.SetFrameDropEnabled(frame_dropper_on);
224 switch (codec_settings.codecType) {
225 case kVideoCodecVP8:
226 codec_settings.VP8()->numberOfTemporalLayers =
227 static_cast<uint8_t>(num_temporal_layers);
228 codec_settings.VP8()->denoisingOn = denoising_on;
229 codec_settings.VP8()->automaticResizeOn = spatial_resize_on;
230 codec_settings.VP8()->keyFrameInterval = kBaseKeyFrameInterval;
231 break;
232 case kVideoCodecVP9:
233 codec_settings.VP9()->numberOfTemporalLayers =
234 static_cast<uint8_t>(num_temporal_layers);
235 codec_settings.VP9()->denoisingOn = denoising_on;
236 codec_settings.VP9()->keyFrameInterval = kBaseKeyFrameInterval;
237 codec_settings.VP9()->automaticResizeOn = spatial_resize_on;
238 codec_settings.VP9()->numberOfSpatialLayers =
239 static_cast<uint8_t>(num_spatial_layers);
240 break;
241 case kVideoCodecAV1:
242 codec_settings.qpMax = 63;
243 break;
244 case kVideoCodecH264:
245 codec_settings.H264()->keyFrameInterval = kBaseKeyFrameInterval;
246 codec_settings.H264()->numberOfTemporalLayers =
247 static_cast<uint8_t>(num_temporal_layers);
248 break;
249 default:
250 break;
251 }
252
253 if (codec_settings.numberOfSimulcastStreams > 1) {
254 ConfigureSimulcast(&codec_settings);
255 } else if (codec_settings.codecType == kVideoCodecVP9 &&
256 codec_settings.VP9()->numberOfSpatialLayers > 1) {
257 ConfigureSvc(&codec_settings);
258 }
259 }
260
NumberOfCores() const261 size_t VideoCodecTestFixtureImpl::Config::NumberOfCores() const {
262 return use_single_core ? 1 : CpuInfo::DetectNumberOfCores();
263 }
264
NumberOfTemporalLayers() const265 size_t VideoCodecTestFixtureImpl::Config::NumberOfTemporalLayers() const {
266 if (codec_settings.codecType == kVideoCodecVP8) {
267 return codec_settings.VP8().numberOfTemporalLayers;
268 } else if (codec_settings.codecType == kVideoCodecVP9) {
269 return codec_settings.VP9().numberOfTemporalLayers;
270 } else if (codec_settings.codecType == kVideoCodecH264) {
271 return codec_settings.H264().numberOfTemporalLayers;
272 } else {
273 return 1;
274 }
275 }
276
NumberOfSpatialLayers() const277 size_t VideoCodecTestFixtureImpl::Config::NumberOfSpatialLayers() const {
278 if (codec_settings.codecType == kVideoCodecVP9) {
279 return codec_settings.VP9().numberOfSpatialLayers;
280 } else {
281 return 1;
282 }
283 }
284
NumberOfSimulcastStreams() const285 size_t VideoCodecTestFixtureImpl::Config::NumberOfSimulcastStreams() const {
286 return codec_settings.numberOfSimulcastStreams;
287 }
288
ToString() const289 std::string VideoCodecTestFixtureImpl::Config::ToString() const {
290 std::string codec_type = CodecTypeToPayloadString(codec_settings.codecType);
291 rtc::StringBuilder ss;
292 ss << "test_name: " << test_name;
293 ss << "\nfilename: " << filename;
294 ss << "\nnum_frames: " << num_frames;
295 ss << "\nmax_payload_size_bytes: " << max_payload_size_bytes;
296 ss << "\ndecode: " << decode;
297 ss << "\nuse_single_core: " << use_single_core;
298 ss << "\nmeasure_cpu: " << measure_cpu;
299 ss << "\nnum_cores: " << NumberOfCores();
300 ss << "\ncodec_type: " << codec_type;
301 ss << "\n\n--> codec_settings";
302 ss << "\nwidth: " << codec_settings.width;
303 ss << "\nheight: " << codec_settings.height;
304 ss << "\nmax_framerate_fps: " << codec_settings.maxFramerate;
305 ss << "\nstart_bitrate_kbps: " << codec_settings.startBitrate;
306 ss << "\nmax_bitrate_kbps: " << codec_settings.maxBitrate;
307 ss << "\nmin_bitrate_kbps: " << codec_settings.minBitrate;
308 ss << "\nmax_qp: " << codec_settings.qpMax;
309 ss << "\nnum_simulcast_streams: "
310 << static_cast<int>(codec_settings.numberOfSimulcastStreams);
311 ss << "\n\n--> codec_settings." << codec_type;
312 ss << "complexity: "
313 << static_cast<int>(codec_settings.GetVideoEncoderComplexity());
314 ss << "\nframe_dropping: " << codec_settings.GetFrameDropEnabled();
315 ss << "\n" << CodecSpecificToString(codec_settings);
316 if (codec_settings.numberOfSimulcastStreams > 1) {
317 for (int i = 0; i < codec_settings.numberOfSimulcastStreams; ++i) {
318 ss << "\n\n--> codec_settings.simulcastStream[" << i << "]";
319 const SimulcastStream& simulcast_stream =
320 codec_settings.simulcastStream[i];
321 ss << "\nwidth: " << simulcast_stream.width;
322 ss << "\nheight: " << simulcast_stream.height;
323 ss << "\nnum_temporal_layers: "
324 << static_cast<int>(simulcast_stream.numberOfTemporalLayers);
325 ss << "\nmin_bitrate_kbps: " << simulcast_stream.minBitrate;
326 ss << "\ntarget_bitrate_kbps: " << simulcast_stream.targetBitrate;
327 ss << "\nmax_bitrate_kbps: " << simulcast_stream.maxBitrate;
328 ss << "\nmax_qp: " << simulcast_stream.qpMax;
329 ss << "\nactive: " << simulcast_stream.active;
330 }
331 }
332 ss << "\n";
333 return ss.Release();
334 }
335
CodecName() const336 std::string VideoCodecTestFixtureImpl::Config::CodecName() const {
337 std::string name = codec_name;
338 if (name.empty()) {
339 name = CodecTypeToPayloadString(codec_settings.codecType);
340 }
341 if (codec_settings.codecType == kVideoCodecH264) {
342 if (h264_codec_settings.profile == H264Profile::kProfileConstrainedHigh) {
343 return name + "-CHP";
344 } else {
345 RTC_DCHECK_EQ(h264_codec_settings.profile,
346 H264Profile::kProfileConstrainedBaseline);
347 return name + "-CBP";
348 }
349 }
350 return name;
351 }
352
353 // TODO(kthelgason): Move this out of the test fixture impl and
354 // make available as a shared utility class.
CheckEncodedFrame(webrtc::VideoCodecType codec,const EncodedImage & encoded_frame) const355 void VideoCodecTestFixtureImpl::H264KeyframeChecker::CheckEncodedFrame(
356 webrtc::VideoCodecType codec,
357 const EncodedImage& encoded_frame) const {
358 EXPECT_EQ(kVideoCodecH264, codec);
359 bool contains_sps = false;
360 bool contains_pps = false;
361 bool contains_idr = false;
362 const std::vector<webrtc::H264::NaluIndex> nalu_indices =
363 webrtc::H264::FindNaluIndices(encoded_frame.data(), encoded_frame.size());
364 for (const webrtc::H264::NaluIndex& index : nalu_indices) {
365 webrtc::H264::NaluType nalu_type = webrtc::H264::ParseNaluType(
366 encoded_frame.data()[index.payload_start_offset]);
367 if (nalu_type == webrtc::H264::NaluType::kSps) {
368 contains_sps = true;
369 } else if (nalu_type == webrtc::H264::NaluType::kPps) {
370 contains_pps = true;
371 } else if (nalu_type == webrtc::H264::NaluType::kIdr) {
372 contains_idr = true;
373 }
374 }
375 if (encoded_frame._frameType == VideoFrameType::kVideoFrameKey) {
376 EXPECT_TRUE(contains_sps) << "Keyframe should contain SPS.";
377 EXPECT_TRUE(contains_pps) << "Keyframe should contain PPS.";
378 EXPECT_TRUE(contains_idr) << "Keyframe should contain IDR.";
379 } else if (encoded_frame._frameType == VideoFrameType::kVideoFrameDelta) {
380 EXPECT_FALSE(contains_sps) << "Delta frame should not contain SPS.";
381 EXPECT_FALSE(contains_pps) << "Delta frame should not contain PPS.";
382 EXPECT_FALSE(contains_idr) << "Delta frame should not contain IDR.";
383 } else {
384 RTC_DCHECK_NOTREACHED();
385 }
386 }
387
388 class VideoCodecTestFixtureImpl::CpuProcessTime final {
389 public:
CpuProcessTime(const Config & config)390 explicit CpuProcessTime(const Config& config) : config_(config) {}
~CpuProcessTime()391 ~CpuProcessTime() {}
392
Start()393 void Start() {
394 if (config_.measure_cpu) {
395 cpu_time_ -= rtc::GetProcessCpuTimeNanos();
396 wallclock_time_ -= rtc::SystemTimeNanos();
397 }
398 }
Stop()399 void Stop() {
400 if (config_.measure_cpu) {
401 cpu_time_ += rtc::GetProcessCpuTimeNanos();
402 wallclock_time_ += rtc::SystemTimeNanos();
403 }
404 }
Print() const405 void Print() const {
406 if (config_.measure_cpu) {
407 RTC_LOG(LS_INFO) << "cpu_usage_percent: "
408 << GetUsagePercent() / config_.NumberOfCores();
409 }
410 }
411
412 private:
GetUsagePercent() const413 double GetUsagePercent() const {
414 return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0;
415 }
416
417 const Config config_;
418 int64_t cpu_time_ = 0;
419 int64_t wallclock_time_ = 0;
420 };
421
VideoCodecTestFixtureImpl(Config config)422 VideoCodecTestFixtureImpl::VideoCodecTestFixtureImpl(Config config)
423 : encoder_factory_(std::make_unique<webrtc::VideoEncoderFactoryTemplate<
424 webrtc::LibvpxVp8EncoderTemplateAdapter,
425 webrtc::LibvpxVp9EncoderTemplateAdapter,
426 webrtc::OpenH264EncoderTemplateAdapter,
427 webrtc::LibaomAv1EncoderTemplateAdapter>>()),
428 decoder_factory_(std::make_unique<webrtc::VideoDecoderFactoryTemplate<
429 webrtc::LibvpxVp8DecoderTemplateAdapter,
430 webrtc::LibvpxVp9DecoderTemplateAdapter,
431 webrtc::OpenH264DecoderTemplateAdapter,
432 webrtc::Dav1dDecoderTemplateAdapter>>()),
433 config_(config) {}
434
VideoCodecTestFixtureImpl(Config config,std::unique_ptr<VideoDecoderFactory> decoder_factory,std::unique_ptr<VideoEncoderFactory> encoder_factory)435 VideoCodecTestFixtureImpl::VideoCodecTestFixtureImpl(
436 Config config,
437 std::unique_ptr<VideoDecoderFactory> decoder_factory,
438 std::unique_ptr<VideoEncoderFactory> encoder_factory)
439 : encoder_factory_(std::move(encoder_factory)),
440 decoder_factory_(std::move(decoder_factory)),
441 config_(config) {}
442
443 VideoCodecTestFixtureImpl::~VideoCodecTestFixtureImpl() = default;
444
445 // Processes all frames in the clip and verifies the result.
RunTest(const std::vector<RateProfile> & rate_profiles,const std::vector<RateControlThresholds> * rc_thresholds,const std::vector<QualityThresholds> * quality_thresholds,const BitstreamThresholds * bs_thresholds)446 void VideoCodecTestFixtureImpl::RunTest(
447 const std::vector<RateProfile>& rate_profiles,
448 const std::vector<RateControlThresholds>* rc_thresholds,
449 const std::vector<QualityThresholds>* quality_thresholds,
450 const BitstreamThresholds* bs_thresholds) {
451 RTC_DCHECK(!rate_profiles.empty());
452
453 // To emulate operation on a production VideoStreamEncoder, we call the
454 // codecs on a task queue.
455 TaskQueueForTest task_queue("VidProc TQ");
456
457 bool is_setup_succeeded = SetUpAndInitObjects(
458 &task_queue, rate_profiles[0].target_kbps, rate_profiles[0].input_fps);
459 EXPECT_TRUE(is_setup_succeeded);
460 if (!is_setup_succeeded) {
461 ReleaseAndCloseObjects(&task_queue);
462 return;
463 }
464
465 PrintSettings(&task_queue);
466 ProcessAllFrames(&task_queue, rate_profiles);
467 ReleaseAndCloseObjects(&task_queue);
468
469 AnalyzeAllFrames(rate_profiles, rc_thresholds, quality_thresholds,
470 bs_thresholds);
471 }
472
ProcessAllFrames(TaskQueueForTest * task_queue,const std::vector<RateProfile> & rate_profiles)473 void VideoCodecTestFixtureImpl::ProcessAllFrames(
474 TaskQueueForTest* task_queue,
475 const std::vector<RateProfile>& rate_profiles) {
476 // Set initial rates.
477 auto rate_profile = rate_profiles.begin();
478 task_queue->PostTask([this, rate_profile] {
479 processor_->SetRates(rate_profile->target_kbps, rate_profile->input_fps);
480 });
481
482 cpu_process_time_->Start();
483
484 for (size_t frame_num = 0; frame_num < config_.num_frames; ++frame_num) {
485 auto next_rate_profile = std::next(rate_profile);
486 if (next_rate_profile != rate_profiles.end() &&
487 frame_num == next_rate_profile->frame_num) {
488 rate_profile = next_rate_profile;
489 task_queue->PostTask([this, rate_profile] {
490 processor_->SetRates(rate_profile->target_kbps,
491 rate_profile->input_fps);
492 });
493 }
494
495 task_queue->PostTask([this] { processor_->ProcessFrame(); });
496
497 if (RunEncodeInRealTime(config_)) {
498 // Roughly pace the frames.
499 const int frame_duration_ms =
500 std::ceil(rtc::kNumMillisecsPerSec / rate_profile->input_fps);
501 SleepMs(frame_duration_ms);
502 }
503 }
504
505 task_queue->PostTask([this] { processor_->Finalize(); });
506
507 // Wait until we know that the last frame has been sent for encode.
508 task_queue->SendTask([] {});
509
510 // Give the VideoProcessor pipeline some time to process the last frame,
511 // and then release the codecs.
512 SleepMs(1 * rtc::kNumMillisecsPerSec);
513 cpu_process_time_->Stop();
514 }
515
AnalyzeAllFrames(const std::vector<RateProfile> & rate_profiles,const std::vector<RateControlThresholds> * rc_thresholds,const std::vector<QualityThresholds> * quality_thresholds,const BitstreamThresholds * bs_thresholds)516 void VideoCodecTestFixtureImpl::AnalyzeAllFrames(
517 const std::vector<RateProfile>& rate_profiles,
518 const std::vector<RateControlThresholds>* rc_thresholds,
519 const std::vector<QualityThresholds>* quality_thresholds,
520 const BitstreamThresholds* bs_thresholds) {
521
522 for (size_t rate_profile_idx = 0; rate_profile_idx < rate_profiles.size();
523 ++rate_profile_idx) {
524 const size_t first_frame_num = rate_profiles[rate_profile_idx].frame_num;
525 const size_t last_frame_num =
526 rate_profile_idx + 1 < rate_profiles.size()
527 ? rate_profiles[rate_profile_idx + 1].frame_num - 1
528 : config_.num_frames - 1;
529 RTC_CHECK(last_frame_num >= first_frame_num);
530
531 VideoStatistics send_stat = stats_.SliceAndCalcAggregatedVideoStatistic(
532 first_frame_num, last_frame_num);
533 RTC_LOG(LS_INFO) << "==> Send stats";
534 RTC_LOG(LS_INFO) << send_stat.ToString("send_") << "\n";
535
536 std::vector<VideoStatistics> layer_stats =
537 stats_.SliceAndCalcLayerVideoStatistic(first_frame_num, last_frame_num);
538 RTC_LOG(LS_INFO) << "==> Receive stats";
539 for (const auto& layer_stat : layer_stats) {
540 RTC_LOG(LS_INFO) << layer_stat.ToString("recv_") << "\n";
541
542 // For perf dashboard.
543 char modifier_buf[256];
544 rtc::SimpleStringBuilder modifier(modifier_buf);
545 modifier << "_r" << rate_profile_idx << "_sl" << layer_stat.spatial_idx;
546
547 auto PrintResultHelper = [&modifier, this](
548 absl::string_view measurement, double value,
549 Unit unit,
550 absl::string_view non_standard_unit_suffix,
551 ImprovementDirection improvement_direction) {
552 rtc::StringBuilder metric_name(measurement);
553 metric_name << modifier.str() << non_standard_unit_suffix;
554 GetGlobalMetricsLogger()->LogSingleValueMetric(
555 metric_name.str(), config_.test_name, value, unit,
556 improvement_direction);
557 };
558
559 if (layer_stat.temporal_idx == config_.NumberOfTemporalLayers() - 1) {
560 PrintResultHelper("enc_speed", layer_stat.enc_speed_fps,
561 Unit::kUnitless, /*non_standard_unit_suffix=*/"_fps",
562 ImprovementDirection::kBiggerIsBetter);
563 PrintResultHelper("avg_key_frame_size",
564 layer_stat.avg_key_frame_size_bytes, Unit::kBytes,
565 /*non_standard_unit_suffix=*/"",
566 ImprovementDirection::kNeitherIsBetter);
567 PrintResultHelper("num_key_frames", layer_stat.num_key_frames,
568 Unit::kCount,
569 /*non_standard_unit_suffix=*/"",
570 ImprovementDirection::kNeitherIsBetter);
571 printf("\n");
572 }
573
574 modifier << "tl" << layer_stat.temporal_idx;
575 PrintResultHelper("dec_speed", layer_stat.dec_speed_fps, Unit::kUnitless,
576 /*non_standard_unit_suffix=*/"_fps",
577 ImprovementDirection::kBiggerIsBetter);
578 PrintResultHelper("avg_delta_frame_size",
579 layer_stat.avg_delta_frame_size_bytes, Unit::kBytes,
580 /*non_standard_unit_suffix=*/"",
581 ImprovementDirection::kNeitherIsBetter);
582 PrintResultHelper("bitrate", layer_stat.bitrate_kbps,
583 Unit::kKilobitsPerSecond,
584 /*non_standard_unit_suffix=*/"",
585 ImprovementDirection::kNeitherIsBetter);
586 PrintResultHelper("framerate", layer_stat.framerate_fps, Unit::kUnitless,
587 /*non_standard_unit_suffix=*/"_fps",
588 ImprovementDirection::kNeitherIsBetter);
589 PrintResultHelper("avg_psnr_y", layer_stat.avg_psnr_y, Unit::kUnitless,
590 /*non_standard_unit_suffix=*/"_dB",
591 ImprovementDirection::kBiggerIsBetter);
592 PrintResultHelper("avg_psnr_u", layer_stat.avg_psnr_u, Unit::kUnitless,
593 /*non_standard_unit_suffix=*/"_dB",
594 ImprovementDirection::kBiggerIsBetter);
595 PrintResultHelper("avg_psnr_v", layer_stat.avg_psnr_v, Unit::kUnitless,
596 /*non_standard_unit_suffix=*/"_dB",
597 ImprovementDirection::kBiggerIsBetter);
598 PrintResultHelper("min_psnr_yuv", layer_stat.min_psnr, Unit::kUnitless,
599 /*non_standard_unit_suffix=*/"_dB",
600 ImprovementDirection::kBiggerIsBetter);
601 PrintResultHelper("avg_qp", layer_stat.avg_qp, Unit::kUnitless,
602 /*non_standard_unit_suffix=*/"",
603 ImprovementDirection::kSmallerIsBetter);
604 printf("\n");
605 if (layer_stat.temporal_idx == config_.NumberOfTemporalLayers() - 1) {
606 printf("\n");
607 }
608 }
609
610 const RateControlThresholds* rc_threshold =
611 rc_thresholds ? &(*rc_thresholds)[rate_profile_idx] : nullptr;
612 const QualityThresholds* quality_threshold =
613 quality_thresholds ? &(*quality_thresholds)[rate_profile_idx] : nullptr;
614
615 VerifyVideoStatistic(send_stat, rc_threshold, quality_threshold,
616 bs_thresholds,
617 rate_profiles[rate_profile_idx].target_kbps,
618 rate_profiles[rate_profile_idx].input_fps);
619 }
620
621 if (config_.print_frame_level_stats) {
622 RTC_LOG(LS_INFO) << "==> Frame stats";
623 std::vector<VideoCodecTestStats::FrameStatistics> frame_stats =
624 stats_.GetFrameStatistics();
625 for (const auto& frame_stat : frame_stats) {
626 RTC_LOG(LS_INFO) << frame_stat.ToString();
627 }
628 }
629
630 cpu_process_time_->Print();
631 }
632
VerifyVideoStatistic(const VideoStatistics & video_stat,const RateControlThresholds * rc_thresholds,const QualityThresholds * quality_thresholds,const BitstreamThresholds * bs_thresholds,size_t target_bitrate_kbps,double input_framerate_fps)633 void VideoCodecTestFixtureImpl::VerifyVideoStatistic(
634 const VideoStatistics& video_stat,
635 const RateControlThresholds* rc_thresholds,
636 const QualityThresholds* quality_thresholds,
637 const BitstreamThresholds* bs_thresholds,
638 size_t target_bitrate_kbps,
639 double input_framerate_fps) {
640 if (rc_thresholds) {
641 const float bitrate_mismatch_percent =
642 100 * std::fabs(1.0f * video_stat.bitrate_kbps - target_bitrate_kbps) /
643 target_bitrate_kbps;
644 const float framerate_mismatch_percent =
645 100 * std::fabs(video_stat.framerate_fps - input_framerate_fps) /
646 input_framerate_fps;
647 EXPECT_LE(bitrate_mismatch_percent,
648 rc_thresholds->max_avg_bitrate_mismatch_percent);
649 EXPECT_LE(video_stat.time_to_reach_target_bitrate_sec,
650 rc_thresholds->max_time_to_reach_target_bitrate_sec);
651 EXPECT_LE(framerate_mismatch_percent,
652 rc_thresholds->max_avg_framerate_mismatch_percent);
653 EXPECT_LE(video_stat.avg_delay_sec,
654 rc_thresholds->max_avg_buffer_level_sec);
655 EXPECT_LE(video_stat.max_key_frame_delay_sec,
656 rc_thresholds->max_max_key_frame_delay_sec);
657 EXPECT_LE(video_stat.max_delta_frame_delay_sec,
658 rc_thresholds->max_max_delta_frame_delay_sec);
659 EXPECT_LE(video_stat.num_spatial_resizes,
660 rc_thresholds->max_num_spatial_resizes);
661 EXPECT_LE(video_stat.num_key_frames, rc_thresholds->max_num_key_frames);
662 }
663
664 if (quality_thresholds) {
665 EXPECT_GT(video_stat.avg_psnr, quality_thresholds->min_avg_psnr);
666 EXPECT_GT(video_stat.min_psnr, quality_thresholds->min_min_psnr);
667
668 // SSIM calculation is not optimized and thus it is disabled in real-time
669 // mode.
670 if (!config_.encode_in_real_time) {
671 EXPECT_GT(video_stat.avg_ssim, quality_thresholds->min_avg_ssim);
672 EXPECT_GT(video_stat.min_ssim, quality_thresholds->min_min_ssim);
673 }
674 }
675
676 if (bs_thresholds) {
677 EXPECT_LE(video_stat.max_nalu_size_bytes,
678 bs_thresholds->max_max_nalu_size_bytes);
679 }
680 }
681
CreateEncoderAndDecoder()682 bool VideoCodecTestFixtureImpl::CreateEncoderAndDecoder() {
683 SdpVideoFormat encoder_format(CreateSdpVideoFormat(config_));
684 SdpVideoFormat decoder_format = encoder_format;
685
686 // Override encoder and decoder formats with explicitly provided ones.
687 if (config_.encoder_format) {
688 RTC_DCHECK_EQ(config_.encoder_format->name, config_.codec_name);
689 encoder_format = *config_.encoder_format;
690 }
691
692 if (config_.decoder_format) {
693 RTC_DCHECK_EQ(config_.decoder_format->name, config_.codec_name);
694 decoder_format = *config_.decoder_format;
695 }
696
697 encoder_ = encoder_factory_->CreateVideoEncoder(encoder_format);
698 EXPECT_TRUE(encoder_) << "Encoder not successfully created.";
699 if (encoder_ == nullptr) {
700 return false;
701 }
702
703 const size_t num_simulcast_or_spatial_layers = std::max(
704 config_.NumberOfSimulcastStreams(), config_.NumberOfSpatialLayers());
705 for (size_t i = 0; i < num_simulcast_or_spatial_layers; ++i) {
706 std::unique_ptr<VideoDecoder> decoder =
707 decoder_factory_->CreateVideoDecoder(decoder_format);
708 EXPECT_TRUE(decoder) << "Decoder not successfully created.";
709 if (decoder == nullptr) {
710 return false;
711 }
712 decoders_.push_back(std::move(decoder));
713 }
714
715 return true;
716 }
717
DestroyEncoderAndDecoder()718 void VideoCodecTestFixtureImpl::DestroyEncoderAndDecoder() {
719 decoders_.clear();
720 encoder_.reset();
721 }
722
GetStats()723 VideoCodecTestStats& VideoCodecTestFixtureImpl::GetStats() {
724 return stats_;
725 }
726
SetUpAndInitObjects(TaskQueueForTest * task_queue,size_t initial_bitrate_kbps,double initial_framerate_fps)727 bool VideoCodecTestFixtureImpl::SetUpAndInitObjects(
728 TaskQueueForTest* task_queue,
729 size_t initial_bitrate_kbps,
730 double initial_framerate_fps) {
731 config_.codec_settings.minBitrate = 0;
732 config_.codec_settings.startBitrate = static_cast<int>(initial_bitrate_kbps);
733 config_.codec_settings.maxFramerate = std::ceil(initial_framerate_fps);
734
735 int clip_width = config_.clip_width.value_or(config_.codec_settings.width);
736 int clip_height = config_.clip_height.value_or(config_.codec_settings.height);
737
738 // Create file objects for quality analysis.
739 source_frame_reader_.reset(new YuvFrameReaderImpl(
740 config_.filepath, clip_width, clip_height,
741 config_.reference_width.value_or(clip_width),
742 config_.reference_height.value_or(clip_height),
743 YuvFrameReaderImpl::RepeatMode::kPingPong, config_.clip_fps,
744 config_.codec_settings.maxFramerate));
745 EXPECT_TRUE(source_frame_reader_->Init());
746
747 RTC_DCHECK(encoded_frame_writers_.empty());
748 RTC_DCHECK(decoded_frame_writers_.empty());
749
750 stats_.Clear();
751
752 cpu_process_time_.reset(new CpuProcessTime(config_));
753
754 bool is_codec_created = false;
755 task_queue->SendTask([this, &is_codec_created]() {
756 is_codec_created = CreateEncoderAndDecoder();
757 });
758
759 if (!is_codec_created) {
760 return false;
761 }
762
763 if (config_.visualization_params.save_encoded_ivf ||
764 config_.visualization_params.save_decoded_y4m) {
765 std::string encoder_name = GetCodecName(task_queue, /*is_encoder=*/true);
766 encoder_name = absl::StrReplaceAll(encoder_name, {{":", ""}, {" ", "-"}});
767
768 const size_t num_simulcast_or_spatial_layers = std::max(
769 config_.NumberOfSimulcastStreams(), config_.NumberOfSpatialLayers());
770 const size_t num_temporal_layers = config_.NumberOfTemporalLayers();
771 for (size_t simulcast_svc_idx = 0;
772 simulcast_svc_idx < num_simulcast_or_spatial_layers;
773 ++simulcast_svc_idx) {
774 const std::string output_filename_base =
775 JoinFilename(config_.output_path,
776 FilenameWithParams(config_) + "_" + encoder_name +
777 "_sl" + std::to_string(simulcast_svc_idx));
778
779 if (config_.visualization_params.save_encoded_ivf) {
780 for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers;
781 ++temporal_idx) {
782 const std::string output_file_path = output_filename_base + "tl" +
783 std::to_string(temporal_idx) +
784 ".ivf";
785 FileWrapper ivf_file = FileWrapper::OpenWriteOnly(output_file_path);
786
787 const VideoProcessor::LayerKey layer_key(simulcast_svc_idx,
788 temporal_idx);
789 encoded_frame_writers_[layer_key] =
790 IvfFileWriter::Wrap(std::move(ivf_file), /*byte_limit=*/0);
791 }
792 }
793
794 if (config_.visualization_params.save_decoded_y4m) {
795 FrameWriter* decoded_frame_writer = new Y4mFrameWriterImpl(
796 output_filename_base + ".y4m", config_.codec_settings.width,
797 config_.codec_settings.height, config_.codec_settings.maxFramerate);
798 EXPECT_TRUE(decoded_frame_writer->Init());
799 decoded_frame_writers_.push_back(
800 std::unique_ptr<FrameWriter>(decoded_frame_writer));
801 }
802 }
803 }
804
805 task_queue->SendTask(
806 [this]() {
807 processor_ = std::make_unique<VideoProcessor>(
808 encoder_.get(), &decoders_, source_frame_reader_.get(), config_,
809 &stats_, &encoded_frame_writers_,
810 decoded_frame_writers_.empty() ? nullptr : &decoded_frame_writers_);
811 });
812 return true;
813 }
814
ReleaseAndCloseObjects(TaskQueueForTest * task_queue)815 void VideoCodecTestFixtureImpl::ReleaseAndCloseObjects(
816 TaskQueueForTest* task_queue) {
817 task_queue->SendTask([this]() {
818 processor_.reset();
819 // The VideoProcessor must be destroyed before the codecs.
820 DestroyEncoderAndDecoder();
821 });
822
823 source_frame_reader_->Close();
824
825 // Close visualization files.
826 for (auto& encoded_frame_writer : encoded_frame_writers_) {
827 EXPECT_TRUE(encoded_frame_writer.second->Close());
828 }
829 encoded_frame_writers_.clear();
830 for (auto& decoded_frame_writer : decoded_frame_writers_) {
831 decoded_frame_writer->Close();
832 }
833 decoded_frame_writers_.clear();
834 }
835
GetCodecName(TaskQueueForTest * task_queue,bool is_encoder) const836 std::string VideoCodecTestFixtureImpl::GetCodecName(
837 TaskQueueForTest* task_queue,
838 bool is_encoder) const {
839 std::string codec_name;
840 task_queue->SendTask([this, is_encoder, &codec_name] {
841 if (is_encoder) {
842 codec_name = encoder_->GetEncoderInfo().implementation_name;
843 } else {
844 codec_name = decoders_.at(0)->ImplementationName();
845 }
846 });
847 return codec_name;
848 }
849
PrintSettings(TaskQueueForTest * task_queue) const850 void VideoCodecTestFixtureImpl::PrintSettings(
851 TaskQueueForTest* task_queue) const {
852 RTC_LOG(LS_INFO) << "==> Config";
853 RTC_LOG(LS_INFO) << config_.ToString();
854
855 RTC_LOG(LS_INFO) << "==> Codec names";
856 RTC_LOG(LS_INFO) << "enc_impl_name: "
857 << GetCodecName(task_queue, /*is_encoder=*/true);
858 RTC_LOG(LS_INFO) << "dec_impl_name: "
859 << GetCodecName(task_queue, /*is_encoder=*/false);
860 }
861
862 } // namespace test
863 } // namespace webrtc
864