1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include <stdio.h>
11
12 #include <deque>
13 #include <map>
14
15 #include "testing/gtest/include/gtest/gtest.h"
16
17 #include "webrtc/base/thread_annotations.h"
18 #include "webrtc/call.h"
19 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
20 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
21 #include "webrtc/system_wrappers/interface/clock.h"
22 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
23 #include "webrtc/system_wrappers/interface/event_wrapper.h"
24 #include "webrtc/system_wrappers/interface/scoped_ptr.h"
25 #include "webrtc/system_wrappers/interface/sleep.h"
26 #include "webrtc/test/call_test.h"
27 #include "webrtc/test/direct_transport.h"
28 #include "webrtc/test/encoder_settings.h"
29 #include "webrtc/test/fake_encoder.h"
30 #include "webrtc/test/frame_generator_capturer.h"
31 #include "webrtc/test/statistics.h"
32 #include "webrtc/test/testsupport/fileutils.h"
33 #include "webrtc/typedefs.h"
34
35 namespace webrtc {
36
37 static const int kFullStackTestDurationSecs = 60;
38
39 struct FullStackTestParams {
40 const char* test_label;
41 struct {
42 const char* name;
43 size_t width, height;
44 int fps;
45 } clip;
46 int min_bitrate_bps;
47 int target_bitrate_bps;
48 int max_bitrate_bps;
49 double avg_psnr_threshold;
50 double avg_ssim_threshold;
51 FakeNetworkPipe::Config link;
52 };
53
54 class FullStackTest : public test::CallTest {
55 protected:
56 void RunTest(const FullStackTestParams& params);
57 };
58
59 class VideoAnalyzer : public PacketReceiver,
60 public newapi::Transport,
61 public VideoRenderer,
62 public VideoSendStreamInput {
63 public:
VideoAnalyzer(VideoSendStreamInput * input,Transport * transport,const char * test_label,double avg_psnr_threshold,double avg_ssim_threshold,int duration_frames)64 VideoAnalyzer(VideoSendStreamInput* input,
65 Transport* transport,
66 const char* test_label,
67 double avg_psnr_threshold,
68 double avg_ssim_threshold,
69 int duration_frames)
70 : input_(input),
71 transport_(transport),
72 receiver_(NULL),
73 test_label_(test_label),
74 frames_left_(duration_frames),
75 dropped_frames_(0),
76 last_render_time_(0),
77 rtp_timestamp_delta_(0),
78 crit_(CriticalSectionWrapper::CreateCriticalSection()),
79 first_send_frame_(NULL),
80 avg_psnr_threshold_(avg_psnr_threshold),
81 avg_ssim_threshold_(avg_ssim_threshold),
82 comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()),
83 comparison_thread_(ThreadWrapper::CreateThread(&FrameComparisonThread,
84 this)),
85 done_(EventWrapper::Create()) {
86 unsigned int id;
87 EXPECT_TRUE(comparison_thread_->Start(id));
88 }
89
~VideoAnalyzer()90 ~VideoAnalyzer() {
91 EXPECT_TRUE(comparison_thread_->Stop());
92
93 while (!frames_.empty()) {
94 delete frames_.back();
95 frames_.pop_back();
96 }
97 while (!frame_pool_.empty()) {
98 delete frame_pool_.back();
99 frame_pool_.pop_back();
100 }
101 }
102
SetReceiver(PacketReceiver * receiver)103 virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
104
DeliverPacket(const uint8_t * packet,size_t length)105 virtual DeliveryStatus DeliverPacket(const uint8_t* packet,
106 size_t length) OVERRIDE {
107 scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
108 RTPHeader header;
109 parser->Parse(packet, length, &header);
110 {
111 CriticalSectionScoped lock(crit_.get());
112 recv_times_[header.timestamp - rtp_timestamp_delta_] =
113 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
114 }
115
116 return receiver_->DeliverPacket(packet, length);
117 }
118
SwapFrame(I420VideoFrame * video_frame)119 virtual void SwapFrame(I420VideoFrame* video_frame) OVERRIDE {
120 I420VideoFrame* copy = NULL;
121 {
122 CriticalSectionScoped lock(crit_.get());
123 if (frame_pool_.size() > 0) {
124 copy = frame_pool_.front();
125 frame_pool_.pop_front();
126 }
127 }
128 if (copy == NULL)
129 copy = new I420VideoFrame();
130
131 copy->CopyFrame(*video_frame);
132 copy->set_timestamp(copy->render_time_ms() * 90);
133
134 {
135 CriticalSectionScoped lock(crit_.get());
136 if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0)
137 first_send_frame_ = copy;
138
139 frames_.push_back(copy);
140 }
141
142 input_->SwapFrame(video_frame);
143 }
144
SendRtp(const uint8_t * packet,size_t length)145 virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE {
146 scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
147 RTPHeader header;
148 parser->Parse(packet, length, &header);
149
150 {
151 CriticalSectionScoped lock(crit_.get());
152 if (rtp_timestamp_delta_ == 0) {
153 rtp_timestamp_delta_ =
154 header.timestamp - first_send_frame_->timestamp();
155 first_send_frame_ = NULL;
156 }
157 send_times_[header.timestamp - rtp_timestamp_delta_] =
158 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
159 }
160
161 return transport_->SendRtp(packet, length);
162 }
163
SendRtcp(const uint8_t * packet,size_t length)164 virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE {
165 return transport_->SendRtcp(packet, length);
166 }
167
RenderFrame(const I420VideoFrame & video_frame,int time_to_render_ms)168 virtual void RenderFrame(const I420VideoFrame& video_frame,
169 int time_to_render_ms) OVERRIDE {
170 int64_t render_time_ms =
171 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
172 uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
173
174 CriticalSectionScoped lock(crit_.get());
175 while (frames_.front()->timestamp() < send_timestamp) {
176 AddFrameComparison(
177 frames_.front(), &last_rendered_frame_, true, render_time_ms);
178 frame_pool_.push_back(frames_.front());
179 frames_.pop_front();
180 }
181
182 I420VideoFrame* reference_frame = frames_.front();
183 frames_.pop_front();
184 assert(reference_frame != NULL);
185 EXPECT_EQ(reference_frame->timestamp(), send_timestamp);
186 assert(reference_frame->timestamp() == send_timestamp);
187
188 AddFrameComparison(reference_frame, &video_frame, false, render_time_ms);
189 frame_pool_.push_back(reference_frame);
190
191 last_rendered_frame_.CopyFrame(video_frame);
192 }
193
Wait()194 void Wait() {
195 EXPECT_EQ(kEventSignaled, done_->Wait(FullStackTest::kLongTimeoutMs));
196 }
197
198 VideoSendStreamInput* input_;
199 Transport* transport_;
200 PacketReceiver* receiver_;
201
202 private:
203 struct FrameComparison {
FrameComparisonwebrtc::VideoAnalyzer::FrameComparison204 FrameComparison(const I420VideoFrame* reference,
205 const I420VideoFrame* render,
206 bool dropped,
207 int64_t send_time_ms,
208 int64_t recv_time_ms,
209 int64_t render_time_ms)
210 : dropped(dropped),
211 send_time_ms(send_time_ms),
212 recv_time_ms(recv_time_ms),
213 render_time_ms(render_time_ms) {
214 this->reference.CopyFrame(*reference);
215 this->render.CopyFrame(*render);
216 }
217
FrameComparisonwebrtc::VideoAnalyzer::FrameComparison218 FrameComparison(const FrameComparison& compare)
219 : dropped(compare.dropped),
220 send_time_ms(compare.send_time_ms),
221 recv_time_ms(compare.recv_time_ms),
222 render_time_ms(compare.render_time_ms) {
223 this->reference.CopyFrame(compare.reference);
224 this->render.CopyFrame(compare.render);
225 }
226
~FrameComparisonwebrtc::VideoAnalyzer::FrameComparison227 ~FrameComparison() {}
228
229 I420VideoFrame reference;
230 I420VideoFrame render;
231 bool dropped;
232 int64_t send_time_ms;
233 int64_t recv_time_ms;
234 int64_t render_time_ms;
235 };
236
AddFrameComparison(const I420VideoFrame * reference,const I420VideoFrame * render,bool dropped,int64_t render_time_ms)237 void AddFrameComparison(const I420VideoFrame* reference,
238 const I420VideoFrame* render,
239 bool dropped,
240 int64_t render_time_ms)
241 EXCLUSIVE_LOCKS_REQUIRED(crit_) {
242 int64_t send_time_ms = send_times_[reference->timestamp()];
243 send_times_.erase(reference->timestamp());
244 int64_t recv_time_ms = recv_times_[reference->timestamp()];
245 recv_times_.erase(reference->timestamp());
246
247 CriticalSectionScoped crit(comparison_lock_.get());
248 comparisons_.push_back(FrameComparison(reference,
249 render,
250 dropped,
251 send_time_ms,
252 recv_time_ms,
253 render_time_ms));
254 }
255
FrameComparisonThread(void * obj)256 static bool FrameComparisonThread(void* obj) {
257 return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
258 }
259
CompareFrames()260 bool CompareFrames() {
261 assert(frames_left_ > 0);
262
263 I420VideoFrame reference;
264 I420VideoFrame render;
265 bool dropped;
266 int64_t send_time_ms;
267 int64_t recv_time_ms;
268 int64_t render_time_ms;
269
270 SleepMs(10);
271
272 while (true) {
273 {
274 CriticalSectionScoped crit(comparison_lock_.get());
275 if (comparisons_.empty())
276 return true;
277 reference.SwapFrame(&comparisons_.front().reference);
278 render.SwapFrame(&comparisons_.front().render);
279 dropped = comparisons_.front().dropped;
280 send_time_ms = comparisons_.front().send_time_ms;
281 recv_time_ms = comparisons_.front().recv_time_ms;
282 render_time_ms = comparisons_.front().render_time_ms;
283 comparisons_.pop_front();
284 }
285
286 PerformFrameComparison(&reference,
287 &render,
288 dropped,
289 send_time_ms,
290 recv_time_ms,
291 render_time_ms);
292
293 if (--frames_left_ == 0) {
294 PrintResult("psnr", psnr_, " dB");
295 PrintResult("ssim", ssim_, "");
296 PrintResult("sender_time", sender_time_, " ms");
297 printf(
298 "RESULT dropped_frames: %s = %d\n", test_label_, dropped_frames_);
299 PrintResult("receiver_time", receiver_time_, " ms");
300 PrintResult("total_delay_incl_network", end_to_end_, " ms");
301 PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
302 EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
303 EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
304 done_->Set();
305
306 return false;
307 }
308 }
309 }
310
PerformFrameComparison(const I420VideoFrame * reference,const I420VideoFrame * render,bool dropped,int64_t send_time_ms,int64_t recv_time_ms,int64_t render_time_ms)311 void PerformFrameComparison(const I420VideoFrame* reference,
312 const I420VideoFrame* render,
313 bool dropped,
314 int64_t send_time_ms,
315 int64_t recv_time_ms,
316 int64_t render_time_ms) {
317 psnr_.AddSample(I420PSNR(reference, render));
318 ssim_.AddSample(I420SSIM(reference, render));
319 if (dropped) {
320 ++dropped_frames_;
321 return;
322 }
323 if (last_render_time_ != 0)
324 rendered_delta_.AddSample(render_time_ms - last_render_time_);
325 last_render_time_ = render_time_ms;
326
327 int64_t input_time_ms = reference->render_time_ms();
328 sender_time_.AddSample(send_time_ms - input_time_ms);
329 receiver_time_.AddSample(render_time_ms - recv_time_ms);
330 end_to_end_.AddSample(render_time_ms - input_time_ms);
331 }
332
PrintResult(const char * result_type,test::Statistics stats,const char * unit)333 void PrintResult(const char* result_type,
334 test::Statistics stats,
335 const char* unit) {
336 printf("RESULT %s: %s = {%f, %f}%s\n",
337 result_type,
338 test_label_,
339 stats.Mean(),
340 stats.StandardDeviation(),
341 unit);
342 }
343
344 const char* const test_label_;
345 test::Statistics sender_time_;
346 test::Statistics receiver_time_;
347 test::Statistics psnr_;
348 test::Statistics ssim_;
349 test::Statistics end_to_end_;
350 test::Statistics rendered_delta_;
351 int frames_left_;
352 int dropped_frames_;
353 int64_t last_render_time_;
354 uint32_t rtp_timestamp_delta_;
355
356 const scoped_ptr<CriticalSectionWrapper> crit_;
357 std::deque<I420VideoFrame*> frames_ GUARDED_BY(crit_);
358 std::deque<I420VideoFrame*> frame_pool_ GUARDED_BY(crit_);
359 I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
360 std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
361 std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
362 I420VideoFrame* first_send_frame_ GUARDED_BY(crit_);
363 double avg_psnr_threshold_ GUARDED_BY(crit_);
364 double avg_ssim_threshold_ GUARDED_BY(crit_);
365
366 const scoped_ptr<CriticalSectionWrapper> comparison_lock_;
367 const scoped_ptr<ThreadWrapper> comparison_thread_;
368 std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
369 const scoped_ptr<EventWrapper> done_;
370 };
371
RunTest(const FullStackTestParams & params)372 void FullStackTest::RunTest(const FullStackTestParams& params) {
373 test::DirectTransport send_transport(params.link);
374 test::DirectTransport recv_transport(params.link);
375 VideoAnalyzer analyzer(NULL,
376 &send_transport,
377 params.test_label,
378 params.avg_psnr_threshold,
379 params.avg_ssim_threshold,
380 kFullStackTestDurationSecs * params.clip.fps);
381
382 CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));
383
384 analyzer.SetReceiver(receiver_call_->Receiver());
385 send_transport.SetReceiver(&analyzer);
386 recv_transport.SetReceiver(sender_call_->Receiver());
387
388 CreateSendConfig(1);
389
390 scoped_ptr<VideoEncoder> encoder(
391 VideoEncoder::Create(VideoEncoder::kVp8));
392 send_config_.encoder_settings.encoder = encoder.get();
393 send_config_.encoder_settings.payload_name = "VP8";
394 send_config_.encoder_settings.payload_type = 124;
395
396 VideoStream* stream = &encoder_config_.streams[0];
397 stream->width = params.clip.width;
398 stream->height = params.clip.height;
399 stream->min_bitrate_bps = params.min_bitrate_bps;
400 stream->target_bitrate_bps = params.target_bitrate_bps;
401 stream->max_bitrate_bps = params.max_bitrate_bps;
402 stream->max_framerate = params.clip.fps;
403
404 CreateMatchingReceiveConfigs();
405 receive_configs_[0].renderer = &analyzer;
406
407 CreateStreams();
408 analyzer.input_ = send_stream_->Input();
409
410 frame_generator_capturer_.reset(
411 test::FrameGeneratorCapturer::CreateFromYuvFile(
412 &analyzer,
413 test::ResourcePath(params.clip.name, "yuv").c_str(),
414 params.clip.width,
415 params.clip.height,
416 params.clip.fps,
417 Clock::GetRealTimeClock()));
418
419 ASSERT_TRUE(frame_generator_capturer_.get() != NULL)
420 << "Could not create capturer for " << params.clip.name
421 << ".yuv. Is this resource file present?";
422
423 Start();
424
425 analyzer.Wait();
426
427 send_transport.StopSending();
428 recv_transport.StopSending();
429
430 Stop();
431
432 DestroyStreams();
433 }
434
TEST_F(FullStackTest,ParisQcifWithoutPacketLoss)435 TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) {
436 FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0",
437 {"paris_qcif", 176, 144, 30},
438 300000,
439 300000,
440 300000,
441 36.0,
442 0.96
443 };
444 RunTest(paris_qcif);
445 }
446
TEST_F(FullStackTest,ForemanCifWithoutPacketLoss)447 TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
448 // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
449 FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0",
450 {"foreman_cif", 352, 288, 30},
451 700000,
452 700000,
453 700000,
454 0.0,
455 0.0
456 };
457 RunTest(foreman_cif);
458 }
459
TEST_F(FullStackTest,ForemanCif500kbps)460 TEST_F(FullStackTest, ForemanCif500kbps) {
461 FullStackTestParams foreman_cif = {"foreman_cif_500kbps",
462 {"foreman_cif", 352, 288, 30},
463 30000,
464 500000,
465 2000000,
466 0.0,
467 0.0
468 };
469 foreman_cif.link.queue_length_packets = 0;
470 foreman_cif.link.queue_delay_ms = 0;
471 foreman_cif.link.link_capacity_kbps = 500;
472 RunTest(foreman_cif);
473 }
474
TEST_F(FullStackTest,ForemanCif500kbpsLimitedQueue)475 TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) {
476 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_32pkts_queue",
477 {"foreman_cif", 352, 288, 30},
478 30000,
479 500000,
480 2000000,
481 0.0,
482 0.0
483 };
484 foreman_cif.link.queue_length_packets = 32;
485 foreman_cif.link.queue_delay_ms = 0;
486 foreman_cif.link.link_capacity_kbps = 500;
487 RunTest(foreman_cif);
488 }
489
TEST_F(FullStackTest,ForemanCif500kbps100ms)490 TEST_F(FullStackTest, ForemanCif500kbps100ms) {
491 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms",
492 {"foreman_cif", 352, 288, 30},
493 30000,
494 500000,
495 2000000,
496 0.0,
497 0.0
498 };
499 foreman_cif.link.queue_length_packets = 0;
500 foreman_cif.link.queue_delay_ms = 100;
501 foreman_cif.link.link_capacity_kbps = 500;
502 RunTest(foreman_cif);
503 }
504
TEST_F(FullStackTest,ForemanCif500kbps100msLimitedQueue)505 TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) {
506 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms_32pkts_queue",
507 {"foreman_cif", 352, 288, 30},
508 30000,
509 500000,
510 2000000,
511 0.0,
512 0.0
513 };
514 foreman_cif.link.queue_length_packets = 32;
515 foreman_cif.link.queue_delay_ms = 100;
516 foreman_cif.link.link_capacity_kbps = 500;
517 RunTest(foreman_cif);
518 }
519
TEST_F(FullStackTest,ForemanCif1000kbps100msLimitedQueue)520 TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) {
521 FullStackTestParams foreman_cif = {"foreman_cif_1000kbps_100ms_32pkts_queue",
522 {"foreman_cif", 352, 288, 30},
523 30000,
524 2000000,
525 2000000,
526 0.0,
527 0.0
528 };
529 foreman_cif.link.queue_length_packets = 32;
530 foreman_cif.link.queue_delay_ms = 100;
531 foreman_cif.link.link_capacity_kbps = 1000;
532 RunTest(foreman_cif);
533 }
534 } // namespace webrtc
535