1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include <stdio.h>
11
12 #include <deque>
13 #include <map>
14
15 #include "testing/gtest/include/gtest/gtest.h"
16
17 #include "webrtc/call.h"
18 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
19 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
20 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
21 #include "webrtc/system_wrappers/interface/clock.h"
22 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
23 #include "webrtc/system_wrappers/interface/event_wrapper.h"
24 #include "webrtc/system_wrappers/interface/scoped_ptr.h"
25 #include "webrtc/system_wrappers/interface/sleep.h"
26 #include "webrtc/system_wrappers/interface/thread_annotations.h"
27 #include "webrtc/test/direct_transport.h"
28 #include "webrtc/test/encoder_settings.h"
29 #include "webrtc/test/fake_encoder.h"
30 #include "webrtc/test/frame_generator_capturer.h"
31 #include "webrtc/test/statistics.h"
32 #include "webrtc/test/testsupport/fileutils.h"
33 #include "webrtc/typedefs.h"
34
35 namespace webrtc {
36
37 static const uint32_t kSendSsrc = 0x654321;
38 static const int kFullStackTestDurationSecs = 10;
39
40 struct FullStackTestParams {
41 const char* test_label;
42 struct {
43 const char* name;
44 size_t width, height;
45 int fps;
46 } clip;
47 unsigned int bitrate;
48 double avg_psnr_threshold;
49 double avg_ssim_threshold;
50 };
51
52 FullStackTestParams paris_qcif = {
53 "net_delay_0_0_plr_0", {"paris_qcif", 176, 144, 30}, 300, 36.0, 0.96};
54
55 // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
56 FullStackTestParams foreman_cif = {
57 "foreman_cif_net_delay_0_0_plr_0",
58 {"foreman_cif", 352, 288, 30},
59 700,
60 0.0,
61 0.0};
62
63 class FullStackTest : public ::testing::TestWithParam<FullStackTestParams> {
64 protected:
65 std::map<uint32_t, bool> reserved_ssrcs_;
66 };
67
68 class VideoAnalyzer : public PacketReceiver,
69 public newapi::Transport,
70 public VideoRenderer,
71 public VideoSendStreamInput {
72 public:
VideoAnalyzer(VideoSendStreamInput * input,Transport * transport,const char * test_label,double avg_psnr_threshold,double avg_ssim_threshold,int duration_frames)73 VideoAnalyzer(VideoSendStreamInput* input,
74 Transport* transport,
75 const char* test_label,
76 double avg_psnr_threshold,
77 double avg_ssim_threshold,
78 int duration_frames)
79 : input_(input),
80 transport_(transport),
81 receiver_(NULL),
82 test_label_(test_label),
83 frames_left_(duration_frames),
84 dropped_frames_(0),
85 last_render_time_(0),
86 rtp_timestamp_delta_(0),
87 crit_(CriticalSectionWrapper::CreateCriticalSection()),
88 first_send_frame_(NULL),
89 avg_psnr_threshold_(avg_psnr_threshold),
90 avg_ssim_threshold_(avg_ssim_threshold),
91 comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()),
92 comparison_thread_(ThreadWrapper::CreateThread(&FrameComparisonThread,
93 this)),
94 done_(EventWrapper::Create()) {
95 unsigned int id;
96 EXPECT_TRUE(comparison_thread_->Start(id));
97 }
98
~VideoAnalyzer()99 ~VideoAnalyzer() {
100 EXPECT_TRUE(comparison_thread_->Stop());
101
102 while (!frames_.empty()) {
103 delete frames_.back();
104 frames_.pop_back();
105 }
106 while (!frame_pool_.empty()) {
107 delete frame_pool_.back();
108 frame_pool_.pop_back();
109 }
110 }
111
SetReceiver(PacketReceiver * receiver)112 virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
113
DeliverPacket(const uint8_t * packet,size_t length)114 virtual DeliveryStatus DeliverPacket(const uint8_t* packet,
115 size_t length) OVERRIDE {
116 scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
117 RTPHeader header;
118 parser->Parse(packet, static_cast<int>(length), &header);
119 {
120 CriticalSectionScoped lock(crit_.get());
121 recv_times_[header.timestamp - rtp_timestamp_delta_] =
122 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
123 }
124
125 return receiver_->DeliverPacket(packet, length);
126 }
127
SwapFrame(I420VideoFrame * video_frame)128 virtual void SwapFrame(I420VideoFrame* video_frame) OVERRIDE {
129 I420VideoFrame* copy = NULL;
130 {
131 CriticalSectionScoped lock(crit_.get());
132 if (frame_pool_.size() > 0) {
133 copy = frame_pool_.front();
134 frame_pool_.pop_front();
135 }
136 }
137 if (copy == NULL)
138 copy = new I420VideoFrame();
139
140 copy->CopyFrame(*video_frame);
141 copy->set_timestamp(copy->render_time_ms() * 90);
142
143 {
144 CriticalSectionScoped lock(crit_.get());
145 if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0)
146 first_send_frame_ = copy;
147
148 frames_.push_back(copy);
149 }
150
151 input_->SwapFrame(video_frame);
152 }
153
SendRtp(const uint8_t * packet,size_t length)154 virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE {
155 scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
156 RTPHeader header;
157 parser->Parse(packet, static_cast<int>(length), &header);
158
159 {
160 CriticalSectionScoped lock(crit_.get());
161 if (rtp_timestamp_delta_ == 0) {
162 rtp_timestamp_delta_ =
163 header.timestamp - first_send_frame_->timestamp();
164 first_send_frame_ = NULL;
165 }
166 send_times_[header.timestamp - rtp_timestamp_delta_] =
167 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
168 }
169
170 return transport_->SendRtp(packet, length);
171 }
172
SendRtcp(const uint8_t * packet,size_t length)173 virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE {
174 return transport_->SendRtcp(packet, length);
175 }
176
RenderFrame(const I420VideoFrame & video_frame,int time_to_render_ms)177 virtual void RenderFrame(const I420VideoFrame& video_frame,
178 int time_to_render_ms) OVERRIDE {
179 int64_t render_time_ms =
180 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
181 uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
182
183 CriticalSectionScoped lock(crit_.get());
184 while (frames_.front()->timestamp() < send_timestamp) {
185 AddFrameComparison(
186 frames_.front(), &last_rendered_frame_, true, render_time_ms);
187 frame_pool_.push_back(frames_.front());
188 frames_.pop_front();
189 }
190
191 I420VideoFrame* reference_frame = frames_.front();
192 frames_.pop_front();
193 assert(reference_frame != NULL);
194 EXPECT_EQ(reference_frame->timestamp(), send_timestamp);
195 assert(reference_frame->timestamp() == send_timestamp);
196
197 AddFrameComparison(reference_frame, &video_frame, false, render_time_ms);
198 frame_pool_.push_back(reference_frame);
199
200 last_rendered_frame_.CopyFrame(video_frame);
201 }
202
Wait()203 void Wait() { done_->Wait(120 * 1000); }
204
205 VideoSendStreamInput* input_;
206 Transport* transport_;
207 PacketReceiver* receiver_;
208
209 private:
210 struct FrameComparison {
FrameComparisonwebrtc::VideoAnalyzer::FrameComparison211 FrameComparison(const I420VideoFrame* reference,
212 const I420VideoFrame* render,
213 bool dropped,
214 int64_t send_time_ms,
215 int64_t recv_time_ms,
216 int64_t render_time_ms)
217 : dropped(dropped),
218 send_time_ms(send_time_ms),
219 recv_time_ms(recv_time_ms),
220 render_time_ms(render_time_ms) {
221 this->reference.CopyFrame(*reference);
222 this->render.CopyFrame(*render);
223 }
224
FrameComparisonwebrtc::VideoAnalyzer::FrameComparison225 FrameComparison(const FrameComparison& compare)
226 : dropped(compare.dropped),
227 send_time_ms(compare.send_time_ms),
228 recv_time_ms(compare.recv_time_ms),
229 render_time_ms(compare.render_time_ms) {
230 this->reference.CopyFrame(compare.reference);
231 this->render.CopyFrame(compare.render);
232 }
233
~FrameComparisonwebrtc::VideoAnalyzer::FrameComparison234 ~FrameComparison() {}
235
236 I420VideoFrame reference;
237 I420VideoFrame render;
238 bool dropped;
239 int64_t send_time_ms;
240 int64_t recv_time_ms;
241 int64_t render_time_ms;
242 };
243
AddFrameComparison(const I420VideoFrame * reference,const I420VideoFrame * render,bool dropped,int64_t render_time_ms)244 void AddFrameComparison(const I420VideoFrame* reference,
245 const I420VideoFrame* render,
246 bool dropped,
247 int64_t render_time_ms)
248 EXCLUSIVE_LOCKS_REQUIRED(crit_) {
249 int64_t send_time_ms = send_times_[reference->timestamp()];
250 send_times_.erase(reference->timestamp());
251 int64_t recv_time_ms = recv_times_[reference->timestamp()];
252 recv_times_.erase(reference->timestamp());
253
254 CriticalSectionScoped crit(comparison_lock_.get());
255 comparisons_.push_back(FrameComparison(reference,
256 render,
257 dropped,
258 send_time_ms,
259 recv_time_ms,
260 render_time_ms));
261 }
262
FrameComparisonThread(void * obj)263 static bool FrameComparisonThread(void* obj) {
264 return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
265 }
266
CompareFrames()267 bool CompareFrames() {
268 assert(frames_left_ > 0);
269
270 I420VideoFrame reference;
271 I420VideoFrame render;
272 bool dropped;
273 int64_t send_time_ms;
274 int64_t recv_time_ms;
275 int64_t render_time_ms;
276
277 SleepMs(10);
278
279 while (true) {
280 {
281 CriticalSectionScoped crit(comparison_lock_.get());
282 if (comparisons_.empty())
283 return true;
284 reference.SwapFrame(&comparisons_.front().reference);
285 render.SwapFrame(&comparisons_.front().render);
286 dropped = comparisons_.front().dropped;
287 send_time_ms = comparisons_.front().send_time_ms;
288 recv_time_ms = comparisons_.front().recv_time_ms;
289 render_time_ms = comparisons_.front().render_time_ms;
290 comparisons_.pop_front();
291 }
292
293 PerformFrameComparison(&reference,
294 &render,
295 dropped,
296 send_time_ms,
297 recv_time_ms,
298 render_time_ms);
299
300 if (--frames_left_ == 0) {
301 PrintResult("psnr", psnr_, " dB");
302 PrintResult("ssim", ssim_, "");
303 PrintResult("sender_time", sender_time_, " ms");
304 printf(
305 "RESULT dropped_frames: %s = %d\n", test_label_, dropped_frames_);
306 PrintResult("receiver_time", receiver_time_, " ms");
307 PrintResult("total_delay_incl_network", end_to_end_, " ms");
308 PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
309 EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
310 EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
311 done_->Set();
312
313 return false;
314 }
315 }
316 }
317
PerformFrameComparison(const I420VideoFrame * reference,const I420VideoFrame * render,bool dropped,int64_t send_time_ms,int64_t recv_time_ms,int64_t render_time_ms)318 void PerformFrameComparison(const I420VideoFrame* reference,
319 const I420VideoFrame* render,
320 bool dropped,
321 int64_t send_time_ms,
322 int64_t recv_time_ms,
323 int64_t render_time_ms) {
324 psnr_.AddSample(I420PSNR(reference, render));
325 ssim_.AddSample(I420SSIM(reference, render));
326 if (dropped) {
327 ++dropped_frames_;
328 return;
329 }
330 if (last_render_time_ != 0)
331 rendered_delta_.AddSample(render_time_ms - last_render_time_);
332 last_render_time_ = render_time_ms;
333
334 int64_t input_time_ms = reference->render_time_ms();
335 sender_time_.AddSample(send_time_ms - input_time_ms);
336 receiver_time_.AddSample(render_time_ms - recv_time_ms);
337 end_to_end_.AddSample(render_time_ms - input_time_ms);
338 }
339
PrintResult(const char * result_type,test::Statistics stats,const char * unit)340 void PrintResult(const char* result_type,
341 test::Statistics stats,
342 const char* unit) {
343 printf("RESULT %s: %s = {%f, %f}%s\n",
344 result_type,
345 test_label_,
346 stats.Mean(),
347 stats.StandardDeviation(),
348 unit);
349 }
350
351 const char* const test_label_;
352 test::Statistics sender_time_;
353 test::Statistics receiver_time_;
354 test::Statistics psnr_;
355 test::Statistics ssim_;
356 test::Statistics end_to_end_;
357 test::Statistics rendered_delta_;
358 int frames_left_;
359 int dropped_frames_;
360 int64_t last_render_time_;
361 uint32_t rtp_timestamp_delta_;
362
363 const scoped_ptr<CriticalSectionWrapper> crit_;
364 std::deque<I420VideoFrame*> frames_ GUARDED_BY(crit_);
365 std::deque<I420VideoFrame*> frame_pool_ GUARDED_BY(crit_);
366 I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
367 std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
368 std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
369 I420VideoFrame* first_send_frame_ GUARDED_BY(crit_);
370 double avg_psnr_threshold_ GUARDED_BY(crit_);
371 double avg_ssim_threshold_ GUARDED_BY(crit_);
372
373 const scoped_ptr<CriticalSectionWrapper> comparison_lock_;
374 const scoped_ptr<ThreadWrapper> comparison_thread_;
375 std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
376 const scoped_ptr<EventWrapper> done_;
377 };
378
TEST_P(FullStackTest,NoPacketLoss)379 TEST_P(FullStackTest, NoPacketLoss) {
380 static const uint32_t kReceiverLocalSsrc = 0x123456;
381 FullStackTestParams params = GetParam();
382
383 test::DirectTransport transport;
384 VideoAnalyzer analyzer(NULL,
385 &transport,
386 params.test_label,
387 params.avg_psnr_threshold,
388 params.avg_ssim_threshold,
389 kFullStackTestDurationSecs * params.clip.fps);
390
391 Call::Config call_config(&analyzer);
392
393 scoped_ptr<Call> call(Call::Create(call_config));
394 analyzer.SetReceiver(call->Receiver());
395 transport.SetReceiver(&analyzer);
396
397 VideoSendStream::Config send_config = call->GetDefaultSendConfig();
398 send_config.rtp.ssrcs.push_back(kSendSsrc);
399
400 scoped_ptr<VP8Encoder> encoder(VP8Encoder::Create());
401 send_config.encoder_settings.encoder = encoder.get();
402 send_config.encoder_settings.payload_name = "VP8";
403 send_config.encoder_settings.payload_type = 124;
404 std::vector<VideoStream> video_streams = test::CreateVideoStreams(1);
405 VideoStream* stream = &video_streams[0];
406 stream->width = params.clip.width;
407 stream->height = params.clip.height;
408 stream->min_bitrate_bps = stream->target_bitrate_bps =
409 stream->max_bitrate_bps = params.bitrate * 1000;
410 stream->max_framerate = params.clip.fps;
411
412 VideoSendStream* send_stream =
413 call->CreateVideoSendStream(send_config, video_streams, NULL);
414 analyzer.input_ = send_stream->Input();
415
416 scoped_ptr<test::FrameGeneratorCapturer> file_capturer(
417 test::FrameGeneratorCapturer::CreateFromYuvFile(
418 &analyzer,
419 test::ResourcePath(params.clip.name, "yuv").c_str(),
420 params.clip.width,
421 params.clip.height,
422 params.clip.fps,
423 Clock::GetRealTimeClock()));
424 ASSERT_TRUE(file_capturer.get() != NULL)
425 << "Could not create capturer for " << params.clip.name
426 << ".yuv. Is this resource file present?";
427
428 VideoReceiveStream::Config receive_config = call->GetDefaultReceiveConfig();
429 VideoCodec codec =
430 test::CreateDecoderVideoCodec(send_config.encoder_settings);
431 receive_config.codecs.push_back(codec);
432 receive_config.rtp.remote_ssrc = send_config.rtp.ssrcs[0];
433 receive_config.rtp.local_ssrc = kReceiverLocalSsrc;
434 receive_config.renderer = &analyzer;
435
436 VideoReceiveStream* receive_stream =
437 call->CreateVideoReceiveStream(receive_config);
438
439 receive_stream->Start();
440 send_stream->Start();
441 file_capturer->Start();
442
443 analyzer.Wait();
444
445 file_capturer->Stop();
446 send_stream->Stop();
447 receive_stream->Stop();
448
449 call->DestroyVideoReceiveStream(receive_stream);
450 call->DestroyVideoSendStream(send_stream);
451
452 transport.StopSending();
453 }
454
455 INSTANTIATE_TEST_CASE_P(FullStack,
456 FullStackTest,
457 ::testing::Values(paris_qcif, foreman_cif));
458
459 } // namespace webrtc
460