1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 // Implementation of Media Optimization Test
12 // testing is done via the VCM module, no specific Media opt functionality.
13
14 #include "webrtc/modules/video_coding/main/test/media_opt_test.h"
15
16 #include <stdio.h>
17 #include <string.h>
18 #include <time.h>
19 #include <vector>
20
21 #include "webrtc/modules/rtp_rtcp/interface/rtp_receiver.h"
22 #include "webrtc/modules/video_coding/main/interface/video_coding.h"
23 #include "webrtc/modules/video_coding/main/test/test_macros.h"
24 #include "webrtc/modules/video_coding/main/test/test_util.h"
25 #include "webrtc/test/testsupport/fileutils.h"
26 #include "webrtc/test/testsupport/metrics/video_metrics.h"
27
28 using namespace webrtc;
29
RunTest(int testNum,CmdArgs & args)30 int MediaOptTest::RunTest(int testNum, CmdArgs& args)
31 {
32 Trace::CreateTrace();
33 Trace::SetTraceFile((test::OutputPath() + "mediaOptTestTrace.txt").c_str());
34 Trace::set_level_filter(webrtc::kTraceAll);
35 VideoCodingModule* vcm = VideoCodingModule::Create();
36 Clock* clock = Clock::GetRealTimeClock();
37 MediaOptTest* mot = new MediaOptTest(vcm, clock);
38 if (testNum == 0)
39 { // regular
40 mot->Setup(0, args);
41 mot->GeneralSetup();
42 mot->Perform();
43 mot->Print(1);// print to screen
44 mot->TearDown();
45 }
46 if (testNum == 1)
47 { // release test
48 mot->Setup(0, args);
49 mot->RTTest();
50 }
51 if (testNum == 2)
52 { // release test, running from script
53 mot->Setup(1, args);
54 mot->GeneralSetup();
55 mot->Perform();
56 mot->Print(1);// print to screen
57 mot->TearDown();
58 }
59
60 VideoCodingModule::Destroy(vcm);
61 delete mot;
62 Trace::ReturnTrace();
63 return 0;
64
65 }
66
67
MediaOptTest(VideoCodingModule * vcm,Clock * clock)68 MediaOptTest::MediaOptTest(VideoCodingModule* vcm, Clock* clock)
69 : _vcm(vcm),
70 _rtp(NULL),
71 _outgoingTransport(NULL),
72 _dataCallback(NULL),
73 _clock(clock),
74 _width(0),
75 _height(0),
76 _lengthSourceFrame(0),
77 _timeStamp(0),
78 _frameRate(30.0f),
79 _nackEnabled(false),
80 _fecEnabled(false),
81 _rttMS(0),
82 _bitRate(300.0f),
83 _lossRate(0.0f),
84 _renderDelayMs(0),
85 _frameCnt(0),
86 _sumEncBytes(0),
87 _numFramesDropped(0),
88 _numberOfCores(4) {
89 }
90
~MediaOptTest()91 MediaOptTest::~MediaOptTest() {
92 delete _rtp;
93 }
94
Setup(int testType,CmdArgs & args)95 void MediaOptTest::Setup(int testType, CmdArgs& args) {
96 /*TEST USER SETTINGS*/
97 // test parameters
98 _inname = args.inputFile;
99 if (args.outputFile == "")
100 _outname = test::OutputPath() + "MOTest_out.vp8";
101 else
102 _outname = args.outputFile;
103 // actual source after frame dropping
104 _actualSourcename = test::OutputPath() + "MOTestSource.yuv";
105 _codecName = args.codecName;
106 _sendCodecType = args.codecType;
107 _width = args.width;
108 _height = args.height;
109 _frameRate = args.frameRate;
110 _bitRate = args.bitRate;
111 _numberOfCores = 4;
112
113 // error resilience
114 _nackEnabled = false;
115 _fecEnabled = true;
116 _nackFecEnabled = false;
117
118 _rttMS = 100;
119 _lossRate = 0.00*255; // no packet loss
120
121 _testType = testType;
122
123 //For multiple runs with script
124 if (_testType == 1)
125 {
126 float rateTest,lossTest;
127 int numRuns;
128 _fpinp = fopen("dat_inp","rb");
129 _fpout = fopen("test_runs/dat_out","ab");
130 _fpout2 = fopen("test_runs/dat_out2","ab");
131 TEST(fscanf(_fpinp,"%f %f %d \n",&rateTest,&lossTest,&numRuns) > 0);
132 _bitRate = rateTest;
133 _lossRate = lossTest;
134 _testNum = 0;
135
136 // for bit rates: 500, 1000, 2000, 3000,4000
137 // for loss rates: 0, 1, 3, 5, 10%
138 _numParRuns = 25;
139
140 _testNum = numRuns + 1;
141 if (rateTest == 0.0) _lossRate = 0.0;
142 else
143 {
144 if (rateTest == 4000) //final bit rate
145 {
146 if (lossTest == 0.1*255) _lossRate = 0.0; //start at 1%
147 else
148 if (lossTest == 0.05*255) _lossRate = 0.1*255; //final loss rate
149 else
150 if (lossTest == 0.0) _lossRate = 0.01*255;
151 else _lossRate = lossTest + 0.02*255;
152 }
153 }
154
155 if (rateTest == 0.0 || rateTest == 4000) _bitRate = 500; //starting bit rate
156 else
157 if (rateTest == 500) _bitRate = 1000;
158 else _bitRate = rateTest + 1000;
159 }
160 //
161
162 _renderDelayMs = 0;
163 /* test settings end*/
164
165 _lengthSourceFrame = 3*_width*_height/2;
166 _log.open((test::OutputPath() + "VCM_MediaOptLog.txt").c_str(),
167 std::fstream::out | std::fstream::app);
168 }
169
170 void
GeneralSetup()171 MediaOptTest::GeneralSetup()
172 {
173 uint32_t minPlayoutDelayMs = 0;
174
175 if ((_sourceFile = fopen(_inname.c_str(), "rb")) == NULL)
176 {
177 printf("Cannot read file %s.\n", _inname.c_str());
178 exit(1);
179 }
180
181 if ((_decodedFile = fopen(_outname.c_str(), "wb")) == NULL)
182 {
183 printf("Cannot read file %s.\n", _outname.c_str());
184 exit(1);
185 }
186
187 if ((_actualSourceFile = fopen(_actualSourcename.c_str(), "wb")) == NULL)
188 {
189 printf("Cannot read file %s.\n", _actualSourcename.c_str());
190 exit(1);
191 }
192 if (_vcm->InitializeReceiver() < 0)
193 {
194 exit(1);
195 }
196 if (_vcm->InitializeSender())
197 {
198 exit(1);
199 }
200 _outgoingTransport = new RTPSendCompleteCallback(_clock);
201 _dataCallback = new RtpDataCallback(_vcm);
202
203 RtpRtcp::Configuration configuration;
204 configuration.id = 1;
205 configuration.audio = false;
206 configuration.outgoing_transport = _outgoingTransport;
207 _rtp = RtpRtcp::CreateRtpRtcp(configuration);
208
209 _outgoingTransport->SetRtpModule(_rtp);
210
211 // Registering codecs for the RTP module
212
213 // Register receive and send payload
214 VideoCodec video_codec;
215 strncpy(video_codec.plName, "VP8", 32);
216 video_codec.plType = VCM_VP8_PAYLOAD_TYPE;
217 rtp_receiver_->RegisterReceivePayload(video_codec.plName,
218 video_codec.plType,
219 90000,
220 0,
221 video_codec.maxBitrate);
222 _rtp->RegisterSendPayload(video_codec);
223
224 strncpy(video_codec.plName, "ULPFEC", 32);
225 video_codec.plType = VCM_ULPFEC_PAYLOAD_TYPE;
226 rtp_receiver_->RegisterReceivePayload(video_codec.plName,
227 video_codec.plType,
228 90000,
229 0,
230 video_codec.maxBitrate);
231 _rtp->RegisterSendPayload(video_codec);
232
233 strncpy(video_codec.plName, "RED", 32);
234 video_codec.plType = VCM_RED_PAYLOAD_TYPE;
235 rtp_receiver_->RegisterReceivePayload(video_codec.plName,
236 video_codec.plType,
237 90000,
238 0,
239 video_codec.maxBitrate);
240 _rtp->RegisterSendPayload(video_codec);
241
242 if (_nackFecEnabled == 1)
243 _rtp->SetGenericFECStatus(_nackFecEnabled, VCM_RED_PAYLOAD_TYPE,
244 VCM_ULPFEC_PAYLOAD_TYPE);
245 else
246 _rtp->SetGenericFECStatus(_fecEnabled, VCM_RED_PAYLOAD_TYPE,
247 VCM_ULPFEC_PAYLOAD_TYPE);
248
249 // VCM: Registering codecs
250 VideoCodec sendCodec;
251 _vcm->InitializeSender();
252 _vcm->InitializeReceiver();
253 int32_t numberOfCodecs = _vcm->NumberOfCodecs();
254 if (numberOfCodecs < 1)
255 {
256 exit(1);
257 }
258
259 if (_vcm->Codec(_sendCodecType, &sendCodec) != 0)
260 {
261 printf("Unknown codec\n");
262 exit(1);
263 }
264 // register codec
265 sendCodec.startBitrate = (int) _bitRate;
266 sendCodec.height = _height;
267 sendCodec.width = _width;
268 sendCodec.maxFramerate = (uint8_t)_frameRate;
269 _vcm->RegisterSendCodec(&sendCodec, _numberOfCores, 1440);
270 _vcm->RegisterReceiveCodec(&sendCodec, _numberOfCores); // same settings for encode and decode
271
272 _vcm->SetRenderDelay(_renderDelayMs);
273 _vcm->SetMinimumPlayoutDelay(minPlayoutDelayMs);
274 }
275 // The following test shall be conducted under release tests
276
277
278
279 int32_t
Perform()280 MediaOptTest::Perform()
281 {
282 VCMDecodeCompleteCallback receiveCallback(_decodedFile);
283
284 VCMRTPEncodeCompleteCallback* encodeCompleteCallback = new VCMRTPEncodeCompleteCallback(_rtp);
285 _vcm->RegisterTransportCallback(encodeCompleteCallback);
286 encodeCompleteCallback->SetCodecType(ConvertCodecType(_codecName.c_str()));
287 encodeCompleteCallback->SetFrameDimensions(_width, _height);
288
289 // callback settings
290 VideoProtectionCallback protectionCallback;
291 protectionCallback.RegisterRtpModule(_rtp);
292 _vcm->RegisterProtectionCallback(&protectionCallback);
293
294 // set error resilience / test parameters:
295 _outgoingTransport->SetLossPct(_lossRate);
296 if (_nackFecEnabled == 1) {
297 _vcm->SetVideoProtection(kProtectionNackFEC, _nackFecEnabled);
298 } else {
299 _vcm->SetVideoProtection(kProtectionNack, _nackEnabled);
300 _vcm->SetVideoProtection(kProtectionFEC, _fecEnabled);
301 }
302
303 // START TEST
304 I420VideoFrame sourceFrame;
305 uint8_t* tmpBuffer = new uint8_t[_lengthSourceFrame];
306 _vcm->SetChannelParameters(static_cast<uint32_t>(1000 * _bitRate),
307 (uint8_t)_lossRate, _rttMS);
308 _vcm->RegisterReceiveCallback(&receiveCallback);
309
310 _frameCnt = 0;
311 _sumEncBytes = 0.0;
312 _numFramesDropped = 0;
313 int half_width = (_width + 1) / 2;
314 int half_height = (_height + 1) / 2;
315 int size_y = _width * _height;
316 int size_uv = half_width * half_height;
317
318 while (feof(_sourceFile)== 0)
319 {
320 TEST(fread(tmpBuffer, 1, _lengthSourceFrame, _sourceFile) > 0);
321 _frameCnt++;
322 sourceFrame.CreateFrame(size_y, tmpBuffer,
323 size_uv, tmpBuffer + size_y,
324 size_uv, tmpBuffer + size_y + size_uv,
325 _width, _height,
326 _width, half_width, half_width);
327 _timeStamp += (uint32_t)(9e4 / static_cast<float>(_frameRate));
328 sourceFrame.set_timestamp(_timeStamp);
329 TEST(_vcm->AddVideoFrame(sourceFrame) == VCM_OK);
330 // inform RTP Module of error resilience features
331 //_rtp->SetFECCodeRate(protectionCallback.FECKeyRate(),protectionCallback.FECDeltaRate());
332 //_rtp->SetNACKStatus(protectionCallback.NACKMethod());
333
334 int32_t ret = _vcm->Decode();
335 if (ret < 0 )
336 {
337 TEST(ret == 0);
338 printf ("Decode error in frame # %d",_frameCnt);
339 }
340
341 float encBytes = encodeCompleteCallback->EncodedBytes();
342 if (encBytes == 0)
343 {
344 _numFramesDropped += 1;
345 //printf("frame #%d dropped \n", _frameCnt );
346 }
347 else
348 {
349 // write frame to file
350 if (PrintI420VideoFrame(sourceFrame, _actualSourceFile) < 0) {
351 return -1;
352 }
353 }
354
355 _sumEncBytes += encBytes;
356 }
357
358 //END TEST
359 delete encodeCompleteCallback;
360 delete tmpBuffer;
361
362 return 0;
363
364 }
365
366 void
RTTest()367 MediaOptTest::RTTest()
368 {
369 // will only calculate PSNR - not create output files for all
370 // SET UP
371 // Set bit rates
372 const float bitRateVec[] = {500, 1000, 2000,3000, 4000};
373 //const float bitRateVec[] = {1000};
374 // Set Packet loss values ([0,255])
375 const double lossPctVec[] = {0.0*255, 0.0*255, 0.01*255, 0.01*255, 0.03*255, 0.03*255, 0.05*255, 0.05*255, 0.1*255, 0.1*255};
376 const bool nackEnabledVec[] = {false , false, false, false, false, false, false, false , false, false};
377 const bool fecEnabledVec[] = {false , true, false, true , false, true , false, true , false, true};
378 // fec and nack are set according to the packet loss values
379
380 const float nBitrates = sizeof(bitRateVec)/sizeof(*bitRateVec);
381 const float nlossPct = sizeof(lossPctVec)/sizeof(*lossPctVec);
382
383 std::vector<const VideoSource*> sources;
384 std::vector<const VideoSource*>::iterator it;
385
386 sources.push_back(new const VideoSource(_inname, _width, _height));
387 int numOfSrc = 1;
388
389 // constant settings (valid for entire run time)
390 _rttMS = 20;
391 _renderDelayMs = 0;
392
393 // same out name for all
394 _outname = test::OutputPath() + "RTMOTest_out.yuv";
395 // actual source after frame dropping
396 _actualSourcename = test::OutputPath() + "RTMOTestSource.yuv";
397
398 _codecName = "VP8"; // for now just this one - later iterate over all codec types
399 _log.open((test::OutputPath() + "/VCM_RTMediaOptLog.txt").c_str(),
400 std::fstream::out | std::fstream::app);
401 _outputRes=fopen((test::OutputPath() + "VCM_MediaOptResults.txt").c_str(),
402 "ab");
403
404 //char filename[128];
405 /* test settings end*/
406
407 // START TEST
408 // iterate over test sequences
409 printf("\n****START TEST OVER ALL RUNS ****\n");
410 int runCnt = 0;
411 for (it = sources.begin() ; it < sources.end(); it++)
412 {
413
414 // test set up
415 _inname = (*it)->GetFileName();
416 _width = (*it)->GetWidth();
417 _height = (*it)->GetHeight();
418 _lengthSourceFrame = 3*_width*_height/2;
419 _frameRate = (*it)->GetFrameRate();
420 //GeneralSetup();
421
422
423 // iterate over all bit rates
424 for (int i = 0; i < nBitrates; i++)
425 {
426 _bitRate = static_cast<float>(bitRateVec[i]);
427 // iterate over all packet loss values
428 for (int j = 0; j < nlossPct; j++)
429 {
430 _lossRate = static_cast<float>(lossPctVec[j]);
431 _nackEnabled = static_cast<bool>(nackEnabledVec[j]);
432 _fecEnabled = static_cast<bool>(fecEnabledVec[j]);
433
434 runCnt++;
435 printf("run #%d out of %d \n", runCnt,(int)(nlossPct*nBitrates*numOfSrc));
436
437 //printf("**FOR RUN: **%d %d %d %d \n",_nackEnabled,_fecEnabled,int(lossPctVec[j]),int(_bitRate));
438
439 /*
440 int ch = sprintf(filename,"../test_mediaOpt/RTMOTest_%d_%d_%d_%d.yuv",_nackEnabled,_fecEnabled,int(lossPctVec[j]),int(_bitRate));
441 _outname = filename;
442
443 printf("**FOR RUN: **%d %d %d %d \n",_nackEnabled,_fecEnabled,int(lossPctVec[j]),int(_bitRate));
444 */
445 GeneralSetup();
446 Perform();
447 Print(1);
448 TearDown();
449
450 printf("\n");
451 //printf("**DONE WITH RUN: **%d %d %f %d \n",_nackEnabled,_fecEnabled,lossPctVec[j],int(_bitRate));
452 //
453
454 }// end of packet loss loop
455 }// end of bit rate loop
456 delete *it;
457 }// end of video sequence loop
458 // at end of sequence
459 fclose(_outputRes);
460 printf("\nVCM Media Optimization Test: \n\n%i tests completed\n", vcmMacrosTests);
461 if (vcmMacrosErrors > 0)
462 {
463 printf("%i FAILED\n\n", vcmMacrosErrors);
464 }
465 else
466 {
467 printf("ALL PASSED\n\n");
468 }
469 }
470
471
472 void
Print(int mode)473 MediaOptTest::Print(int mode)
474 {
475 double ActualBitRate = 8.0 *( _sumEncBytes / (_frameCnt / _frameRate));
476 double actualBitRate = ActualBitRate / 1000.0;
477 webrtc::test::QualityMetricsResult psnr;
478 I420PSNRFromFiles(_actualSourcename.c_str(), _outname.c_str(), _width,
479 _height, &psnr);
480
481 (_log) << "VCM: Media Optimization Test Cycle Completed!" << std::endl;
482 (_log) << "Input file: " << _inname << std::endl;
483 (_log) << "Output file:" << _outname << std::endl;
484 ( _log) << "Actual bitrate: " << actualBitRate<< " kbps\tTarget: " << _bitRate << " kbps" << std::endl;
485 (_log) << "Error Reslience: NACK:" << _nackEnabled << "; FEC: " << _fecEnabled << std::endl;
486 (_log) << "Packet Loss applied= %f " << _lossRate << std::endl;
487 (_log) << _numFramesDropped << " FRames were dropped" << std::endl;
488 ( _log) << "PSNR: " << psnr.average << std::endl;
489 (_log) << std::endl;
490
491 if (_testType == 2)
492 {
493 fprintf(_outputRes,"************\n");
494 fprintf(_outputRes,"\n\n\n");
495 fprintf(_outputRes,"Actual bitrate: %f kbps\n", actualBitRate);
496 fprintf(_outputRes,"Target bitrate: %f kbps\n", _bitRate);
497 fprintf(_outputRes,"NACK: %s ",(_nackEnabled)?"true":"false");
498 fprintf(_outputRes,"FEC: %s \n ",(_fecEnabled)?"true":"false");
499 fprintf(_outputRes,"Packet loss applied = %f\n", _lossRate);
500 fprintf(_outputRes,"%d frames were dropped, and total number of frames processed %d \n",_numFramesDropped,_frameCnt);
501 fprintf(_outputRes,"PSNR: %f \n", psnr.average);
502 fprintf(_outputRes,"************\n");
503 }
504
505
506 //
507 if (_testType == 1)
508 {
509 fprintf(_fpout,"************\n");
510 fprintf(_fpout,"\n\n\n");
511 fprintf(_fpout,"Actual bitrate: %f kbps\n", actualBitRate);
512 fprintf(_fpout,"Target bitrate: %f kbps\n", _bitRate);
513 fprintf(_fpout,"NACK: %s ",(_nackEnabled)?"true":"false");
514 fprintf(_fpout,"FEC: %s \n ",(_fecEnabled)?"true":"false");
515 fprintf(_fpout,"Packet loss applied = %f\n", _lossRate);
516 fprintf(_fpout,"%d frames were dropped, and total number of frames processed %d \n",_numFramesDropped,_frameCnt);
517 fprintf(_fpout,"PSNR: %f \n", psnr.average);
518 fprintf(_fpout,"************\n");
519
520 int testNum1 = _testNum/(_numParRuns +1) + 1;
521 int testNum2 = _testNum%_numParRuns;
522 if (testNum2 == 0) testNum2 = _numParRuns;
523 fprintf(_fpout2,"%d %d %f %f %f %f \n",testNum1,testNum2,_bitRate,actualBitRate,_lossRate,psnr.average);
524 fclose(_fpinp);
525 _fpinp = fopen("dat_inp","wb");
526 fprintf(_fpinp,"%f %f %d \n",_bitRate,_lossRate,_testNum);
527 }
528 //
529
530
531 if (mode == 1)
532 {
533 // print to screen
534 printf("\n\n\n");
535 printf("Actual bitrate: %f kbps\n", actualBitRate);
536 printf("Target bitrate: %f kbps\n", _bitRate);
537 printf("NACK: %s ",(_nackEnabled)?"true":"false");
538 printf("FEC: %s \n",(_fecEnabled)?"true":"false");
539 printf("Packet loss applied = %f\n", _lossRate);
540 printf("%d frames were dropped, and total number of frames processed %d \n",_numFramesDropped,_frameCnt);
541 printf("PSNR: %f \n", psnr.average);
542 }
543 TEST(psnr.average > 10); // low becuase of possible frame dropping (need to verify that OK for all packet loss values/ rates)
544 }
545
TearDown()546 void MediaOptTest::TearDown() {
547 delete _rtp;
548 _rtp = NULL;
549 delete _outgoingTransport;
550 _outgoingTransport = NULL;
551 delete _dataCallback;
552 _dataCallback = NULL;
553 _log.close();
554 fclose(_sourceFile);
555 fclose(_decodedFile);
556 fclose(_actualSourceFile);
557 }
558