1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <vector>
12
13 #include "testing/gtest/include/gtest/gtest.h"
14 #include "webrtc/base/scoped_ptr.h"
15 #include "webrtc/common.h"
16 #include "webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h"
17 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
18 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
19 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
20 #include "webrtc/modules/video_coding/include/mock/mock_vcm_callbacks.h"
21 #include "webrtc/modules/video_coding/include/video_coding.h"
22 #include "webrtc/modules/video_coding/video_coding_impl.h"
23 #include "webrtc/modules/video_coding/test/test_util.h"
24 #include "webrtc/system_wrappers/include/clock.h"
25 #include "webrtc/test/frame_generator.h"
26 #include "webrtc/test/testsupport/fileutils.h"
27
28 using ::testing::_;
29 using ::testing::AllOf;
30 using ::testing::ElementsAre;
31 using ::testing::ElementsAreArray;
32 using ::testing::Field;
33 using ::testing::NiceMock;
34 using ::testing::Pointee;
35 using ::testing::Return;
36 using ::testing::FloatEq;
37 using std::vector;
38 using webrtc::test::FrameGenerator;
39
40 namespace webrtc {
41 namespace vcm {
42 namespace {
43 enum { kMaxNumberOfTemporalLayers = 3 };
44
45 struct Vp8StreamInfo {
46 float framerate_fps[kMaxNumberOfTemporalLayers];
47 int bitrate_kbps[kMaxNumberOfTemporalLayers];
48 };
49
50 MATCHER_P(MatchesVp8StreamInfo, expected, "") {
51 bool res = true;
52 for (int tl = 0; tl < kMaxNumberOfTemporalLayers; ++tl) {
53 if (fabs(expected.framerate_fps[tl] - arg.framerate_fps[tl]) > 0.5) {
54 *result_listener << " framerate_fps[" << tl
55 << "] = " << arg.framerate_fps[tl] << " (expected "
56 << expected.framerate_fps[tl] << ") ";
57 res = false;
58 }
59 if (abs(expected.bitrate_kbps[tl] - arg.bitrate_kbps[tl]) > 10) {
60 *result_listener << " bitrate_kbps[" << tl
61 << "] = " << arg.bitrate_kbps[tl] << " (expected "
62 << expected.bitrate_kbps[tl] << ") ";
63 res = false;
64 }
65 }
66 return res;
67 }
68
69 class EmptyFrameGenerator : public FrameGenerator {
70 public:
EmptyFrameGenerator(int width,int height)71 EmptyFrameGenerator(int width, int height) : width_(width), height_(height) {}
NextFrame()72 VideoFrame* NextFrame() override {
73 frame_.reset(new VideoFrame());
74 frame_->CreateEmptyFrame(width_, height_, width_, (width_ + 1) / 2,
75 (width_ + 1) / 2);
76 return frame_.get();
77 }
78
79 private:
80 const int width_;
81 const int height_;
82 rtc::scoped_ptr<VideoFrame> frame_;
83 };
84
85 class PacketizationCallback : public VCMPacketizationCallback {
86 public:
PacketizationCallback(Clock * clock)87 explicit PacketizationCallback(Clock* clock)
88 : clock_(clock), start_time_ms_(clock_->TimeInMilliseconds()) {}
89
~PacketizationCallback()90 virtual ~PacketizationCallback() {}
91
SendData(uint8_t payload_type,const EncodedImage & encoded_image,const RTPFragmentationHeader & fragmentation_header,const RTPVideoHeader * rtp_video_header)92 int32_t SendData(uint8_t payload_type,
93 const EncodedImage& encoded_image,
94 const RTPFragmentationHeader& fragmentation_header,
95 const RTPVideoHeader* rtp_video_header) override {
96 assert(rtp_video_header);
97 frame_data_.push_back(FrameData(encoded_image._length, *rtp_video_header));
98 return 0;
99 }
100
Reset()101 void Reset() {
102 frame_data_.clear();
103 start_time_ms_ = clock_->TimeInMilliseconds();
104 }
105
FramerateFpsWithinTemporalLayer(int temporal_layer)106 float FramerateFpsWithinTemporalLayer(int temporal_layer) {
107 return CountFramesWithinTemporalLayer(temporal_layer) *
108 (1000.0 / interval_ms());
109 }
110
BitrateKbpsWithinTemporalLayer(int temporal_layer)111 float BitrateKbpsWithinTemporalLayer(int temporal_layer) {
112 return SumPayloadBytesWithinTemporalLayer(temporal_layer) * 8.0 /
113 interval_ms();
114 }
115
CalculateVp8StreamInfo()116 Vp8StreamInfo CalculateVp8StreamInfo() {
117 Vp8StreamInfo info;
118 for (int tl = 0; tl < 3; ++tl) {
119 info.framerate_fps[tl] = FramerateFpsWithinTemporalLayer(tl);
120 info.bitrate_kbps[tl] = BitrateKbpsWithinTemporalLayer(tl);
121 }
122 return info;
123 }
124
125 private:
126 struct FrameData {
FrameDatawebrtc::vcm::__anon85d6b7670111::PacketizationCallback::FrameData127 FrameData() {}
128
FrameDatawebrtc::vcm::__anon85d6b7670111::PacketizationCallback::FrameData129 FrameData(size_t payload_size, const RTPVideoHeader& rtp_video_header)
130 : payload_size(payload_size), rtp_video_header(rtp_video_header) {}
131
132 size_t payload_size;
133 RTPVideoHeader rtp_video_header;
134 };
135
interval_ms()136 int64_t interval_ms() {
137 int64_t diff = (clock_->TimeInMilliseconds() - start_time_ms_);
138 EXPECT_GT(diff, 0);
139 return diff;
140 }
141
CountFramesWithinTemporalLayer(int temporal_layer)142 int CountFramesWithinTemporalLayer(int temporal_layer) {
143 int frames = 0;
144 for (size_t i = 0; i < frame_data_.size(); ++i) {
145 EXPECT_EQ(kRtpVideoVp8, frame_data_[i].rtp_video_header.codec);
146 const uint8_t temporal_idx =
147 frame_data_[i].rtp_video_header.codecHeader.VP8.temporalIdx;
148 if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
149 frames++;
150 }
151 return frames;
152 }
153
SumPayloadBytesWithinTemporalLayer(int temporal_layer)154 size_t SumPayloadBytesWithinTemporalLayer(int temporal_layer) {
155 size_t payload_size = 0;
156 for (size_t i = 0; i < frame_data_.size(); ++i) {
157 EXPECT_EQ(kRtpVideoVp8, frame_data_[i].rtp_video_header.codec);
158 const uint8_t temporal_idx =
159 frame_data_[i].rtp_video_header.codecHeader.VP8.temporalIdx;
160 if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx)
161 payload_size += frame_data_[i].payload_size;
162 }
163 return payload_size;
164 }
165
166 Clock* clock_;
167 int64_t start_time_ms_;
168 vector<FrameData> frame_data_;
169 };
170
171 class TestVideoSender : public ::testing::Test {
172 protected:
173 // Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as
174 // a special case (e.g. frame rate in media optimization).
TestVideoSender()175 TestVideoSender() : clock_(1000), packetization_callback_(&clock_) {}
176
SetUp()177 void SetUp() override {
178 sender_.reset(
179 new VideoSender(&clock_, &post_encode_callback_, nullptr, nullptr));
180 EXPECT_EQ(0, sender_->RegisterTransportCallback(&packetization_callback_));
181 }
182
AddFrame()183 void AddFrame() {
184 assert(generator_.get());
185 sender_->AddVideoFrame(*generator_->NextFrame(), NULL, NULL);
186 }
187
188 SimulatedClock clock_;
189 PacketizationCallback packetization_callback_;
190 MockEncodedImageCallback post_encode_callback_;
191 // Used by subclassing tests, need to outlive sender_.
192 rtc::scoped_ptr<VideoEncoder> encoder_;
193 rtc::scoped_ptr<VideoSender> sender_;
194 rtc::scoped_ptr<FrameGenerator> generator_;
195 };
196
197 class TestVideoSenderWithMockEncoder : public TestVideoSender {
198 protected:
199 static const int kDefaultWidth = 1280;
200 static const int kDefaultHeight = 720;
201 static const int kNumberOfStreams = 3;
202 static const int kNumberOfLayers = 3;
203 static const int kUnusedPayloadType = 10;
204
SetUp()205 void SetUp() override {
206 TestVideoSender::SetUp();
207 sender_->RegisterExternalEncoder(&encoder_, kUnusedPayloadType, false);
208 VideoCodingModule::Codec(kVideoCodecVP8, &settings_);
209 settings_.numberOfSimulcastStreams = kNumberOfStreams;
210 ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, 100,
211 &settings_.simulcastStream[0]);
212 ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, 500,
213 &settings_.simulcastStream[1]);
214 ConfigureStream(kDefaultWidth, kDefaultHeight, 1200,
215 &settings_.simulcastStream[2]);
216 settings_.plType = kUnusedPayloadType; // Use the mocked encoder.
217 generator_.reset(
218 new EmptyFrameGenerator(settings_.width, settings_.height));
219 EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
220 }
221
TearDown()222 void TearDown() override { sender_.reset(); }
223
ExpectIntraRequest(int stream)224 void ExpectIntraRequest(int stream) {
225 if (stream == -1) {
226 // No intra request expected.
227 EXPECT_CALL(
228 encoder_,
229 Encode(_, _, Pointee(ElementsAre(kVideoFrameDelta, kVideoFrameDelta,
230 kVideoFrameDelta))))
231 .Times(1)
232 .WillRepeatedly(Return(0));
233 return;
234 }
235 assert(stream >= 0);
236 assert(stream < kNumberOfStreams);
237 std::vector<FrameType> frame_types(kNumberOfStreams, kVideoFrameDelta);
238 frame_types[stream] = kVideoFrameKey;
239 EXPECT_CALL(encoder_,
240 Encode(_, _, Pointee(ElementsAreArray(&frame_types[0],
241 frame_types.size()))))
242 .Times(1)
243 .WillRepeatedly(Return(0));
244 }
245
ConfigureStream(int width,int height,int max_bitrate,SimulcastStream * stream)246 static void ConfigureStream(int width,
247 int height,
248 int max_bitrate,
249 SimulcastStream* stream) {
250 assert(stream);
251 stream->width = width;
252 stream->height = height;
253 stream->maxBitrate = max_bitrate;
254 stream->numberOfTemporalLayers = kNumberOfLayers;
255 stream->qpMax = 45;
256 }
257
258 VideoCodec settings_;
259 NiceMock<MockVideoEncoder> encoder_;
260 };
261
TEST_F(TestVideoSenderWithMockEncoder,TestIntraRequests)262 TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) {
263 EXPECT_EQ(0, sender_->IntraFrameRequest(0));
264 ExpectIntraRequest(0);
265 AddFrame();
266 ExpectIntraRequest(-1);
267 AddFrame();
268
269 EXPECT_EQ(0, sender_->IntraFrameRequest(1));
270 ExpectIntraRequest(1);
271 AddFrame();
272 ExpectIntraRequest(-1);
273 AddFrame();
274
275 EXPECT_EQ(0, sender_->IntraFrameRequest(2));
276 ExpectIntraRequest(2);
277 AddFrame();
278 ExpectIntraRequest(-1);
279 AddFrame();
280
281 EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
282 ExpectIntraRequest(-1);
283 AddFrame();
284
285 EXPECT_EQ(-1, sender_->IntraFrameRequest(-1));
286 ExpectIntraRequest(-1);
287 AddFrame();
288 }
289
TEST_F(TestVideoSenderWithMockEncoder,TestIntraRequestsInternalCapture)290 TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequestsInternalCapture) {
291 // De-register current external encoder.
292 sender_->RegisterExternalEncoder(nullptr, kUnusedPayloadType, false);
293 // Register encoder with internal capture.
294 sender_->RegisterExternalEncoder(&encoder_, kUnusedPayloadType, true);
295 EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200));
296 ExpectIntraRequest(0);
297 EXPECT_EQ(0, sender_->IntraFrameRequest(0));
298 ExpectIntraRequest(1);
299 EXPECT_EQ(0, sender_->IntraFrameRequest(1));
300 ExpectIntraRequest(2);
301 EXPECT_EQ(0, sender_->IntraFrameRequest(2));
302 // No requests expected since these indices are out of bounds.
303 EXPECT_EQ(-1, sender_->IntraFrameRequest(3));
304 EXPECT_EQ(-1, sender_->IntraFrameRequest(-1));
305 }
306
TEST_F(TestVideoSenderWithMockEncoder,EncoderFramerateUpdatedViaProcess)307 TEST_F(TestVideoSenderWithMockEncoder, EncoderFramerateUpdatedViaProcess) {
308 sender_->SetChannelParameters(settings_.startBitrate * 1000, 0, 200);
309 const int64_t kRateStatsWindowMs = 2000;
310 const uint32_t kInputFps = 20;
311 int64_t start_time = clock_.TimeInMilliseconds();
312 while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
313 AddFrame();
314 clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
315 }
316 EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0));
317 sender_->Process();
318 AddFrame();
319 }
320
TEST_F(TestVideoSenderWithMockEncoder,NoRedundantSetChannelParameterOrSetRatesCalls)321 TEST_F(TestVideoSenderWithMockEncoder,
322 NoRedundantSetChannelParameterOrSetRatesCalls) {
323 const uint8_t kLossRate = 4;
324 const uint8_t kRtt = 200;
325 const int64_t kRateStatsWindowMs = 2000;
326 const uint32_t kInputFps = 20;
327 int64_t start_time = clock_.TimeInMilliseconds();
328 // Expect initial call to SetChannelParameters. Rates are initialized through
329 // InitEncode and expects no additional call before the framerate (or bitrate)
330 // updates.
331 EXPECT_CALL(encoder_, SetChannelParameters(kLossRate, kRtt))
332 .Times(1)
333 .WillOnce(Return(0));
334 sender_->SetChannelParameters(settings_.startBitrate * 1000, kLossRate, kRtt);
335 while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
336 AddFrame();
337 clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
338 }
339 // After process, input framerate should be updated but not ChannelParameters
340 // as they are the same as before.
341 EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0));
342 sender_->Process();
343 AddFrame();
344 // Call to SetChannelParameters with changed bitrate should call encoder
345 // SetRates but not encoder SetChannelParameters (that are unchanged).
346 EXPECT_CALL(encoder_, SetRates(2 * settings_.startBitrate, kInputFps))
347 .Times(1)
348 .WillOnce(Return(0));
349 sender_->SetChannelParameters(2 * settings_.startBitrate * 1000, kLossRate,
350 kRtt);
351 AddFrame();
352 }
353
354 class TestVideoSenderWithVp8 : public TestVideoSender {
355 public:
TestVideoSenderWithVp8()356 TestVideoSenderWithVp8()
357 : codec_bitrate_kbps_(300), available_bitrate_kbps_(1000) {}
358
SetUp()359 void SetUp() override {
360 TestVideoSender::SetUp();
361
362 const char* input_video = "foreman_cif";
363 const int width = 352;
364 const int height = 288;
365 generator_.reset(FrameGenerator::CreateFromYuvFile(
366 std::vector<std::string>(1, test::ResourcePath(input_video, "yuv")),
367 width, height, 1));
368
369 codec_ = MakeVp8VideoCodec(width, height, 3);
370 codec_.minBitrate = 10;
371 codec_.startBitrate = codec_bitrate_kbps_;
372 codec_.maxBitrate = codec_bitrate_kbps_;
373 encoder_.reset(VP8Encoder::Create());
374 sender_->RegisterExternalEncoder(encoder_.get(), codec_.plType, false);
375 EXPECT_EQ(0, sender_->RegisterSendCodec(&codec_, 1, 1200));
376 }
377
MakeVp8VideoCodec(int width,int height,int temporal_layers)378 static VideoCodec MakeVp8VideoCodec(int width,
379 int height,
380 int temporal_layers) {
381 VideoCodec codec;
382 VideoCodingModule::Codec(kVideoCodecVP8, &codec);
383 codec.width = width;
384 codec.height = height;
385 codec.codecSpecific.VP8.numberOfTemporalLayers = temporal_layers;
386 return codec;
387 }
388
InsertFrames(float framerate,float seconds)389 void InsertFrames(float framerate, float seconds) {
390 for (int i = 0; i < seconds * framerate; ++i) {
391 clock_.AdvanceTimeMilliseconds(1000.0f / framerate);
392 EXPECT_CALL(post_encode_callback_, Encoded(_, NULL, NULL))
393 .WillOnce(Return(0));
394 AddFrame();
395 // SetChannelParameters needs to be called frequently to propagate
396 // framerate from the media optimization into the encoder.
397 // Note: SetChannelParameters fails if less than 2 frames are in the
398 // buffer since it will fail to calculate the framerate.
399 if (i != 0) {
400 EXPECT_EQ(VCM_OK, sender_->SetChannelParameters(
401 available_bitrate_kbps_ * 1000, 0, 200));
402 }
403 }
404 }
405
SimulateWithFramerate(float framerate)406 Vp8StreamInfo SimulateWithFramerate(float framerate) {
407 const float short_simulation_interval = 5.0;
408 const float long_simulation_interval = 10.0;
409 // It appears that this 5 seconds simulation is needed to allow
410 // bitrate and framerate to stabilize.
411 InsertFrames(framerate, short_simulation_interval);
412 packetization_callback_.Reset();
413
414 InsertFrames(framerate, long_simulation_interval);
415 return packetization_callback_.CalculateVp8StreamInfo();
416 }
417
418 protected:
419 VideoCodec codec_;
420 int codec_bitrate_kbps_;
421 int available_bitrate_kbps_;
422 };
423
424 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
425 #define MAYBE_FixedTemporalLayersStrategy DISABLED_FixedTemporalLayersStrategy
426 #else
427 #define MAYBE_FixedTemporalLayersStrategy FixedTemporalLayersStrategy
428 #endif
TEST_F(TestVideoSenderWithVp8,MAYBE_FixedTemporalLayersStrategy)429 TEST_F(TestVideoSenderWithVp8, MAYBE_FixedTemporalLayersStrategy) {
430 const int low_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0];
431 const int mid_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1];
432 const int high_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2];
433 {
434 Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
435 EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
436 }
437 {
438 Vp8StreamInfo expected = {{3.75, 7.5, 15.0}, {low_b, mid_b, high_b}};
439 EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
440 }
441 }
442
443 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
444 #define MAYBE_RealTimeTemporalLayersStrategy \
445 DISABLED_RealTimeTemporalLayersStrategy
446 #else
447 #define MAYBE_RealTimeTemporalLayersStrategy RealTimeTemporalLayersStrategy
448 #endif
TEST_F(TestVideoSenderWithVp8,MAYBE_RealTimeTemporalLayersStrategy)449 TEST_F(TestVideoSenderWithVp8, MAYBE_RealTimeTemporalLayersStrategy) {
450 Config extra_options;
451 extra_options.Set<TemporalLayers::Factory>(
452 new RealTimeTemporalLayersFactory());
453 VideoCodec codec = MakeVp8VideoCodec(352, 288, 3);
454 codec.extra_options = &extra_options;
455 codec.minBitrate = 10;
456 codec.startBitrate = codec_bitrate_kbps_;
457 codec.maxBitrate = codec_bitrate_kbps_;
458 EXPECT_EQ(0, sender_->RegisterSendCodec(&codec, 1, 1200));
459
460 const int low_b = codec_bitrate_kbps_ * 0.4;
461 const int mid_b = codec_bitrate_kbps_ * 0.6;
462 const int high_b = codec_bitrate_kbps_;
463
464 {
465 Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
466 EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
467 }
468 {
469 Vp8StreamInfo expected = {{5.0, 10.0, 20.0}, {low_b, mid_b, high_b}};
470 EXPECT_THAT(SimulateWithFramerate(20.0), MatchesVp8StreamInfo(expected));
471 }
472 {
473 Vp8StreamInfo expected = {{7.5, 15.0, 15.0}, {mid_b, high_b, high_b}};
474 EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
475 }
476 {
477 Vp8StreamInfo expected = {{5.0, 10.0, 10.0}, {mid_b, high_b, high_b}};
478 EXPECT_THAT(SimulateWithFramerate(10.0), MatchesVp8StreamInfo(expected));
479 }
480 {
481 // TODO(andresp): Find out why this fails with framerate = 7.5
482 Vp8StreamInfo expected = {{7.0, 7.0, 7.0}, {high_b, high_b, high_b}};
483 EXPECT_THAT(SimulateWithFramerate(7.0), MatchesVp8StreamInfo(expected));
484 }
485 }
486 } // namespace
487 } // namespace vcm
488 } // namespace webrtc
489