1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "call/rtp_video_sender.h"
12
13 #include <atomic>
14 #include <memory>
15 #include <string>
16
17 #include "call/rtp_transport_controller_send.h"
18 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
19 #include "modules/rtp_rtcp/source/byte_io.h"
20 #include "modules/rtp_rtcp/source/rtcp_packet/nack.h"
21 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
22 #include "modules/rtp_rtcp/source/rtp_packet.h"
23 #include "modules/video_coding/fec_controller_default.h"
24 #include "modules/video_coding/include/video_codec_interface.h"
25 #include "rtc_base/rate_limiter.h"
26 #include "test/field_trial.h"
27 #include "test/gmock.h"
28 #include "test/gtest.h"
29 #include "test/mock_frame_transformer.h"
30 #include "test/mock_transport.h"
31 #include "test/scenario/scenario.h"
32 #include "test/time_controller/simulated_time_controller.h"
33 #include "video/call_stats.h"
34 #include "video/send_delay_stats.h"
35 #include "video/send_statistics_proxy.h"
36
37 using ::testing::_;
38 using ::testing::NiceMock;
39 using ::testing::SaveArg;
40 using ::testing::SizeIs;
41
42 namespace webrtc {
43 namespace {
44 const int8_t kPayloadType = 96;
45 const uint32_t kSsrc1 = 12345;
46 const uint32_t kSsrc2 = 23456;
47 const uint32_t kRtxSsrc1 = 34567;
48 const uint32_t kRtxSsrc2 = 45678;
49 const int16_t kInitialPictureId1 = 222;
50 const int16_t kInitialPictureId2 = 44;
51 const int16_t kInitialTl0PicIdx1 = 99;
52 const int16_t kInitialTl0PicIdx2 = 199;
53 const int64_t kRetransmitWindowSizeMs = 500;
54 const int kTransportsSequenceExtensionId = 7;
55 const int kDependencyDescriptorExtensionId = 8;
56
57 class MockRtcpIntraFrameObserver : public RtcpIntraFrameObserver {
58 public:
59 MOCK_METHOD(void, OnReceivedIntraFrameRequest, (uint32_t), (override));
60 };
61
CreateObservers(RtcpRttStats * rtcp_rtt_stats,RtcpIntraFrameObserver * intra_frame_callback,RtcpStatisticsCallback * rtcp_stats,ReportBlockDataObserver * report_block_data_observer,StreamDataCountersCallback * rtp_stats,BitrateStatisticsObserver * bitrate_observer,FrameCountObserver * frame_count_observer,RtcpPacketTypeCounterObserver * rtcp_type_observer,SendSideDelayObserver * send_delay_observer,SendPacketObserver * send_packet_observer)62 RtpSenderObservers CreateObservers(
63 RtcpRttStats* rtcp_rtt_stats,
64 RtcpIntraFrameObserver* intra_frame_callback,
65 RtcpStatisticsCallback* rtcp_stats,
66 ReportBlockDataObserver* report_block_data_observer,
67 StreamDataCountersCallback* rtp_stats,
68 BitrateStatisticsObserver* bitrate_observer,
69 FrameCountObserver* frame_count_observer,
70 RtcpPacketTypeCounterObserver* rtcp_type_observer,
71 SendSideDelayObserver* send_delay_observer,
72 SendPacketObserver* send_packet_observer) {
73 RtpSenderObservers observers;
74 observers.rtcp_rtt_stats = rtcp_rtt_stats;
75 observers.intra_frame_callback = intra_frame_callback;
76 observers.rtcp_loss_notification_observer = nullptr;
77 observers.rtcp_stats = rtcp_stats;
78 observers.report_block_data_observer = report_block_data_observer;
79 observers.rtp_stats = rtp_stats;
80 observers.bitrate_observer = bitrate_observer;
81 observers.frame_count_observer = frame_count_observer;
82 observers.rtcp_type_observer = rtcp_type_observer;
83 observers.send_delay_observer = send_delay_observer;
84 observers.send_packet_observer = send_packet_observer;
85 return observers;
86 }
87
GetBitrateConfig()88 BitrateConstraints GetBitrateConfig() {
89 BitrateConstraints bitrate_config;
90 bitrate_config.min_bitrate_bps = 30000;
91 bitrate_config.start_bitrate_bps = 300000;
92 bitrate_config.max_bitrate_bps = 3000000;
93 return bitrate_config;
94 }
95
CreateVideoSendStreamConfig(Transport * transport,const std::vector<uint32_t> & ssrcs,const std::vector<uint32_t> & rtx_ssrcs,int payload_type)96 VideoSendStream::Config CreateVideoSendStreamConfig(
97 Transport* transport,
98 const std::vector<uint32_t>& ssrcs,
99 const std::vector<uint32_t>& rtx_ssrcs,
100 int payload_type) {
101 VideoSendStream::Config config(transport);
102 config.rtp.ssrcs = ssrcs;
103 config.rtp.rtx.ssrcs = rtx_ssrcs;
104 config.rtp.payload_type = payload_type;
105 config.rtp.rtx.payload_type = payload_type + 1;
106 config.rtp.nack.rtp_history_ms = 1000;
107 config.rtp.extensions.emplace_back(RtpExtension::kTransportSequenceNumberUri,
108 kTransportsSequenceExtensionId);
109 config.rtp.extensions.emplace_back(RtpDependencyDescriptorExtension::kUri,
110 kDependencyDescriptorExtensionId);
111 return config;
112 }
113
114 class RtpVideoSenderTestFixture {
115 public:
RtpVideoSenderTestFixture(const std::vector<uint32_t> & ssrcs,const std::vector<uint32_t> & rtx_ssrcs,int payload_type,const std::map<uint32_t,RtpPayloadState> & suspended_payload_states,FrameCountObserver * frame_count_observer,rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)116 RtpVideoSenderTestFixture(
117 const std::vector<uint32_t>& ssrcs,
118 const std::vector<uint32_t>& rtx_ssrcs,
119 int payload_type,
120 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
121 FrameCountObserver* frame_count_observer,
122 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
123 : time_controller_(Timestamp::Millis(1000000)),
124 config_(CreateVideoSendStreamConfig(&transport_,
125 ssrcs,
126 rtx_ssrcs,
127 payload_type)),
128 send_delay_stats_(time_controller_.GetClock()),
129 bitrate_config_(GetBitrateConfig()),
130 transport_controller_(
131 time_controller_.GetClock(),
132 &event_log_,
133 nullptr,
134 nullptr,
135 bitrate_config_,
136 time_controller_.CreateProcessThread("PacerThread"),
137 time_controller_.GetTaskQueueFactory(),
138 &field_trials_),
139 process_thread_(time_controller_.CreateProcessThread("test_thread")),
140 call_stats_(time_controller_.GetClock(), process_thread_.get()),
141 stats_proxy_(time_controller_.GetClock(),
142 config_,
143 VideoEncoderConfig::ContentType::kRealtimeVideo),
144 retransmission_rate_limiter_(time_controller_.GetClock(),
145 kRetransmitWindowSizeMs) {
146 std::map<uint32_t, RtpState> suspended_ssrcs;
147 router_ = std::make_unique<RtpVideoSender>(
148 time_controller_.GetClock(), suspended_ssrcs, suspended_payload_states,
149 config_.rtp, config_.rtcp_report_interval_ms, &transport_,
150 CreateObservers(&call_stats_, &encoder_feedback_, &stats_proxy_,
151 &stats_proxy_, &stats_proxy_, &stats_proxy_,
152 frame_count_observer, &stats_proxy_, &stats_proxy_,
153 &send_delay_stats_),
154 &transport_controller_, &event_log_, &retransmission_rate_limiter_,
155 std::make_unique<FecControllerDefault>(time_controller_.GetClock()),
156 nullptr, CryptoOptions{}, frame_transformer);
157 }
158
RtpVideoSenderTestFixture(const std::vector<uint32_t> & ssrcs,const std::vector<uint32_t> & rtx_ssrcs,int payload_type,const std::map<uint32_t,RtpPayloadState> & suspended_payload_states,FrameCountObserver * frame_count_observer)159 RtpVideoSenderTestFixture(
160 const std::vector<uint32_t>& ssrcs,
161 const std::vector<uint32_t>& rtx_ssrcs,
162 int payload_type,
163 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
164 FrameCountObserver* frame_count_observer)
165 : RtpVideoSenderTestFixture(ssrcs,
166 rtx_ssrcs,
167 payload_type,
168 suspended_payload_states,
169 frame_count_observer,
170 /*frame_transformer=*/nullptr) {}
171
RtpVideoSenderTestFixture(const std::vector<uint32_t> & ssrcs,const std::vector<uint32_t> & rtx_ssrcs,int payload_type,const std::map<uint32_t,RtpPayloadState> & suspended_payload_states)172 RtpVideoSenderTestFixture(
173 const std::vector<uint32_t>& ssrcs,
174 const std::vector<uint32_t>& rtx_ssrcs,
175 int payload_type,
176 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states)
177 : RtpVideoSenderTestFixture(ssrcs,
178 rtx_ssrcs,
179 payload_type,
180 suspended_payload_states,
181 /*frame_count_observer=*/nullptr,
182 /*frame_transformer=*/nullptr) {}
183
router()184 RtpVideoSender* router() { return router_.get(); }
transport()185 MockTransport& transport() { return transport_; }
AdvanceTime(TimeDelta delta)186 void AdvanceTime(TimeDelta delta) { time_controller_.AdvanceTime(delta); }
187
188 private:
189 NiceMock<MockTransport> transport_;
190 NiceMock<MockRtcpIntraFrameObserver> encoder_feedback_;
191 GlobalSimulatedTimeController time_controller_;
192 RtcEventLogNull event_log_;
193 VideoSendStream::Config config_;
194 SendDelayStats send_delay_stats_;
195 BitrateConstraints bitrate_config_;
196 const FieldTrialBasedConfig field_trials_;
197 RtpTransportControllerSend transport_controller_;
198 std::unique_ptr<ProcessThread> process_thread_;
199 // TODO(tommi): Use internal::CallStats.
200 CallStats call_stats_;
201 SendStatisticsProxy stats_proxy_;
202 RateLimiter retransmission_rate_limiter_;
203 std::unique_ptr<RtpVideoSender> router_;
204 };
205 } // namespace
206
TEST(RtpVideoSenderTest,SendOnOneModule)207 TEST(RtpVideoSenderTest, SendOnOneModule) {
208 constexpr uint8_t kPayload = 'a';
209 EncodedImage encoded_image;
210 encoded_image.SetTimestamp(1);
211 encoded_image.capture_time_ms_ = 2;
212 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
213 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
214
215 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {});
216 EXPECT_NE(
217 EncodedImageCallback::Result::OK,
218 test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
219
220 test.router()->SetActive(true);
221 EXPECT_EQ(
222 EncodedImageCallback::Result::OK,
223 test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
224
225 test.router()->SetActive(false);
226 EXPECT_NE(
227 EncodedImageCallback::Result::OK,
228 test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
229
230 test.router()->SetActive(true);
231 EXPECT_EQ(
232 EncodedImageCallback::Result::OK,
233 test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
234 }
235
TEST(RtpVideoSenderTest,SendSimulcastSetActive)236 TEST(RtpVideoSenderTest, SendSimulcastSetActive) {
237 constexpr uint8_t kPayload = 'a';
238 EncodedImage encoded_image_1;
239 encoded_image_1.SetTimestamp(1);
240 encoded_image_1.capture_time_ms_ = 2;
241 encoded_image_1._frameType = VideoFrameType::kVideoFrameKey;
242 encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
243
244 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
245 kPayloadType, {});
246
247 CodecSpecificInfo codec_info;
248 codec_info.codecType = kVideoCodecVP8;
249
250 test.router()->SetActive(true);
251 EXPECT_EQ(EncodedImageCallback::Result::OK,
252 test.router()
253 ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
254 .error);
255
256 EncodedImage encoded_image_2(encoded_image_1);
257 encoded_image_2.SetSpatialIndex(1);
258 EXPECT_EQ(EncodedImageCallback::Result::OK,
259 test.router()
260 ->OnEncodedImage(encoded_image_2, &codec_info, nullptr)
261 .error);
262
263 // Inactive.
264 test.router()->SetActive(false);
265 EXPECT_NE(EncodedImageCallback::Result::OK,
266 test.router()
267 ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
268 .error);
269 EXPECT_NE(EncodedImageCallback::Result::OK,
270 test.router()
271 ->OnEncodedImage(encoded_image_2, &codec_info, nullptr)
272 .error);
273 }
274
275 // Tests how setting individual rtp modules to active affects the overall
276 // behavior of the payload router. First sets one module to active and checks
277 // that outgoing data can be sent on this module, and checks that no data can
278 // be sent if both modules are inactive.
TEST(RtpVideoSenderTest,SendSimulcastSetActiveModules)279 TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) {
280 constexpr uint8_t kPayload = 'a';
281 EncodedImage encoded_image_1;
282 encoded_image_1.SetTimestamp(1);
283 encoded_image_1.capture_time_ms_ = 2;
284 encoded_image_1._frameType = VideoFrameType::kVideoFrameKey;
285 encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
286
287 EncodedImage encoded_image_2(encoded_image_1);
288 encoded_image_2.SetSpatialIndex(1);
289
290 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
291 kPayloadType, {});
292 CodecSpecificInfo codec_info;
293 codec_info.codecType = kVideoCodecVP8;
294
295 // Only setting one stream to active will still set the payload router to
296 // active and allow sending data on the active stream.
297 std::vector<bool> active_modules({true, false});
298 test.router()->SetActiveModules(active_modules);
299 EXPECT_EQ(EncodedImageCallback::Result::OK,
300 test.router()
301 ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
302 .error);
303
304 // Setting both streams to inactive will turn the payload router to
305 // inactive.
306 active_modules = {false, false};
307 test.router()->SetActiveModules(active_modules);
308 // An incoming encoded image will not ask the module to send outgoing data
309 // because the payload router is inactive.
310 EXPECT_NE(EncodedImageCallback::Result::OK,
311 test.router()
312 ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
313 .error);
314 EXPECT_NE(EncodedImageCallback::Result::OK,
315 test.router()
316 ->OnEncodedImage(encoded_image_1, &codec_info, nullptr)
317 .error);
318 }
319
TEST(RtpVideoSenderTest,CreateWithNoPreviousStates)320 TEST(RtpVideoSenderTest, CreateWithNoPreviousStates) {
321 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
322 kPayloadType, {});
323 test.router()->SetActive(true);
324
325 std::map<uint32_t, RtpPayloadState> initial_states =
326 test.router()->GetRtpPayloadStates();
327 EXPECT_EQ(2u, initial_states.size());
328 EXPECT_NE(initial_states.find(kSsrc1), initial_states.end());
329 EXPECT_NE(initial_states.find(kSsrc2), initial_states.end());
330 }
331
TEST(RtpVideoSenderTest,CreateWithPreviousStates)332 TEST(RtpVideoSenderTest, CreateWithPreviousStates) {
333 const int64_t kState1SharedFrameId = 123;
334 const int64_t kState2SharedFrameId = 234;
335 RtpPayloadState state1;
336 state1.picture_id = kInitialPictureId1;
337 state1.tl0_pic_idx = kInitialTl0PicIdx1;
338 state1.shared_frame_id = kState1SharedFrameId;
339 RtpPayloadState state2;
340 state2.picture_id = kInitialPictureId2;
341 state2.tl0_pic_idx = kInitialTl0PicIdx2;
342 state2.shared_frame_id = kState2SharedFrameId;
343 std::map<uint32_t, RtpPayloadState> states = {{kSsrc1, state1},
344 {kSsrc2, state2}};
345
346 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
347 kPayloadType, states);
348 test.router()->SetActive(true);
349
350 std::map<uint32_t, RtpPayloadState> initial_states =
351 test.router()->GetRtpPayloadStates();
352 EXPECT_EQ(2u, initial_states.size());
353 EXPECT_EQ(kInitialPictureId1, initial_states[kSsrc1].picture_id);
354 EXPECT_EQ(kInitialTl0PicIdx1, initial_states[kSsrc1].tl0_pic_idx);
355 EXPECT_EQ(kInitialPictureId2, initial_states[kSsrc2].picture_id);
356 EXPECT_EQ(kInitialTl0PicIdx2, initial_states[kSsrc2].tl0_pic_idx);
357 EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc1].shared_frame_id);
358 EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc2].shared_frame_id);
359 }
360
TEST(RtpVideoSenderTest,FrameCountCallbacks)361 TEST(RtpVideoSenderTest, FrameCountCallbacks) {
362 class MockFrameCountObserver : public FrameCountObserver {
363 public:
364 MOCK_METHOD(void,
365 FrameCountUpdated,
366 (const FrameCounts& frame_counts, uint32_t ssrc),
367 (override));
368 } callback;
369
370 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {},
371 &callback);
372
373 constexpr uint8_t kPayload = 'a';
374 EncodedImage encoded_image;
375 encoded_image.SetTimestamp(1);
376 encoded_image.capture_time_ms_ = 2;
377 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
378 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
379
380 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
381
382 // No callbacks when not active.
383 EXPECT_CALL(callback, FrameCountUpdated).Times(0);
384 EXPECT_NE(
385 EncodedImageCallback::Result::OK,
386 test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
387 ::testing::Mock::VerifyAndClearExpectations(&callback);
388
389 test.router()->SetActive(true);
390
391 FrameCounts frame_counts;
392 EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1))
393 .WillOnce(SaveArg<0>(&frame_counts));
394 EXPECT_EQ(
395 EncodedImageCallback::Result::OK,
396 test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
397
398 EXPECT_EQ(1, frame_counts.key_frames);
399 EXPECT_EQ(0, frame_counts.delta_frames);
400
401 ::testing::Mock::VerifyAndClearExpectations(&callback);
402
403 encoded_image._frameType = VideoFrameType::kVideoFrameDelta;
404 EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1))
405 .WillOnce(SaveArg<0>(&frame_counts));
406 EXPECT_EQ(
407 EncodedImageCallback::Result::OK,
408 test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
409
410 EXPECT_EQ(1, frame_counts.key_frames);
411 EXPECT_EQ(1, frame_counts.delta_frames);
412 }
413
414 // Integration test verifying that ack of packet via TransportFeedback means
415 // that the packet is removed from RtpPacketHistory and won't be retransmitted
416 // again.
TEST(RtpVideoSenderTest,DoesNotRetrasmitAckedPackets)417 TEST(RtpVideoSenderTest, DoesNotRetrasmitAckedPackets) {
418 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
419 kPayloadType, {});
420 test.router()->SetActive(true);
421
422 constexpr uint8_t kPayload = 'a';
423 EncodedImage encoded_image;
424 encoded_image.SetTimestamp(1);
425 encoded_image.capture_time_ms_ = 2;
426 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
427 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
428
429 // Send two tiny images, mapping to two RTP packets. Capture sequence numbers.
430 std::vector<uint16_t> rtp_sequence_numbers;
431 std::vector<uint16_t> transport_sequence_numbers;
432 EXPECT_CALL(test.transport(), SendRtp)
433 .Times(2)
434 .WillRepeatedly([&rtp_sequence_numbers, &transport_sequence_numbers](
435 const uint8_t* packet, size_t length,
436 const PacketOptions& options) {
437 RtpPacket rtp_packet;
438 EXPECT_TRUE(rtp_packet.Parse(packet, length));
439 rtp_sequence_numbers.push_back(rtp_packet.SequenceNumber());
440 transport_sequence_numbers.push_back(options.packet_id);
441 return true;
442 });
443 EXPECT_EQ(
444 EncodedImageCallback::Result::OK,
445 test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
446 encoded_image.SetTimestamp(2);
447 encoded_image.capture_time_ms_ = 3;
448 EXPECT_EQ(
449 EncodedImageCallback::Result::OK,
450 test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error);
451
452 test.AdvanceTime(TimeDelta::Millis(33));
453
454 // Construct a NACK message for requesting retransmission of both packet.
455 rtcp::Nack nack;
456 nack.SetMediaSsrc(kSsrc1);
457 nack.SetPacketIds(rtp_sequence_numbers);
458 rtc::Buffer nack_buffer = nack.Build();
459
460 std::vector<uint16_t> retransmitted_rtp_sequence_numbers;
461 EXPECT_CALL(test.transport(), SendRtp)
462 .Times(2)
463 .WillRepeatedly([&retransmitted_rtp_sequence_numbers](
464 const uint8_t* packet, size_t length,
465 const PacketOptions& options) {
466 RtpPacket rtp_packet;
467 EXPECT_TRUE(rtp_packet.Parse(packet, length));
468 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1);
469 // Capture the retransmitted sequence number from the RTX header.
470 rtc::ArrayView<const uint8_t> payload = rtp_packet.payload();
471 retransmitted_rtp_sequence_numbers.push_back(
472 ByteReader<uint16_t>::ReadBigEndian(payload.data()));
473 return true;
474 });
475 test.router()->DeliverRtcp(nack_buffer.data(), nack_buffer.size());
476 test.AdvanceTime(TimeDelta::Millis(33));
477
478 // Verify that both packets were retransmitted.
479 EXPECT_EQ(retransmitted_rtp_sequence_numbers, rtp_sequence_numbers);
480
481 // Simulate transport feedback indicating fist packet received, next packet
482 // lost (not other way around as that would trigger early retransmit).
483 StreamFeedbackObserver::StreamPacketInfo lost_packet_feedback;
484 lost_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[0];
485 lost_packet_feedback.ssrc = kSsrc1;
486 lost_packet_feedback.received = false;
487
488 StreamFeedbackObserver::StreamPacketInfo received_packet_feedback;
489 received_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[1];
490 received_packet_feedback.ssrc = kSsrc1;
491 received_packet_feedback.received = true;
492
493 test.router()->OnPacketFeedbackVector(
494 {lost_packet_feedback, received_packet_feedback});
495
496 // Advance time to make sure retransmission would be allowed and try again.
497 // This time the retransmission should not happen for the first packet since
498 // the history has been notified of the ack and removed the packet. The
499 // second packet, included in the feedback but not marked as received, should
500 // still be retransmitted.
501 test.AdvanceTime(TimeDelta::Millis(33));
502 EXPECT_CALL(test.transport(), SendRtp)
503 .WillOnce([&lost_packet_feedback](const uint8_t* packet, size_t length,
504 const PacketOptions& options) {
505 RtpPacket rtp_packet;
506 EXPECT_TRUE(rtp_packet.Parse(packet, length));
507 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1);
508 // Capture the retransmitted sequence number from the RTX header.
509 rtc::ArrayView<const uint8_t> payload = rtp_packet.payload();
510 EXPECT_EQ(lost_packet_feedback.rtp_sequence_number,
511 ByteReader<uint16_t>::ReadBigEndian(payload.data()));
512 return true;
513 });
514 test.router()->DeliverRtcp(nack_buffer.data(), nack_buffer.size());
515 test.AdvanceTime(TimeDelta::Millis(33));
516 }
517
518 // This tests that we utilize transport wide feedback to retransmit lost
519 // packets. This is tested by dropping all ordirary packets from a "lossy"
520 // stream send along with an secondary untouched stream. The transport wide
521 // feedback packets from the secondary stream allows the sending side to
522 // detect and retreansmit the lost packets from the lossy stream.
TEST(RtpVideoSenderTest,RetransmitsOnTransportWideLossInfo)523 TEST(RtpVideoSenderTest, RetransmitsOnTransportWideLossInfo) {
524 int rtx_packets;
525 test::Scenario s(test_info_);
526 test::CallClientConfig call_conf;
527 // Keeping the bitrate fixed to avoid RTX due to probing.
528 call_conf.transport.rates.max_rate = DataRate::KilobitsPerSec(300);
529 call_conf.transport.rates.start_rate = DataRate::KilobitsPerSec(300);
530 test::NetworkSimulationConfig net_conf;
531 net_conf.bandwidth = DataRate::KilobitsPerSec(300);
532 auto send_node = s.CreateSimulationNode(net_conf);
533 auto* callee = s.CreateClient("return", call_conf);
534 auto* route = s.CreateRoutes(s.CreateClient("send", call_conf), {send_node},
535 callee, {s.CreateSimulationNode(net_conf)});
536
537 test::VideoStreamConfig lossy_config;
538 lossy_config.source.framerate = 5;
539 auto* lossy = s.CreateVideoStream(route->forward(), lossy_config);
540 // The secondary stream acts a driver for transport feedback messages,
541 // ensuring that lost packets on the lossy stream are retransmitted.
542 s.CreateVideoStream(route->forward(), test::VideoStreamConfig());
543
544 send_node->router()->SetFilter([&](const EmulatedIpPacket& packet) {
545 RtpPacket rtp;
546 if (rtp.Parse(packet.data)) {
547 // Drops all regular packets for the lossy stream and counts all RTX
548 // packets. Since no packets are let trough, NACKs can't be triggered
549 // by the receiving side.
550 if (lossy->send()->UsingSsrc(rtp.Ssrc())) {
551 return false;
552 } else if (lossy->send()->UsingRtxSsrc(rtp.Ssrc())) {
553 ++rtx_packets;
554 }
555 }
556 return true;
557 });
558
559 // Run for a short duration and reset counters to avoid counting RTX packets
560 // from initial probing.
561 s.RunFor(TimeDelta::Seconds(1));
562 rtx_packets = 0;
563 int decoded_baseline = 0;
564 callee->SendTask([&decoded_baseline, &lossy]() {
565 decoded_baseline = lossy->receive()->GetStats().frames_decoded;
566 });
567 s.RunFor(TimeDelta::Seconds(1));
568 // We expect both that RTX packets were sent and that an appropriate number of
569 // frames were received. This is somewhat redundant but reduces the risk of
570 // false positives in future regressions (e.g. RTX is send due to probing).
571 EXPECT_GE(rtx_packets, 1);
572 int frames_decoded = 0;
573 callee->SendTask([&decoded_baseline, &frames_decoded, &lossy]() {
574 frames_decoded =
575 lossy->receive()->GetStats().frames_decoded - decoded_baseline;
576 });
577 EXPECT_EQ(frames_decoded, 5);
578 }
579
580 // Integration test verifying that retransmissions are sent for packets which
581 // can be detected as lost early, using transport wide feedback.
TEST(RtpVideoSenderTest,EarlyRetransmits)582 TEST(RtpVideoSenderTest, EarlyRetransmits) {
583 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
584 kPayloadType, {});
585 test.router()->SetActive(true);
586
587 const uint8_t kPayload[1] = {'a'};
588 EncodedImage encoded_image;
589 encoded_image.SetTimestamp(1);
590 encoded_image.capture_time_ms_ = 2;
591 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
592 encoded_image.SetEncodedData(
593 EncodedImageBuffer::Create(kPayload, sizeof(kPayload)));
594 encoded_image.SetSpatialIndex(0);
595
596 CodecSpecificInfo codec_specific;
597 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric;
598
599 // Send two tiny images, mapping to single RTP packets. Capture sequence
600 // numbers.
601 uint16_t frame1_rtp_sequence_number = 0;
602 uint16_t frame1_transport_sequence_number = 0;
603 EXPECT_CALL(test.transport(), SendRtp)
604 .WillOnce(
605 [&frame1_rtp_sequence_number, &frame1_transport_sequence_number](
606 const uint8_t* packet, size_t length,
607 const PacketOptions& options) {
608 RtpPacket rtp_packet;
609 EXPECT_TRUE(rtp_packet.Parse(packet, length));
610 frame1_rtp_sequence_number = rtp_packet.SequenceNumber();
611 frame1_transport_sequence_number = options.packet_id;
612 EXPECT_EQ(rtp_packet.Ssrc(), kSsrc1);
613 return true;
614 });
615 EXPECT_EQ(test.router()
616 ->OnEncodedImage(encoded_image, &codec_specific, nullptr)
617 .error,
618 EncodedImageCallback::Result::OK);
619
620 test.AdvanceTime(TimeDelta::Millis(33));
621
622 uint16_t frame2_rtp_sequence_number = 0;
623 uint16_t frame2_transport_sequence_number = 0;
624 encoded_image.SetSpatialIndex(1);
625 EXPECT_CALL(test.transport(), SendRtp)
626 .WillOnce(
627 [&frame2_rtp_sequence_number, &frame2_transport_sequence_number](
628 const uint8_t* packet, size_t length,
629 const PacketOptions& options) {
630 RtpPacket rtp_packet;
631 EXPECT_TRUE(rtp_packet.Parse(packet, length));
632 frame2_rtp_sequence_number = rtp_packet.SequenceNumber();
633 frame2_transport_sequence_number = options.packet_id;
634 EXPECT_EQ(rtp_packet.Ssrc(), kSsrc2);
635 return true;
636 });
637 EXPECT_EQ(test.router()
638 ->OnEncodedImage(encoded_image, &codec_specific, nullptr)
639 .error,
640 EncodedImageCallback::Result::OK);
641 test.AdvanceTime(TimeDelta::Millis(33));
642
643 EXPECT_NE(frame1_transport_sequence_number, frame2_transport_sequence_number);
644
645 // Inject a transport feedback where the packet for the first frame is lost,
646 // expect a retransmission for it.
647 EXPECT_CALL(test.transport(), SendRtp)
648 .WillOnce([&frame1_rtp_sequence_number](const uint8_t* packet,
649 size_t length,
650 const PacketOptions& options) {
651 RtpPacket rtp_packet;
652 EXPECT_TRUE(rtp_packet.Parse(packet, length));
653 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1);
654
655 // Retransmitted sequence number from the RTX header should match
656 // the lost packet.
657 rtc::ArrayView<const uint8_t> payload = rtp_packet.payload();
658 EXPECT_EQ(ByteReader<uint16_t>::ReadBigEndian(payload.data()),
659 frame1_rtp_sequence_number);
660 return true;
661 });
662
663 StreamFeedbackObserver::StreamPacketInfo first_packet_feedback;
664 first_packet_feedback.rtp_sequence_number = frame1_rtp_sequence_number;
665 first_packet_feedback.ssrc = kSsrc1;
666 first_packet_feedback.received = false;
667
668 StreamFeedbackObserver::StreamPacketInfo second_packet_feedback;
669 second_packet_feedback.rtp_sequence_number = frame2_rtp_sequence_number;
670 second_packet_feedback.ssrc = kSsrc2;
671 second_packet_feedback.received = true;
672
673 test.router()->OnPacketFeedbackVector(
674 {first_packet_feedback, second_packet_feedback});
675
676 // Wait for pacer to run and send the RTX packet.
677 test.AdvanceTime(TimeDelta::Millis(33));
678 }
679
TEST(RtpVideoSenderTest,SupportsDependencyDescriptor)680 TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) {
681 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {});
682 test.router()->SetActive(true);
683
684 RtpHeaderExtensionMap extensions;
685 extensions.Register<RtpDependencyDescriptorExtension>(
686 kDependencyDescriptorExtensionId);
687 std::vector<RtpPacket> sent_packets;
688 ON_CALL(test.transport(), SendRtp)
689 .WillByDefault([&](const uint8_t* packet, size_t length,
690 const PacketOptions& options) {
691 sent_packets.emplace_back(&extensions);
692 EXPECT_TRUE(sent_packets.back().Parse(packet, length));
693 return true;
694 });
695
696 const uint8_t kPayload[1] = {'a'};
697 EncodedImage encoded_image;
698 encoded_image.SetTimestamp(1);
699 encoded_image.capture_time_ms_ = 2;
700 encoded_image.SetEncodedData(
701 EncodedImageBuffer::Create(kPayload, sizeof(kPayload)));
702
703 CodecSpecificInfo codec_specific;
704 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric;
705 codec_specific.template_structure.emplace();
706 codec_specific.template_structure->num_decode_targets = 1;
707 codec_specific.template_structure->templates = {
708 FrameDependencyTemplate().T(0).Dtis("S"),
709 FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}),
710 FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}),
711 };
712
713 // Send two tiny images, mapping to single RTP packets.
714 // Send in key frame.
715 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
716 codec_specific.generic_frame_info =
717 GenericFrameInfo::Builder().T(0).Dtis("S").Build();
718 codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}};
719 EXPECT_EQ(test.router()
720 ->OnEncodedImage(encoded_image, &codec_specific, nullptr)
721 .error,
722 EncodedImageCallback::Result::OK);
723 test.AdvanceTime(TimeDelta::Millis(33));
724 ASSERT_THAT(sent_packets, SizeIs(1));
725 EXPECT_TRUE(
726 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>());
727
728 // Send in delta frame.
729 encoded_image._frameType = VideoFrameType::kVideoFrameDelta;
730 codec_specific.template_structure = absl::nullopt;
731 codec_specific.generic_frame_info =
732 GenericFrameInfo::Builder().T(1).Dtis("D").Build();
733 codec_specific.generic_frame_info->encoder_buffers = {{0, true, false}};
734 EXPECT_EQ(test.router()
735 ->OnEncodedImage(encoded_image, &codec_specific, nullptr)
736 .error,
737 EncodedImageCallback::Result::OK);
738 test.AdvanceTime(TimeDelta::Millis(33));
739 ASSERT_THAT(sent_packets, SizeIs(2));
740 EXPECT_TRUE(
741 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>());
742 }
743
TEST(RtpVideoSenderTest,SupportsStoppingUsingDependencyDescriptor)744 TEST(RtpVideoSenderTest, SupportsStoppingUsingDependencyDescriptor) {
745 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {});
746 test.router()->SetActive(true);
747
748 RtpHeaderExtensionMap extensions;
749 extensions.Register<RtpDependencyDescriptorExtension>(
750 kDependencyDescriptorExtensionId);
751 std::vector<RtpPacket> sent_packets;
752 ON_CALL(test.transport(), SendRtp)
753 .WillByDefault([&](const uint8_t* packet, size_t length,
754 const PacketOptions& options) {
755 sent_packets.emplace_back(&extensions);
756 EXPECT_TRUE(sent_packets.back().Parse(packet, length));
757 return true;
758 });
759
760 const uint8_t kPayload[1] = {'a'};
761 EncodedImage encoded_image;
762 encoded_image.SetTimestamp(1);
763 encoded_image.capture_time_ms_ = 2;
764 encoded_image.SetEncodedData(
765 EncodedImageBuffer::Create(kPayload, sizeof(kPayload)));
766
767 CodecSpecificInfo codec_specific;
768 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric;
769 codec_specific.template_structure.emplace();
770 codec_specific.template_structure->num_decode_targets = 1;
771 codec_specific.template_structure->templates = {
772 FrameDependencyTemplate().T(0).Dtis("S"),
773 FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}),
774 FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}),
775 };
776
777 // Send two tiny images, mapping to single RTP packets.
778 // Send in a key frame.
779 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
780 codec_specific.generic_frame_info =
781 GenericFrameInfo::Builder().T(0).Dtis("S").Build();
782 codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}};
783 EXPECT_EQ(test.router()
784 ->OnEncodedImage(encoded_image, &codec_specific, nullptr)
785 .error,
786 EncodedImageCallback::Result::OK);
787 test.AdvanceTime(TimeDelta::Millis(33));
788 ASSERT_THAT(sent_packets, SizeIs(1));
789 EXPECT_TRUE(
790 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>());
791
792 // Send in a new key frame without the support for the dependency descriptor.
793 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
794 codec_specific.template_structure = absl::nullopt;
795 EXPECT_EQ(test.router()
796 ->OnEncodedImage(encoded_image, &codec_specific, nullptr)
797 .error,
798 EncodedImageCallback::Result::OK);
799 test.AdvanceTime(TimeDelta::Millis(33));
800 ASSERT_THAT(sent_packets, SizeIs(2));
801 EXPECT_FALSE(
802 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>());
803 }
804
TEST(RtpVideoSenderTest,CanSetZeroBitrateWithOverhead)805 TEST(RtpVideoSenderTest, CanSetZeroBitrateWithOverhead) {
806 test::ScopedFieldTrials trials("WebRTC-SendSideBwe-WithOverhead/Enabled/");
807 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {});
808 BitrateAllocationUpdate update;
809 update.target_bitrate = DataRate::Zero();
810 update.packet_loss_ratio = 0;
811 update.round_trip_time = TimeDelta::Zero();
812
813 test.router()->OnBitrateUpdated(update, /*framerate*/ 0);
814 }
815
TEST(RtpVideoSenderTest,CanSetZeroBitrateWithoutOverhead)816 TEST(RtpVideoSenderTest, CanSetZeroBitrateWithoutOverhead) {
817 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {});
818
819 BitrateAllocationUpdate update;
820 update.target_bitrate = DataRate::Zero();
821 update.packet_loss_ratio = 0;
822 update.round_trip_time = TimeDelta::Zero();
823
824 test.router()->OnBitrateUpdated(update, /*framerate*/ 0);
825 }
826
TEST(RtpVideoSenderTest,SimulcastSenderRegistersFrameTransformers)827 TEST(RtpVideoSenderTest, SimulcastSenderRegistersFrameTransformers) {
828 rtc::scoped_refptr<MockFrameTransformer> transformer =
829 new rtc::RefCountedObject<MockFrameTransformer>();
830
831 EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc1));
832 EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc2));
833 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
834 kPayloadType, {}, nullptr, transformer);
835
836 EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc1));
837 EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc2));
838 }
839 } // namespace webrtc
840