1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h"
12
13 #include <stdlib.h>
14 #include <string.h>
15
16 #include <vector>
17
18 #include "webrtc/base/checks.h"
19 #include "webrtc/base/logging.h"
20 #include "webrtc/base/trace_event.h"
21 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
22 #include "webrtc/modules/rtp_rtcp/source/byte_io.h"
23 #include "webrtc/modules/rtp_rtcp/source/producer_fec.h"
24 #include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h"
25 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h"
26 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h"
27 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
28
29 namespace webrtc {
30 enum { REDForFECHeaderLength = 1 };
31
32 struct RtpPacket {
33 uint16_t rtpHeaderLength;
34 ForwardErrorCorrection::Packet* pkt;
35 };
36
RTPSenderVideo(Clock * clock,RTPSenderInterface * rtpSender)37 RTPSenderVideo::RTPSenderVideo(Clock* clock, RTPSenderInterface* rtpSender)
38 : _rtpSender(*rtpSender),
39 crit_(CriticalSectionWrapper::CreateCriticalSection()),
40 _videoType(kRtpVideoGeneric),
41 _maxBitrate(0),
42 _retransmissionSettings(kRetransmitBaseLayer),
43
44 // Generic FEC
45 fec_(),
46 fec_enabled_(false),
47 red_payload_type_(-1),
48 fec_payload_type_(-1),
49 delta_fec_params_(),
50 key_fec_params_(),
51 producer_fec_(&fec_),
52 _fecOverheadRate(clock, NULL),
53 _videoBitrate(clock, NULL) {
54 memset(&delta_fec_params_, 0, sizeof(delta_fec_params_));
55 memset(&key_fec_params_, 0, sizeof(key_fec_params_));
56 delta_fec_params_.max_fec_frames = key_fec_params_.max_fec_frames = 1;
57 delta_fec_params_.fec_mask_type = key_fec_params_.fec_mask_type =
58 kFecMaskRandom;
59 }
60
~RTPSenderVideo()61 RTPSenderVideo::~RTPSenderVideo() {
62 }
63
SetVideoCodecType(RtpVideoCodecTypes videoType)64 void RTPSenderVideo::SetVideoCodecType(RtpVideoCodecTypes videoType) {
65 _videoType = videoType;
66 }
67
VideoCodecType() const68 RtpVideoCodecTypes RTPSenderVideo::VideoCodecType() const {
69 return _videoType;
70 }
71
72 // Static.
CreateVideoPayload(const char payloadName[RTP_PAYLOAD_NAME_SIZE],const int8_t payloadType,const uint32_t maxBitRate)73 RtpUtility::Payload* RTPSenderVideo::CreateVideoPayload(
74 const char payloadName[RTP_PAYLOAD_NAME_SIZE],
75 const int8_t payloadType,
76 const uint32_t maxBitRate) {
77 RtpVideoCodecTypes videoType = kRtpVideoGeneric;
78 if (RtpUtility::StringCompare(payloadName, "VP8", 3)) {
79 videoType = kRtpVideoVp8;
80 } else if (RtpUtility::StringCompare(payloadName, "VP9", 3)) {
81 videoType = kRtpVideoVp9;
82 } else if (RtpUtility::StringCompare(payloadName, "H264", 4)) {
83 videoType = kRtpVideoH264;
84 } else if (RtpUtility::StringCompare(payloadName, "I420", 4)) {
85 videoType = kRtpVideoGeneric;
86 } else {
87 videoType = kRtpVideoGeneric;
88 }
89 RtpUtility::Payload* payload = new RtpUtility::Payload();
90 payload->name[RTP_PAYLOAD_NAME_SIZE - 1] = 0;
91 strncpy(payload->name, payloadName, RTP_PAYLOAD_NAME_SIZE - 1);
92 payload->typeSpecific.Video.videoCodecType = videoType;
93 payload->typeSpecific.Video.maxRate = maxBitRate;
94 payload->audio = false;
95 return payload;
96 }
97
SendVideoPacket(uint8_t * data_buffer,const size_t payload_length,const size_t rtp_header_length,uint16_t seq_num,const uint32_t capture_timestamp,int64_t capture_time_ms,StorageType storage)98 void RTPSenderVideo::SendVideoPacket(uint8_t* data_buffer,
99 const size_t payload_length,
100 const size_t rtp_header_length,
101 uint16_t seq_num,
102 const uint32_t capture_timestamp,
103 int64_t capture_time_ms,
104 StorageType storage) {
105 if (_rtpSender.SendToNetwork(data_buffer, payload_length, rtp_header_length,
106 capture_time_ms, storage,
107 RtpPacketSender::kLowPriority) == 0) {
108 _videoBitrate.Update(payload_length + rtp_header_length);
109 TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
110 "Video::PacketNormal", "timestamp", capture_timestamp,
111 "seqnum", seq_num);
112 } else {
113 LOG(LS_WARNING) << "Failed to send video packet " << seq_num;
114 }
115 }
116
SendVideoPacketAsRed(uint8_t * data_buffer,const size_t payload_length,const size_t rtp_header_length,uint16_t media_seq_num,const uint32_t capture_timestamp,int64_t capture_time_ms,StorageType media_packet_storage,bool protect)117 void RTPSenderVideo::SendVideoPacketAsRed(uint8_t* data_buffer,
118 const size_t payload_length,
119 const size_t rtp_header_length,
120 uint16_t media_seq_num,
121 const uint32_t capture_timestamp,
122 int64_t capture_time_ms,
123 StorageType media_packet_storage,
124 bool protect) {
125 rtc::scoped_ptr<RedPacket> red_packet;
126 std::vector<RedPacket*> fec_packets;
127 StorageType fec_storage = kDontRetransmit;
128 uint16_t next_fec_sequence_number = 0;
129 {
130 // Only protect while creating RED and FEC packets, not when sending.
131 CriticalSectionScoped cs(crit_.get());
132 red_packet.reset(producer_fec_.BuildRedPacket(
133 data_buffer, payload_length, rtp_header_length, red_payload_type_));
134 if (protect) {
135 producer_fec_.AddRtpPacketAndGenerateFec(data_buffer, payload_length,
136 rtp_header_length);
137 }
138 uint16_t num_fec_packets = producer_fec_.NumAvailableFecPackets();
139 if (num_fec_packets > 0) {
140 next_fec_sequence_number =
141 _rtpSender.AllocateSequenceNumber(num_fec_packets);
142 fec_packets = producer_fec_.GetFecPackets(
143 red_payload_type_, fec_payload_type_, next_fec_sequence_number,
144 rtp_header_length);
145 RTC_DCHECK_EQ(num_fec_packets, fec_packets.size());
146 if (_retransmissionSettings & kRetransmitFECPackets)
147 fec_storage = kAllowRetransmission;
148 }
149 }
150 if (_rtpSender.SendToNetwork(
151 red_packet->data(), red_packet->length() - rtp_header_length,
152 rtp_header_length, capture_time_ms, media_packet_storage,
153 RtpPacketSender::kLowPriority) == 0) {
154 _videoBitrate.Update(red_packet->length());
155 TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
156 "Video::PacketRed", "timestamp", capture_timestamp,
157 "seqnum", media_seq_num);
158 } else {
159 LOG(LS_WARNING) << "Failed to send RED packet " << media_seq_num;
160 }
161 for (RedPacket* fec_packet : fec_packets) {
162 if (_rtpSender.SendToNetwork(
163 fec_packet->data(), fec_packet->length() - rtp_header_length,
164 rtp_header_length, capture_time_ms, fec_storage,
165 RtpPacketSender::kLowPriority) == 0) {
166 _fecOverheadRate.Update(fec_packet->length());
167 TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
168 "Video::PacketFec", "timestamp", capture_timestamp,
169 "seqnum", next_fec_sequence_number);
170 } else {
171 LOG(LS_WARNING) << "Failed to send FEC packet "
172 << next_fec_sequence_number;
173 }
174 delete fec_packet;
175 ++next_fec_sequence_number;
176 }
177 }
178
SetGenericFECStatus(const bool enable,const uint8_t payloadTypeRED,const uint8_t payloadTypeFEC)179 void RTPSenderVideo::SetGenericFECStatus(const bool enable,
180 const uint8_t payloadTypeRED,
181 const uint8_t payloadTypeFEC) {
182 CriticalSectionScoped cs(crit_.get());
183 fec_enabled_ = enable;
184 red_payload_type_ = payloadTypeRED;
185 fec_payload_type_ = payloadTypeFEC;
186 memset(&delta_fec_params_, 0, sizeof(delta_fec_params_));
187 memset(&key_fec_params_, 0, sizeof(key_fec_params_));
188 delta_fec_params_.max_fec_frames = key_fec_params_.max_fec_frames = 1;
189 delta_fec_params_.fec_mask_type = key_fec_params_.fec_mask_type =
190 kFecMaskRandom;
191 }
192
GenericFECStatus(bool * enable,uint8_t * payloadTypeRED,uint8_t * payloadTypeFEC) const193 void RTPSenderVideo::GenericFECStatus(bool* enable,
194 uint8_t* payloadTypeRED,
195 uint8_t* payloadTypeFEC) const {
196 CriticalSectionScoped cs(crit_.get());
197 *enable = fec_enabled_;
198 *payloadTypeRED = red_payload_type_;
199 *payloadTypeFEC = fec_payload_type_;
200 }
201
FECPacketOverhead() const202 size_t RTPSenderVideo::FECPacketOverhead() const {
203 CriticalSectionScoped cs(crit_.get());
204 if (fec_enabled_) {
205 // Overhead is FEC headers plus RED for FEC header plus anything in RTP
206 // header beyond the 12 bytes base header (CSRC list, extensions...)
207 // This reason for the header extensions to be included here is that
208 // from an FEC viewpoint, they are part of the payload to be protected.
209 // (The base RTP header is already protected by the FEC header.)
210 return ForwardErrorCorrection::PacketOverhead() + REDForFECHeaderLength +
211 (_rtpSender.RTPHeaderLength() - kRtpHeaderSize);
212 }
213 return 0;
214 }
215
SetFecParameters(const FecProtectionParams * delta_params,const FecProtectionParams * key_params)216 void RTPSenderVideo::SetFecParameters(const FecProtectionParams* delta_params,
217 const FecProtectionParams* key_params) {
218 CriticalSectionScoped cs(crit_.get());
219 RTC_DCHECK(delta_params);
220 RTC_DCHECK(key_params);
221 delta_fec_params_ = *delta_params;
222 key_fec_params_ = *key_params;
223 }
224
SendVideo(const RtpVideoCodecTypes videoType,const FrameType frameType,const int8_t payloadType,const uint32_t captureTimeStamp,int64_t capture_time_ms,const uint8_t * payloadData,const size_t payloadSize,const RTPFragmentationHeader * fragmentation,const RTPVideoHeader * rtpHdr)225 int32_t RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType,
226 const FrameType frameType,
227 const int8_t payloadType,
228 const uint32_t captureTimeStamp,
229 int64_t capture_time_ms,
230 const uint8_t* payloadData,
231 const size_t payloadSize,
232 const RTPFragmentationHeader* fragmentation,
233 const RTPVideoHeader* rtpHdr) {
234 if (payloadSize == 0) {
235 return -1;
236 }
237
238 rtc::scoped_ptr<RtpPacketizer> packetizer(
239 RtpPacketizer::Create(videoType, _rtpSender.MaxDataPayloadLength(),
240 &(rtpHdr->codecHeader), frameType));
241
242 StorageType storage;
243 bool fec_enabled;
244 {
245 CriticalSectionScoped cs(crit_.get());
246 FecProtectionParams* fec_params =
247 frameType == kVideoFrameKey ? &key_fec_params_ : &delta_fec_params_;
248 producer_fec_.SetFecParameters(fec_params, 0);
249 storage = packetizer->GetStorageType(_retransmissionSettings);
250 fec_enabled = fec_enabled_;
251 }
252
253 // Register CVO rtp header extension at the first time when we receive a frame
254 // with pending rotation.
255 RTPSenderInterface::CVOMode cvo_mode = RTPSenderInterface::kCVONone;
256 if (rtpHdr && rtpHdr->rotation != kVideoRotation_0) {
257 cvo_mode = _rtpSender.ActivateCVORtpHeaderExtension();
258 }
259
260 uint16_t rtp_header_length = _rtpSender.RTPHeaderLength();
261 size_t payload_bytes_to_send = payloadSize;
262 const uint8_t* data = payloadData;
263
264 // TODO(changbin): we currently don't support to configure the codec to
265 // output multiple partitions for VP8. Should remove below check after the
266 // issue is fixed.
267 const RTPFragmentationHeader* frag =
268 (videoType == kRtpVideoVp8) ? NULL : fragmentation;
269
270 packetizer->SetPayloadData(data, payload_bytes_to_send, frag);
271
272 bool last = false;
273 while (!last) {
274 uint8_t dataBuffer[IP_PACKET_SIZE] = {0};
275 size_t payload_bytes_in_packet = 0;
276 if (!packetizer->NextPacket(&dataBuffer[rtp_header_length],
277 &payload_bytes_in_packet, &last)) {
278 return -1;
279 }
280 // Write RTP header.
281 // Set marker bit true if this is the last packet in frame.
282 _rtpSender.BuildRTPheader(
283 dataBuffer, payloadType, last, captureTimeStamp, capture_time_ms);
284 // According to
285 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
286 // ts_126114v120700p.pdf Section 7.4.5:
287 // The MTSI client shall add the payload bytes as defined in this clause
288 // onto the last RTP packet in each group of packets which make up a key
289 // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
290 // (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
291 // packet in each group of packets which make up another type of frame
292 // (e.g. a P-Frame) only if the current value is different from the previous
293 // value sent.
294 // Here we are adding it to every packet of every frame at this point.
295 if (!rtpHdr) {
296 RTC_DCHECK(!_rtpSender.IsRtpHeaderExtensionRegistered(
297 kRtpExtensionVideoRotation));
298 } else if (cvo_mode == RTPSenderInterface::kCVOActivated) {
299 // Checking whether CVO header extension is registered will require taking
300 // a lock. It'll be a no-op if it's not registered.
301 // TODO(guoweis): For now, all packets sent will carry the CVO such that
302 // the RTP header length is consistent, although the receiver side will
303 // only exam the packets with marker bit set.
304 size_t packetSize = payloadSize + rtp_header_length;
305 RtpUtility::RtpHeaderParser rtp_parser(dataBuffer, packetSize);
306 RTPHeader rtp_header;
307 rtp_parser.Parse(&rtp_header);
308 _rtpSender.UpdateVideoRotation(dataBuffer, packetSize, rtp_header,
309 rtpHdr->rotation);
310 }
311 if (fec_enabled) {
312 SendVideoPacketAsRed(dataBuffer, payload_bytes_in_packet,
313 rtp_header_length, _rtpSender.SequenceNumber(),
314 captureTimeStamp, capture_time_ms, storage,
315 packetizer->GetProtectionType() == kProtectedPacket);
316 } else {
317 SendVideoPacket(dataBuffer, payload_bytes_in_packet, rtp_header_length,
318 _rtpSender.SequenceNumber(), captureTimeStamp,
319 capture_time_ms, storage);
320 }
321 }
322
323 TRACE_EVENT_ASYNC_END1(
324 "webrtc", "Video", capture_time_ms, "timestamp", _rtpSender.Timestamp());
325 return 0;
326 }
327
SetMaxConfiguredBitrateVideo(const uint32_t maxBitrate)328 void RTPSenderVideo::SetMaxConfiguredBitrateVideo(const uint32_t maxBitrate) {
329 _maxBitrate = maxBitrate;
330 }
331
MaxConfiguredBitrateVideo() const332 uint32_t RTPSenderVideo::MaxConfiguredBitrateVideo() const {
333 return _maxBitrate;
334 }
335
ProcessBitrate()336 void RTPSenderVideo::ProcessBitrate() {
337 _videoBitrate.Process();
338 _fecOverheadRate.Process();
339 }
340
VideoBitrateSent() const341 uint32_t RTPSenderVideo::VideoBitrateSent() const {
342 return _videoBitrate.BitrateLast();
343 }
344
FecOverheadRate() const345 uint32_t RTPSenderVideo::FecOverheadRate() const {
346 return _fecOverheadRate.BitrateLast();
347 }
348
SelectiveRetransmissions() const349 int RTPSenderVideo::SelectiveRetransmissions() const {
350 CriticalSectionScoped cs(crit_.get());
351 return _retransmissionSettings;
352 }
353
SetSelectiveRetransmissions(uint8_t settings)354 void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) {
355 CriticalSectionScoped cs(crit_.get());
356 _retransmissionSettings = settings;
357 }
358
359 } // namespace webrtc
360