1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/video/video_send_stream.h"
12
13 #include <algorithm>
14 #include <sstream>
15 #include <string>
16 #include <vector>
17
18 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
19 #include "webrtc/system_wrappers/interface/logging.h"
20 #include "webrtc/video_engine/include/vie_base.h"
21 #include "webrtc/video_engine/include/vie_capture.h"
22 #include "webrtc/video_engine/include/vie_codec.h"
23 #include "webrtc/video_engine/include/vie_external_codec.h"
24 #include "webrtc/video_engine/include/vie_image_process.h"
25 #include "webrtc/video_engine/include/vie_network.h"
26 #include "webrtc/video_engine/include/vie_rtp_rtcp.h"
27 #include "webrtc/video_engine/vie_defines.h"
28 #include "webrtc/video_send_stream.h"
29
30 namespace webrtc {
31 std::string
ToString() const32 VideoSendStream::Config::EncoderSettings::ToString() const {
33 std::stringstream ss;
34 ss << "{payload_name: " << payload_name;
35 ss << ", payload_type: " << payload_type;
36 if (encoder != NULL)
37 ss << ", encoder: " << (encoder != NULL ? "(encoder)" : "NULL");
38 ss << '}';
39 return ss.str();
40 }
41
ToString() const42 std::string VideoSendStream::Config::Rtp::Rtx::ToString()
43 const {
44 std::stringstream ss;
45 ss << "{ssrcs: {";
46 for (size_t i = 0; i < ssrcs.size(); ++i) {
47 ss << ssrcs[i];
48 if (i != ssrcs.size() - 1)
49 ss << "}, {";
50 }
51 ss << '}';
52
53 ss << ", payload_type: " << payload_type;
54 ss << '}';
55 return ss.str();
56 }
57
ToString() const58 std::string VideoSendStream::Config::Rtp::ToString() const {
59 std::stringstream ss;
60 ss << "{ssrcs: {";
61 for (size_t i = 0; i < ssrcs.size(); ++i) {
62 ss << ssrcs[i];
63 if (i != ssrcs.size() - 1)
64 ss << "}, {";
65 }
66 ss << '}';
67
68 ss << ", max_packet_size: " << max_packet_size;
69 if (min_transmit_bitrate_bps != 0)
70 ss << ", min_transmit_bitrate_bps: " << min_transmit_bitrate_bps;
71
72 ss << ", extensions: {";
73 for (size_t i = 0; i < extensions.size(); ++i) {
74 ss << extensions[i].ToString();
75 if (i != extensions.size() - 1)
76 ss << "}, {";
77 }
78 ss << '}';
79
80 if (nack.rtp_history_ms != 0)
81 ss << ", nack.rtp_history_ms: " << nack.rtp_history_ms;
82 if (fec.ulpfec_payload_type != -1 || fec.red_payload_type != -1)
83 ss << ", fec: " << fec.ToString();
84 if (rtx.payload_type != 0 || !rtx.ssrcs.empty())
85 ss << ", rtx: " << rtx.ToString();
86 if (c_name != "")
87 ss << ", c_name: " << c_name;
88 ss << '}';
89 return ss.str();
90 }
91
ToString() const92 std::string VideoSendStream::Config::ToString() const {
93 std::stringstream ss;
94 ss << "{encoder_settings: " << encoder_settings.ToString();
95 ss << ", rtp: " << rtp.ToString();
96 if (pre_encode_callback != NULL)
97 ss << ", (pre_encode_callback)";
98 if (post_encode_callback != NULL)
99 ss << ", (post_encode_callback)";
100 if (local_renderer != NULL) {
101 ss << ", (local_renderer, render_delay_ms: " << render_delay_ms << ")";
102 }
103 if (target_delay_ms > 0)
104 ss << ", target_delay_ms: " << target_delay_ms;
105 if (suspend_below_min_bitrate)
106 ss << ", suspend_below_min_bitrate: on";
107 ss << '}';
108 return ss.str();
109 }
110
111 namespace internal {
VideoSendStream(newapi::Transport * transport,CpuOveruseObserver * overuse_observer,webrtc::VideoEngine * video_engine,const VideoSendStream::Config & config,const VideoEncoderConfig & encoder_config,const std::map<uint32_t,RtpState> & suspended_ssrcs,int base_channel,int start_bitrate_bps)112 VideoSendStream::VideoSendStream(
113 newapi::Transport* transport,
114 CpuOveruseObserver* overuse_observer,
115 webrtc::VideoEngine* video_engine,
116 const VideoSendStream::Config& config,
117 const VideoEncoderConfig& encoder_config,
118 const std::map<uint32_t, RtpState>& suspended_ssrcs,
119 int base_channel,
120 int start_bitrate_bps)
121 : transport_adapter_(transport),
122 encoded_frame_proxy_(config.post_encode_callback),
123 config_(config),
124 start_bitrate_bps_(start_bitrate_bps),
125 suspended_ssrcs_(suspended_ssrcs),
126 external_codec_(NULL),
127 channel_(-1),
128 stats_proxy_(config) {
129 video_engine_base_ = ViEBase::GetInterface(video_engine);
130 video_engine_base_->CreateChannel(channel_, base_channel);
131 assert(channel_ != -1);
132 assert(start_bitrate_bps_ > 0);
133
134 rtp_rtcp_ = ViERTP_RTCP::GetInterface(video_engine);
135 assert(rtp_rtcp_ != NULL);
136
137 assert(config_.rtp.ssrcs.size() > 0);
138
139 assert(config_.rtp.min_transmit_bitrate_bps >= 0);
140 rtp_rtcp_->SetMinTransmitBitrate(channel_,
141 config_.rtp.min_transmit_bitrate_bps / 1000);
142
143 for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) {
144 const std::string& extension = config_.rtp.extensions[i].name;
145 int id = config_.rtp.extensions[i].id;
146 if (extension == RtpExtension::kTOffset) {
147 if (rtp_rtcp_->SetSendTimestampOffsetStatus(channel_, true, id) != 0)
148 abort();
149 } else if (extension == RtpExtension::kAbsSendTime) {
150 if (rtp_rtcp_->SetSendAbsoluteSendTimeStatus(channel_, true, id) != 0)
151 abort();
152 } else {
153 abort(); // Unsupported extension.
154 }
155 }
156
157 rtp_rtcp_->SetRembStatus(channel_, true, false);
158
159 // Enable NACK, FEC or both.
160 if (config_.rtp.fec.red_payload_type != -1) {
161 assert(config_.rtp.fec.ulpfec_payload_type != -1);
162 if (config_.rtp.nack.rtp_history_ms > 0) {
163 rtp_rtcp_->SetHybridNACKFECStatus(
164 channel_,
165 true,
166 static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
167 static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
168 } else {
169 rtp_rtcp_->SetFECStatus(
170 channel_,
171 true,
172 static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
173 static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
174 }
175 } else {
176 rtp_rtcp_->SetNACKStatus(channel_, config_.rtp.nack.rtp_history_ms > 0);
177 }
178
179 ConfigureSsrcs();
180
181 char rtcp_cname[ViERTP_RTCP::KMaxRTCPCNameLength];
182 assert(config_.rtp.c_name.length() < ViERTP_RTCP::KMaxRTCPCNameLength);
183 strncpy(rtcp_cname, config_.rtp.c_name.c_str(), sizeof(rtcp_cname) - 1);
184 rtcp_cname[sizeof(rtcp_cname) - 1] = '\0';
185
186 rtp_rtcp_->SetRTCPCName(channel_, rtcp_cname);
187
188 capture_ = ViECapture::GetInterface(video_engine);
189 capture_->AllocateExternalCaptureDevice(capture_id_, external_capture_);
190 capture_->ConnectCaptureDevice(capture_id_, channel_);
191
192 network_ = ViENetwork::GetInterface(video_engine);
193 assert(network_ != NULL);
194
195 network_->RegisterSendTransport(channel_, transport_adapter_);
196 // 28 to match packet overhead in ModuleRtpRtcpImpl.
197 network_->SetMTU(channel_,
198 static_cast<unsigned int>(config_.rtp.max_packet_size + 28));
199
200 assert(config.encoder_settings.encoder != NULL);
201 assert(config.encoder_settings.payload_type >= 0);
202 assert(config.encoder_settings.payload_type <= 127);
203 external_codec_ = ViEExternalCodec::GetInterface(video_engine);
204 if (external_codec_->RegisterExternalSendCodec(
205 channel_,
206 config.encoder_settings.payload_type,
207 config.encoder_settings.encoder,
208 false) != 0) {
209 abort();
210 }
211
212 codec_ = ViECodec::GetInterface(video_engine);
213 if (!ReconfigureVideoEncoder(encoder_config))
214 abort();
215
216 if (overuse_observer)
217 video_engine_base_->RegisterCpuOveruseObserver(channel_, overuse_observer);
218
219 video_engine_base_->RegisterSendSideDelayObserver(channel_, &stats_proxy_);
220
221 image_process_ = ViEImageProcess::GetInterface(video_engine);
222 image_process_->RegisterPreEncodeCallback(channel_,
223 config_.pre_encode_callback);
224 if (config_.post_encode_callback) {
225 image_process_->RegisterPostEncodeImageCallback(channel_,
226 &encoded_frame_proxy_);
227 }
228
229 if (config_.suspend_below_min_bitrate)
230 codec_->SuspendBelowMinBitrate(channel_);
231
232 rtp_rtcp_->RegisterSendChannelRtcpStatisticsCallback(channel_,
233 &stats_proxy_);
234 rtp_rtcp_->RegisterSendChannelRtpStatisticsCallback(channel_,
235 &stats_proxy_);
236 rtp_rtcp_->RegisterSendBitrateObserver(channel_, &stats_proxy_);
237 rtp_rtcp_->RegisterSendFrameCountObserver(channel_, &stats_proxy_);
238
239 codec_->RegisterEncoderObserver(channel_, stats_proxy_);
240 capture_->RegisterObserver(capture_id_, stats_proxy_);
241 }
242
~VideoSendStream()243 VideoSendStream::~VideoSendStream() {
244 capture_->DeregisterObserver(capture_id_);
245 codec_->DeregisterEncoderObserver(channel_);
246
247 rtp_rtcp_->DeregisterSendFrameCountObserver(channel_, &stats_proxy_);
248 rtp_rtcp_->DeregisterSendBitrateObserver(channel_, &stats_proxy_);
249 rtp_rtcp_->DeregisterSendChannelRtpStatisticsCallback(channel_,
250 &stats_proxy_);
251 rtp_rtcp_->DeregisterSendChannelRtcpStatisticsCallback(channel_,
252 &stats_proxy_);
253
254 image_process_->DeRegisterPreEncodeCallback(channel_);
255
256 network_->DeregisterSendTransport(channel_);
257
258 capture_->DisconnectCaptureDevice(channel_);
259 capture_->ReleaseCaptureDevice(capture_id_);
260
261 external_codec_->DeRegisterExternalSendCodec(
262 channel_, config_.encoder_settings.payload_type);
263
264 video_engine_base_->DeleteChannel(channel_);
265
266 image_process_->Release();
267 video_engine_base_->Release();
268 capture_->Release();
269 codec_->Release();
270 if (external_codec_)
271 external_codec_->Release();
272 network_->Release();
273 rtp_rtcp_->Release();
274 }
275
SwapFrame(I420VideoFrame * frame)276 void VideoSendStream::SwapFrame(I420VideoFrame* frame) {
277 // TODO(pbos): Local rendering should not be done on the capture thread.
278 if (config_.local_renderer != NULL)
279 config_.local_renderer->RenderFrame(*frame, 0);
280
281 external_capture_->SwapFrame(frame);
282 }
283
Input()284 VideoSendStreamInput* VideoSendStream::Input() { return this; }
285
Start()286 void VideoSendStream::Start() {
287 transport_adapter_.Enable();
288 video_engine_base_->StartSend(channel_);
289 video_engine_base_->StartReceive(channel_);
290 }
291
Stop()292 void VideoSendStream::Stop() {
293 video_engine_base_->StopSend(channel_);
294 video_engine_base_->StopReceive(channel_);
295 transport_adapter_.Disable();
296 }
297
ReconfigureVideoEncoder(const VideoEncoderConfig & config)298 bool VideoSendStream::ReconfigureVideoEncoder(
299 const VideoEncoderConfig& config) {
300 const std::vector<VideoStream>& streams = config.streams;
301 assert(!streams.empty());
302 assert(config_.rtp.ssrcs.size() >= streams.size());
303
304 VideoCodec video_codec;
305 memset(&video_codec, 0, sizeof(video_codec));
306 if (config_.encoder_settings.payload_name == "VP8") {
307 video_codec.codecType = kVideoCodecVP8;
308 } else if (config_.encoder_settings.payload_name == "H264") {
309 video_codec.codecType = kVideoCodecH264;
310 } else {
311 video_codec.codecType = kVideoCodecGeneric;
312 }
313 switch (config.content_type) {
314 case VideoEncoderConfig::kRealtimeVideo:
315 video_codec.mode = kRealtimeVideo;
316 break;
317 case VideoEncoderConfig::kScreenshare:
318 video_codec.mode = kScreensharing;
319 break;
320 }
321
322 if (video_codec.codecType == kVideoCodecVP8) {
323 video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings();
324 } else if (video_codec.codecType == kVideoCodecH264) {
325 video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings();
326 }
327
328 if (video_codec.codecType == kVideoCodecVP8) {
329 if (config.encoder_specific_settings != NULL) {
330 video_codec.codecSpecific.VP8 = *reinterpret_cast<const VideoCodecVP8*>(
331 config.encoder_specific_settings);
332 }
333 video_codec.codecSpecific.VP8.numberOfTemporalLayers =
334 static_cast<unsigned char>(streams.back().temporal_layers.size());
335 } else {
336 // TODO(pbos): Support encoder_settings codec-agnostically.
337 assert(config.encoder_specific_settings == NULL);
338 }
339
340 strncpy(video_codec.plName,
341 config_.encoder_settings.payload_name.c_str(),
342 kPayloadNameSize - 1);
343 video_codec.plName[kPayloadNameSize - 1] = '\0';
344 video_codec.plType = config_.encoder_settings.payload_type;
345 video_codec.numberOfSimulcastStreams =
346 static_cast<unsigned char>(streams.size());
347 video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
348 assert(streams.size() <= kMaxSimulcastStreams);
349 for (size_t i = 0; i < streams.size(); ++i) {
350 SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
351 assert(streams[i].width > 0);
352 assert(streams[i].height > 0);
353 assert(streams[i].max_framerate > 0);
354 // Different framerates not supported per stream at the moment.
355 assert(streams[i].max_framerate == streams[0].max_framerate);
356 assert(streams[i].min_bitrate_bps >= 0);
357 assert(streams[i].target_bitrate_bps >= streams[i].min_bitrate_bps);
358 assert(streams[i].max_bitrate_bps >= streams[i].target_bitrate_bps);
359 assert(streams[i].max_qp >= 0);
360
361 sim_stream->width = static_cast<unsigned short>(streams[i].width);
362 sim_stream->height = static_cast<unsigned short>(streams[i].height);
363 sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
364 sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
365 sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
366 sim_stream->qpMax = streams[i].max_qp;
367 sim_stream->numberOfTemporalLayers =
368 static_cast<unsigned char>(streams[i].temporal_layers.size());
369
370 video_codec.width = std::max(video_codec.width,
371 static_cast<unsigned short>(streams[i].width));
372 video_codec.height = std::max(
373 video_codec.height, static_cast<unsigned short>(streams[i].height));
374 video_codec.minBitrate =
375 std::min(video_codec.minBitrate,
376 static_cast<unsigned int>(streams[i].min_bitrate_bps / 1000));
377 video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
378 video_codec.qpMax = std::max(video_codec.qpMax,
379 static_cast<unsigned int>(streams[i].max_qp));
380 }
381 video_codec.startBitrate =
382 static_cast<unsigned int>(start_bitrate_bps_) / 1000;
383
384 if (video_codec.minBitrate < kViEMinCodecBitrate)
385 video_codec.minBitrate = kViEMinCodecBitrate;
386 if (video_codec.maxBitrate < kViEMinCodecBitrate)
387 video_codec.maxBitrate = kViEMinCodecBitrate;
388 if (video_codec.startBitrate < video_codec.minBitrate)
389 video_codec.startBitrate = video_codec.minBitrate;
390 if (video_codec.startBitrate > video_codec.maxBitrate)
391 video_codec.startBitrate = video_codec.maxBitrate;
392
393 if (video_codec.startBitrate < video_codec.minBitrate)
394 video_codec.startBitrate = video_codec.minBitrate;
395 if (video_codec.startBitrate > video_codec.maxBitrate)
396 video_codec.startBitrate = video_codec.maxBitrate;
397
398 assert(streams[0].max_framerate > 0);
399 video_codec.maxFramerate = streams[0].max_framerate;
400
401 return codec_->SetSendCodec(channel_, video_codec) == 0;
402 }
403
DeliverRtcp(const uint8_t * packet,size_t length)404 bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
405 return network_->ReceivedRTCPPacket(
406 channel_, packet, static_cast<int>(length)) == 0;
407 }
408
GetStats() const409 VideoSendStream::Stats VideoSendStream::GetStats() const {
410 return stats_proxy_.GetStats();
411 }
412
ConfigureSsrcs()413 void VideoSendStream::ConfigureSsrcs() {
414 for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
415 uint32_t ssrc = config_.rtp.ssrcs[i];
416 rtp_rtcp_->SetLocalSSRC(
417 channel_, ssrc, kViEStreamTypeNormal, static_cast<unsigned char>(i));
418 RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
419 if (it != suspended_ssrcs_.end())
420 rtp_rtcp_->SetRtpStateForSsrc(channel_, ssrc, it->second);
421 }
422
423 if (config_.rtp.rtx.ssrcs.empty()) {
424 assert(!config_.rtp.rtx.pad_with_redundant_payloads);
425 return;
426 }
427
428 // Set up RTX.
429 assert(config_.rtp.rtx.ssrcs.size() == config_.rtp.ssrcs.size());
430 for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
431 uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
432 rtp_rtcp_->SetLocalSSRC(channel_,
433 config_.rtp.rtx.ssrcs[i],
434 kViEStreamTypeRtx,
435 static_cast<unsigned char>(i));
436 RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
437 if (it != suspended_ssrcs_.end())
438 rtp_rtcp_->SetRtpStateForSsrc(channel_, ssrc, it->second);
439 }
440
441 if (config_.rtp.rtx.pad_with_redundant_payloads) {
442 rtp_rtcp_->SetPadWithRedundantPayloads(channel_, true);
443 }
444
445 assert(config_.rtp.rtx.payload_type >= 0);
446 rtp_rtcp_->SetRtxSendPayloadType(channel_, config_.rtp.rtx.payload_type);
447 }
448
GetRtpStates() const449 std::map<uint32_t, RtpState> VideoSendStream::GetRtpStates() const {
450 std::map<uint32_t, RtpState> rtp_states;
451 for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
452 uint32_t ssrc = config_.rtp.ssrcs[i];
453 rtp_states[ssrc] = rtp_rtcp_->GetRtpStateForSsrc(channel_, ssrc);
454 }
455
456 for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
457 uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
458 rtp_states[ssrc] = rtp_rtcp_->GetRtpStateForSsrc(channel_, ssrc);
459 }
460
461 return rtp_states;
462 }
463
SignalNetworkState(Call::NetworkState state)464 void VideoSendStream::SignalNetworkState(Call::NetworkState state) {
465 // When network goes up, enable RTCP status before setting transmission state.
466 // When it goes down, disable RTCP afterwards. This ensures that any packets
467 // sent due to the network state changed will not be dropped.
468 if (state == Call::kNetworkUp)
469 rtp_rtcp_->SetRTCPStatus(channel_, kRtcpCompound_RFC4585);
470 network_->SetNetworkTransmissionState(channel_, state == Call::kNetworkUp);
471 if (state == Call::kNetworkDown)
472 rtp_rtcp_->SetRTCPStatus(channel_, kRtcpNone);
473 }
474
475 } // namespace internal
476 } // namespace webrtc
477