1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/video/video_send_stream.h"
12
13 #include <sstream>
14 #include <string>
15 #include <vector>
16
17 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
18 #include "webrtc/system_wrappers/interface/logging.h"
19 #include "webrtc/video_engine/include/vie_base.h"
20 #include "webrtc/video_engine/include/vie_capture.h"
21 #include "webrtc/video_engine/include/vie_codec.h"
22 #include "webrtc/video_engine/include/vie_external_codec.h"
23 #include "webrtc/video_engine/include/vie_image_process.h"
24 #include "webrtc/video_engine/include/vie_network.h"
25 #include "webrtc/video_engine/include/vie_rtp_rtcp.h"
26 #include "webrtc/video_engine/vie_defines.h"
27 #include "webrtc/video_send_stream.h"
28
29 namespace webrtc {
30 std::string
ToString() const31 VideoSendStream::Config::EncoderSettings::ToString() const {
32 std::stringstream ss;
33 ss << "{payload_name: " << payload_name;
34 ss << ", payload_type: " << payload_type;
35 if (encoder != NULL)
36 ss << ", encoder: " << (encoder != NULL ? "(encoder)" : "NULL");
37 ss << '}';
38 return ss.str();
39 }
40
ToString() const41 std::string VideoSendStream::Config::Rtp::Rtx::ToString()
42 const {
43 std::stringstream ss;
44 ss << "{ssrcs: {";
45 for (size_t i = 0; i < ssrcs.size(); ++i) {
46 ss << ssrcs[i];
47 if (i != ssrcs.size() - 1)
48 ss << "}, {";
49 }
50 ss << '}';
51
52 ss << ", payload_type: " << payload_type;
53 ss << '}';
54 return ss.str();
55 }
56
ToString() const57 std::string VideoSendStream::Config::Rtp::ToString() const {
58 std::stringstream ss;
59 ss << "{ssrcs: {";
60 for (size_t i = 0; i < ssrcs.size(); ++i) {
61 ss << ssrcs[i];
62 if (i != ssrcs.size() - 1)
63 ss << "}, {";
64 }
65 ss << '}';
66
67 ss << ", max_packet_size: " << max_packet_size;
68 if (min_transmit_bitrate_bps != 0)
69 ss << ", min_transmit_bitrate_bps: " << min_transmit_bitrate_bps;
70
71 ss << ", extensions: {";
72 for (size_t i = 0; i < extensions.size(); ++i) {
73 ss << extensions[i].ToString();
74 if (i != extensions.size() - 1)
75 ss << "}, {";
76 }
77 ss << '}';
78
79 if (nack.rtp_history_ms != 0)
80 ss << ", nack.rtp_history_ms: " << nack.rtp_history_ms;
81 if (fec.ulpfec_payload_type != -1 || fec.red_payload_type != -1)
82 ss << ", fec: " << fec.ToString();
83 if (rtx.payload_type != 0 || !rtx.ssrcs.empty())
84 ss << ", rtx: " << rtx.ToString();
85 if (c_name != "")
86 ss << ", c_name: " << c_name;
87 ss << '}';
88 return ss.str();
89 }
90
ToString() const91 std::string VideoSendStream::Config::ToString() const {
92 std::stringstream ss;
93 ss << "{encoder_settings: " << encoder_settings.ToString();
94 ss << ", rtp: " << rtp.ToString();
95 if (pre_encode_callback != NULL)
96 ss << ", (pre_encode_callback)";
97 if (post_encode_callback != NULL)
98 ss << ", (post_encode_callback)";
99 if (local_renderer != NULL) {
100 ss << ", (local_renderer, render_delay_ms: " << render_delay_ms << ")";
101 }
102 if (target_delay_ms > 0)
103 ss << ", target_delay_ms: " << target_delay_ms;
104 if (suspend_below_min_bitrate)
105 ss << ", suspend_below_min_bitrate: on";
106 ss << '}';
107 return ss.str();
108 }
109
110 namespace internal {
VideoSendStream(newapi::Transport * transport,CpuOveruseObserver * overuse_observer,webrtc::VideoEngine * video_engine,const VideoSendStream::Config & config,const std::vector<VideoStream> video_streams,const void * encoder_settings,int base_channel,int start_bitrate_bps)111 VideoSendStream::VideoSendStream(newapi::Transport* transport,
112 CpuOveruseObserver* overuse_observer,
113 webrtc::VideoEngine* video_engine,
114 const VideoSendStream::Config& config,
115 const std::vector<VideoStream> video_streams,
116 const void* encoder_settings,
117 int base_channel,
118 int start_bitrate_bps)
119 : transport_adapter_(transport),
120 encoded_frame_proxy_(config.post_encode_callback),
121 config_(config),
122 start_bitrate_bps_(start_bitrate_bps),
123 external_codec_(NULL),
124 channel_(-1),
125 stats_proxy_(new SendStatisticsProxy(config, this)) {
126 video_engine_base_ = ViEBase::GetInterface(video_engine);
127 video_engine_base_->CreateChannel(channel_, base_channel);
128 assert(channel_ != -1);
129 assert(start_bitrate_bps_ > 0);
130
131 rtp_rtcp_ = ViERTP_RTCP::GetInterface(video_engine);
132 assert(rtp_rtcp_ != NULL);
133
134 assert(config_.rtp.ssrcs.size() > 0);
135
136 assert(config_.rtp.min_transmit_bitrate_bps >= 0);
137 rtp_rtcp_->SetMinTransmitBitrate(channel_,
138 config_.rtp.min_transmit_bitrate_bps / 1000);
139
140 for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) {
141 const std::string& extension = config_.rtp.extensions[i].name;
142 int id = config_.rtp.extensions[i].id;
143 if (extension == RtpExtension::kTOffset) {
144 if (rtp_rtcp_->SetSendTimestampOffsetStatus(channel_, true, id) != 0)
145 abort();
146 } else if (extension == RtpExtension::kAbsSendTime) {
147 if (rtp_rtcp_->SetSendAbsoluteSendTimeStatus(channel_, true, id) != 0)
148 abort();
149 } else {
150 abort(); // Unsupported extension.
151 }
152 }
153
154 rtp_rtcp_->SetRembStatus(channel_, true, false);
155
156 // Enable NACK, FEC or both.
157 if (config_.rtp.fec.red_payload_type != -1) {
158 assert(config_.rtp.fec.ulpfec_payload_type != -1);
159 if (config_.rtp.nack.rtp_history_ms > 0) {
160 rtp_rtcp_->SetHybridNACKFECStatus(
161 channel_,
162 true,
163 static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
164 static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
165 } else {
166 rtp_rtcp_->SetFECStatus(
167 channel_,
168 true,
169 static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
170 static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
171 }
172 } else {
173 rtp_rtcp_->SetNACKStatus(channel_, config_.rtp.nack.rtp_history_ms > 0);
174 }
175
176 char rtcp_cname[ViERTP_RTCP::KMaxRTCPCNameLength];
177 assert(config_.rtp.c_name.length() < ViERTP_RTCP::KMaxRTCPCNameLength);
178 strncpy(rtcp_cname, config_.rtp.c_name.c_str(), sizeof(rtcp_cname) - 1);
179 rtcp_cname[sizeof(rtcp_cname) - 1] = '\0';
180
181 rtp_rtcp_->SetRTCPCName(channel_, rtcp_cname);
182
183 capture_ = ViECapture::GetInterface(video_engine);
184 capture_->AllocateExternalCaptureDevice(capture_id_, external_capture_);
185 capture_->ConnectCaptureDevice(capture_id_, channel_);
186
187 network_ = ViENetwork::GetInterface(video_engine);
188 assert(network_ != NULL);
189
190 network_->RegisterSendTransport(channel_, transport_adapter_);
191 // 28 to match packet overhead in ModuleRtpRtcpImpl.
192 network_->SetMTU(channel_,
193 static_cast<unsigned int>(config_.rtp.max_packet_size + 28));
194
195 assert(config.encoder_settings.encoder != NULL);
196 assert(config.encoder_settings.payload_type >= 0);
197 assert(config.encoder_settings.payload_type <= 127);
198 external_codec_ = ViEExternalCodec::GetInterface(video_engine);
199 if (external_codec_->RegisterExternalSendCodec(
200 channel_,
201 config.encoder_settings.payload_type,
202 config.encoder_settings.encoder,
203 false) != 0) {
204 abort();
205 }
206
207 codec_ = ViECodec::GetInterface(video_engine);
208 if (!ReconfigureVideoEncoder(video_streams, encoder_settings))
209 abort();
210
211 if (overuse_observer)
212 video_engine_base_->RegisterCpuOveruseObserver(channel_, overuse_observer);
213
214 image_process_ = ViEImageProcess::GetInterface(video_engine);
215 image_process_->RegisterPreEncodeCallback(channel_,
216 config_.pre_encode_callback);
217 if (config_.post_encode_callback) {
218 image_process_->RegisterPostEncodeImageCallback(channel_,
219 &encoded_frame_proxy_);
220 }
221
222 if (config_.suspend_below_min_bitrate)
223 codec_->SuspendBelowMinBitrate(channel_);
224
225 rtp_rtcp_->RegisterSendChannelRtcpStatisticsCallback(channel_,
226 stats_proxy_.get());
227 rtp_rtcp_->RegisterSendChannelRtpStatisticsCallback(channel_,
228 stats_proxy_.get());
229 rtp_rtcp_->RegisterSendBitrateObserver(channel_, stats_proxy_.get());
230 rtp_rtcp_->RegisterSendFrameCountObserver(channel_, stats_proxy_.get());
231
232 codec_->RegisterEncoderObserver(channel_, *stats_proxy_);
233 capture_->RegisterObserver(capture_id_, *stats_proxy_);
234 }
235
~VideoSendStream()236 VideoSendStream::~VideoSendStream() {
237 capture_->DeregisterObserver(capture_id_);
238 codec_->DeregisterEncoderObserver(channel_);
239
240 rtp_rtcp_->DeregisterSendFrameCountObserver(channel_, stats_proxy_.get());
241 rtp_rtcp_->DeregisterSendBitrateObserver(channel_, stats_proxy_.get());
242 rtp_rtcp_->DeregisterSendChannelRtpStatisticsCallback(channel_,
243 stats_proxy_.get());
244 rtp_rtcp_->DeregisterSendChannelRtcpStatisticsCallback(channel_,
245 stats_proxy_.get());
246
247 image_process_->DeRegisterPreEncodeCallback(channel_);
248
249 network_->DeregisterSendTransport(channel_);
250
251 capture_->DisconnectCaptureDevice(channel_);
252 capture_->ReleaseCaptureDevice(capture_id_);
253
254 external_codec_->DeRegisterExternalSendCodec(
255 channel_, config_.encoder_settings.payload_type);
256
257 video_engine_base_->DeleteChannel(channel_);
258
259 image_process_->Release();
260 video_engine_base_->Release();
261 capture_->Release();
262 codec_->Release();
263 if (external_codec_)
264 external_codec_->Release();
265 network_->Release();
266 rtp_rtcp_->Release();
267 }
268
SwapFrame(I420VideoFrame * frame)269 void VideoSendStream::SwapFrame(I420VideoFrame* frame) {
270 // TODO(pbos): Local rendering should not be done on the capture thread.
271 if (config_.local_renderer != NULL)
272 config_.local_renderer->RenderFrame(*frame, 0);
273
274 external_capture_->SwapFrame(frame);
275 }
276
Input()277 VideoSendStreamInput* VideoSendStream::Input() { return this; }
278
Start()279 void VideoSendStream::Start() {
280 transport_adapter_.Enable();
281 video_engine_base_->StartSend(channel_);
282 video_engine_base_->StartReceive(channel_);
283 }
284
Stop()285 void VideoSendStream::Stop() {
286 video_engine_base_->StopSend(channel_);
287 video_engine_base_->StopReceive(channel_);
288 transport_adapter_.Disable();
289 }
290
ReconfigureVideoEncoder(const std::vector<VideoStream> & streams,const void * encoder_settings)291 bool VideoSendStream::ReconfigureVideoEncoder(
292 const std::vector<VideoStream>& streams,
293 const void* encoder_settings) {
294 assert(!streams.empty());
295 assert(config_.rtp.ssrcs.size() >= streams.size());
296 // TODO(pbos): Wire encoder_settings.
297 assert(encoder_settings == NULL);
298
299 VideoCodec video_codec;
300 memset(&video_codec, 0, sizeof(video_codec));
301 video_codec.codecType =
302 (config_.encoder_settings.payload_name == "VP8" ? kVideoCodecVP8
303 : kVideoCodecGeneric);
304
305 if (video_codec.codecType == kVideoCodecVP8) {
306 video_codec.codecSpecific.VP8.resilience = kResilientStream;
307 video_codec.codecSpecific.VP8.numberOfTemporalLayers = 1;
308 video_codec.codecSpecific.VP8.denoisingOn = true;
309 video_codec.codecSpecific.VP8.errorConcealmentOn = false;
310 video_codec.codecSpecific.VP8.automaticResizeOn = false;
311 video_codec.codecSpecific.VP8.frameDroppingOn = true;
312 video_codec.codecSpecific.VP8.keyFrameInterval = 3000;
313 }
314
315 strncpy(video_codec.plName,
316 config_.encoder_settings.payload_name.c_str(),
317 kPayloadNameSize - 1);
318 video_codec.plName[kPayloadNameSize - 1] = '\0';
319 video_codec.plType = config_.encoder_settings.payload_type;
320 video_codec.numberOfSimulcastStreams =
321 static_cast<unsigned char>(streams.size());
322 video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
323 assert(streams.size() <= kMaxSimulcastStreams);
324 for (size_t i = 0; i < streams.size(); ++i) {
325 SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
326 assert(streams[i].width > 0);
327 assert(streams[i].height > 0);
328 assert(streams[i].max_framerate > 0);
329 // Different framerates not supported per stream at the moment.
330 assert(streams[i].max_framerate == streams[0].max_framerate);
331 assert(streams[i].min_bitrate_bps >= 0);
332 assert(streams[i].target_bitrate_bps >= streams[i].min_bitrate_bps);
333 assert(streams[i].max_bitrate_bps >= streams[i].target_bitrate_bps);
334 assert(streams[i].max_qp >= 0);
335
336 sim_stream->width = static_cast<unsigned short>(streams[i].width);
337 sim_stream->height = static_cast<unsigned short>(streams[i].height);
338 sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
339 sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
340 sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
341 sim_stream->qpMax = streams[i].max_qp;
342 // TODO(pbos): Implement mapping for temporal layers.
343 assert(streams[i].temporal_layers.empty());
344
345 video_codec.width = std::max(video_codec.width,
346 static_cast<unsigned short>(streams[i].width));
347 video_codec.height = std::max(
348 video_codec.height, static_cast<unsigned short>(streams[i].height));
349 video_codec.minBitrate =
350 std::min(video_codec.minBitrate,
351 static_cast<unsigned int>(streams[i].min_bitrate_bps / 1000));
352 video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
353 video_codec.qpMax = std::max(video_codec.qpMax,
354 static_cast<unsigned int>(streams[i].max_qp));
355 }
356 video_codec.startBitrate =
357 static_cast<unsigned int>(start_bitrate_bps_) / 1000;
358
359 if (video_codec.minBitrate < kViEMinCodecBitrate)
360 video_codec.minBitrate = kViEMinCodecBitrate;
361 if (video_codec.maxBitrate < kViEMinCodecBitrate)
362 video_codec.maxBitrate = kViEMinCodecBitrate;
363 if (video_codec.startBitrate < video_codec.minBitrate)
364 video_codec.startBitrate = video_codec.minBitrate;
365 if (video_codec.startBitrate > video_codec.maxBitrate)
366 video_codec.startBitrate = video_codec.maxBitrate;
367
368 if (video_codec.startBitrate < video_codec.minBitrate)
369 video_codec.startBitrate = video_codec.minBitrate;
370 if (video_codec.startBitrate > video_codec.maxBitrate)
371 video_codec.startBitrate = video_codec.maxBitrate;
372
373 assert(streams[0].max_framerate > 0);
374 video_codec.maxFramerate = streams[0].max_framerate;
375
376 if (codec_->SetSendCodec(channel_, video_codec) != 0)
377 return false;
378
379 for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
380 rtp_rtcp_->SetLocalSSRC(channel_,
381 config_.rtp.ssrcs[i],
382 kViEStreamTypeNormal,
383 static_cast<unsigned char>(i));
384 }
385
386 if (config_.rtp.rtx.ssrcs.empty()) {
387 assert(!config_.rtp.rtx.pad_with_redundant_payloads);
388 return true;
389 }
390
391 // Set up RTX.
392 assert(config_.rtp.rtx.ssrcs.size() == config_.rtp.ssrcs.size());
393 for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
394 rtp_rtcp_->SetLocalSSRC(channel_,
395 config_.rtp.rtx.ssrcs[i],
396 kViEStreamTypeRtx,
397 static_cast<unsigned char>(i));
398 }
399
400 if (config_.rtp.rtx.pad_with_redundant_payloads) {
401 rtp_rtcp_->SetPadWithRedundantPayloads(channel_, true);
402 }
403
404 assert(config_.rtp.rtx.payload_type >= 0);
405 rtp_rtcp_->SetRtxSendPayloadType(channel_, config_.rtp.rtx.payload_type);
406
407 return true;
408 }
409
DeliverRtcp(const uint8_t * packet,size_t length)410 bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
411 return network_->ReceivedRTCPPacket(
412 channel_, packet, static_cast<int>(length)) == 0;
413 }
414
GetStats() const415 VideoSendStream::Stats VideoSendStream::GetStats() const {
416 return stats_proxy_->GetStats();
417 }
418
GetSendSideDelay(VideoSendStream::Stats * stats)419 bool VideoSendStream::GetSendSideDelay(VideoSendStream::Stats* stats) {
420 return codec_->GetSendSideDelay(
421 channel_, &stats->avg_delay_ms, &stats->max_delay_ms);
422 }
423
GetCName()424 std::string VideoSendStream::GetCName() {
425 char rtcp_cname[ViERTP_RTCP::KMaxRTCPCNameLength];
426 rtp_rtcp_->GetRTCPCName(channel_, rtcp_cname);
427 return rtcp_cname;
428 }
429
430 } // namespace internal
431 } // namespace webrtc
432