1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/video/video_receive_stream.h"
12
13 #include <stdlib.h>
14
15 #include <set>
16 #include <string>
17
18 #include "webrtc/base/checks.h"
19 #include "webrtc/base/logging.h"
20 #include "webrtc/call/congestion_controller.h"
21 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
22 #include "webrtc/system_wrappers/include/clock.h"
23 #include "webrtc/video/call_stats.h"
24 #include "webrtc/video/receive_statistics_proxy.h"
25 #include "webrtc/video_receive_stream.h"
26
27 namespace webrtc {
28
UseSendSideBwe(const std::vector<RtpExtension> & extensions)29 static bool UseSendSideBwe(const std::vector<RtpExtension>& extensions) {
30 for (const auto& extension : extensions) {
31 if (extension.name == RtpExtension::kTransportSequenceNumber)
32 return true;
33 }
34 return false;
35 }
36
ToString() const37 std::string VideoReceiveStream::Decoder::ToString() const {
38 std::stringstream ss;
39 ss << "{decoder: " << (decoder != nullptr ? "(VideoDecoder)" : "nullptr");
40 ss << ", payload_type: " << payload_type;
41 ss << ", payload_name: " << payload_name;
42 ss << '}';
43
44 return ss.str();
45 }
46
ToString() const47 std::string VideoReceiveStream::Config::ToString() const {
48 std::stringstream ss;
49 ss << "{decoders: [";
50 for (size_t i = 0; i < decoders.size(); ++i) {
51 ss << decoders[i].ToString();
52 if (i != decoders.size() - 1)
53 ss << ", ";
54 }
55 ss << ']';
56 ss << ", rtp: " << rtp.ToString();
57 ss << ", renderer: " << (renderer != nullptr ? "(renderer)" : "nullptr");
58 ss << ", render_delay_ms: " << render_delay_ms;
59 if (!sync_group.empty())
60 ss << ", sync_group: " << sync_group;
61 ss << ", pre_decode_callback: "
62 << (pre_decode_callback != nullptr ? "(EncodedFrameObserver)" : "nullptr");
63 ss << ", pre_render_callback: "
64 << (pre_render_callback != nullptr ? "(I420FrameCallback)" : "nullptr");
65 ss << ", target_delay_ms: " << target_delay_ms;
66 ss << '}';
67
68 return ss.str();
69 }
70
ToString() const71 std::string VideoReceiveStream::Config::Rtp::ToString() const {
72 std::stringstream ss;
73 ss << "{remote_ssrc: " << remote_ssrc;
74 ss << ", local_ssrc: " << local_ssrc;
75 ss << ", rtcp_mode: "
76 << (rtcp_mode == RtcpMode::kCompound ? "RtcpMode::kCompound"
77 : "RtcpMode::kReducedSize");
78 ss << ", rtcp_xr: ";
79 ss << "{receiver_reference_time_report: "
80 << (rtcp_xr.receiver_reference_time_report ? "on" : "off");
81 ss << '}';
82 ss << ", remb: " << (remb ? "on" : "off");
83 ss << ", transport_cc: " << (transport_cc ? "on" : "off");
84 ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}';
85 ss << ", fec: " << fec.ToString();
86 ss << ", rtx: {";
87 for (auto& kv : rtx) {
88 ss << kv.first << " -> ";
89 ss << "{ssrc: " << kv.second.ssrc;
90 ss << ", payload_type: " << kv.second.payload_type;
91 ss << '}';
92 }
93 ss << '}';
94 ss << ", extensions: [";
95 for (size_t i = 0; i < extensions.size(); ++i) {
96 ss << extensions[i].ToString();
97 if (i != extensions.size() - 1)
98 ss << ", ";
99 }
100 ss << ']';
101 ss << '}';
102 return ss.str();
103 }
104
105 namespace internal {
106 namespace {
107
CreateDecoderVideoCodec(const VideoReceiveStream::Decoder & decoder)108 VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) {
109 VideoCodec codec;
110 memset(&codec, 0, sizeof(codec));
111
112 codec.plType = decoder.payload_type;
113 strncpy(codec.plName, decoder.payload_name.c_str(), sizeof(codec.plName));
114 if (decoder.payload_name == "VP8") {
115 codec.codecType = kVideoCodecVP8;
116 } else if (decoder.payload_name == "VP9") {
117 codec.codecType = kVideoCodecVP9;
118 } else if (decoder.payload_name == "H264") {
119 codec.codecType = kVideoCodecH264;
120 } else {
121 codec.codecType = kVideoCodecGeneric;
122 }
123
124 if (codec.codecType == kVideoCodecVP8) {
125 codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings();
126 } else if (codec.codecType == kVideoCodecVP9) {
127 codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings();
128 } else if (codec.codecType == kVideoCodecH264) {
129 codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings();
130 }
131
132 codec.width = 320;
133 codec.height = 180;
134 codec.startBitrate = codec.minBitrate = codec.maxBitrate =
135 Call::Config::kDefaultStartBitrateBps / 1000;
136
137 return codec;
138 }
139 } // namespace
140
VideoReceiveStream(int num_cpu_cores,CongestionController * congestion_controller,const VideoReceiveStream::Config & config,webrtc::VoiceEngine * voice_engine,ProcessThread * process_thread,CallStats * call_stats)141 VideoReceiveStream::VideoReceiveStream(
142 int num_cpu_cores,
143 CongestionController* congestion_controller,
144 const VideoReceiveStream::Config& config,
145 webrtc::VoiceEngine* voice_engine,
146 ProcessThread* process_thread,
147 CallStats* call_stats)
148 : transport_adapter_(config.rtcp_send_transport),
149 encoded_frame_proxy_(config.pre_decode_callback),
150 config_(config),
151 clock_(Clock::GetRealTimeClock()),
152 congestion_controller_(congestion_controller),
153 call_stats_(call_stats) {
154 LOG(LS_INFO) << "VideoReceiveStream: " << config_.ToString();
155
156 bool send_side_bwe =
157 config.rtp.transport_cc && UseSendSideBwe(config_.rtp.extensions);
158
159 RemoteBitrateEstimator* bitrate_estimator =
160 congestion_controller_->GetRemoteBitrateEstimator(send_side_bwe);
161
162 vie_channel_.reset(new ViEChannel(
163 num_cpu_cores, &transport_adapter_, process_thread, nullptr,
164 nullptr, nullptr, bitrate_estimator, call_stats_->rtcp_rtt_stats(),
165 congestion_controller_->pacer(), congestion_controller_->packet_router(),
166 1, false));
167
168 RTC_CHECK(vie_channel_->Init() == 0);
169
170 // Register the channel to receive stats updates.
171 call_stats_->RegisterStatsObserver(vie_channel_->GetStatsObserver());
172
173 // TODO(pbos): This is not fine grained enough...
174 vie_channel_->SetProtectionMode(config_.rtp.nack.rtp_history_ms > 0, false,
175 -1, -1);
176 RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff)
177 << "A stream should not be configured with RTCP disabled. This value is "
178 "reserved for internal usage.";
179 vie_channel_->SetRTCPMode(config_.rtp.rtcp_mode);
180
181 RTC_DCHECK(config_.rtp.remote_ssrc != 0);
182 // TODO(pbos): What's an appropriate local_ssrc for receive-only streams?
183 RTC_DCHECK(config_.rtp.local_ssrc != 0);
184 RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc);
185
186 vie_channel_->SetSSRC(config_.rtp.local_ssrc, kViEStreamTypeNormal, 0);
187 // TODO(pbos): Support multiple RTX, per video payload.
188 Config::Rtp::RtxMap::const_iterator it = config_.rtp.rtx.begin();
189 for (; it != config_.rtp.rtx.end(); ++it) {
190 RTC_DCHECK(it->second.ssrc != 0);
191 RTC_DCHECK(it->second.payload_type != 0);
192
193 vie_channel_->SetRemoteSSRCType(kViEStreamTypeRtx, it->second.ssrc);
194 vie_channel_->SetRtxReceivePayloadType(it->second.payload_type, it->first);
195 }
196 // TODO(holmer): When Chrome no longer depends on this being false by default,
197 // always use the mapping and remove this whole codepath.
198 vie_channel_->SetUseRtxPayloadMappingOnRestore(
199 config_.rtp.use_rtx_payload_mapping_on_restore);
200
201 congestion_controller_->SetChannelRembStatus(false, config_.rtp.remb,
202 vie_channel_->rtp_rtcp());
203
204 for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) {
205 const std::string& extension = config_.rtp.extensions[i].name;
206 int id = config_.rtp.extensions[i].id;
207 // One-byte-extension local identifiers are in the range 1-14 inclusive.
208 RTC_DCHECK_GE(id, 1);
209 RTC_DCHECK_LE(id, 14);
210 if (extension == RtpExtension::kTOffset) {
211 RTC_CHECK_EQ(0, vie_channel_->SetReceiveTimestampOffsetStatus(true, id));
212 } else if (extension == RtpExtension::kAbsSendTime) {
213 RTC_CHECK_EQ(0, vie_channel_->SetReceiveAbsoluteSendTimeStatus(true, id));
214 } else if (extension == RtpExtension::kVideoRotation) {
215 RTC_CHECK_EQ(0, vie_channel_->SetReceiveVideoRotationStatus(true, id));
216 } else if (extension == RtpExtension::kTransportSequenceNumber) {
217 RTC_CHECK_EQ(0,
218 vie_channel_->SetReceiveTransportSequenceNumber(true, id));
219 } else {
220 RTC_NOTREACHED() << "Unsupported RTP extension.";
221 }
222 }
223
224 if (config_.rtp.fec.ulpfec_payload_type != -1) {
225 // ULPFEC without RED doesn't make sense.
226 RTC_DCHECK(config_.rtp.fec.red_payload_type != -1);
227 VideoCodec codec;
228 memset(&codec, 0, sizeof(codec));
229 codec.codecType = kVideoCodecULPFEC;
230 strncpy(codec.plName, "ulpfec", sizeof(codec.plName));
231 codec.plType = config_.rtp.fec.ulpfec_payload_type;
232 RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec));
233 }
234 if (config_.rtp.fec.red_payload_type != -1) {
235 VideoCodec codec;
236 memset(&codec, 0, sizeof(codec));
237 codec.codecType = kVideoCodecRED;
238 strncpy(codec.plName, "red", sizeof(codec.plName));
239 codec.plType = config_.rtp.fec.red_payload_type;
240 RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec));
241 if (config_.rtp.fec.red_rtx_payload_type != -1) {
242 vie_channel_->SetRtxReceivePayloadType(
243 config_.rtp.fec.red_rtx_payload_type,
244 config_.rtp.fec.red_payload_type);
245 }
246 }
247
248 if (config.rtp.rtcp_xr.receiver_reference_time_report)
249 vie_channel_->SetRtcpXrRrtrStatus(true);
250
251 stats_proxy_.reset(
252 new ReceiveStatisticsProxy(config_.rtp.remote_ssrc, clock_));
253
254 vie_channel_->RegisterReceiveStatisticsProxy(stats_proxy_.get());
255 vie_channel_->RegisterReceiveChannelRtcpStatisticsCallback(
256 stats_proxy_.get());
257 vie_channel_->RegisterReceiveChannelRtpStatisticsCallback(stats_proxy_.get());
258 vie_channel_->RegisterRtcpPacketTypeCounterObserver(stats_proxy_.get());
259
260 RTC_DCHECK(!config_.decoders.empty());
261 std::set<int> decoder_payload_types;
262 for (size_t i = 0; i < config_.decoders.size(); ++i) {
263 const Decoder& decoder = config_.decoders[i];
264 RTC_CHECK(decoder.decoder);
265 RTC_CHECK(decoder_payload_types.find(decoder.payload_type) ==
266 decoder_payload_types.end())
267 << "Duplicate payload type (" << decoder.payload_type
268 << ") for different decoders.";
269 decoder_payload_types.insert(decoder.payload_type);
270 vie_channel_->RegisterExternalDecoder(decoder.payload_type,
271 decoder.decoder);
272
273 VideoCodec codec = CreateDecoderVideoCodec(decoder);
274
275 RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec));
276 }
277
278 incoming_video_stream_.reset(new IncomingVideoStream(
279 0, config.renderer ? config.renderer->SmoothsRenderedFrames() : false));
280 incoming_video_stream_->SetExpectedRenderDelay(config.render_delay_ms);
281 vie_channel_->SetExpectedRenderDelay(config.render_delay_ms);
282 incoming_video_stream_->SetExternalCallback(this);
283 vie_channel_->SetIncomingVideoStream(incoming_video_stream_.get());
284
285 vie_channel_->RegisterPreDecodeImageCallback(this);
286 vie_channel_->RegisterPreRenderCallback(this);
287 }
288
~VideoReceiveStream()289 VideoReceiveStream::~VideoReceiveStream() {
290 LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString();
291 incoming_video_stream_->Stop();
292 vie_channel_->RegisterPreRenderCallback(nullptr);
293 vie_channel_->RegisterPreDecodeImageCallback(nullptr);
294
295 call_stats_->DeregisterStatsObserver(vie_channel_->GetStatsObserver());
296 congestion_controller_->SetChannelRembStatus(false, false,
297 vie_channel_->rtp_rtcp());
298
299 uint32_t remote_ssrc = vie_channel_->GetRemoteSSRC();
300 bool send_side_bwe = UseSendSideBwe(config_.rtp.extensions);
301 congestion_controller_->GetRemoteBitrateEstimator(send_side_bwe)->
302 RemoveStream(remote_ssrc);
303 }
304
Start()305 void VideoReceiveStream::Start() {
306 transport_adapter_.Enable();
307 incoming_video_stream_->Start();
308 vie_channel_->StartReceive();
309 }
310
Stop()311 void VideoReceiveStream::Stop() {
312 incoming_video_stream_->Stop();
313 vie_channel_->StopReceive();
314 transport_adapter_.Disable();
315 }
316
SetSyncChannel(VoiceEngine * voice_engine,int audio_channel_id)317 void VideoReceiveStream::SetSyncChannel(VoiceEngine* voice_engine,
318 int audio_channel_id) {
319 if (voice_engine != nullptr && audio_channel_id != -1) {
320 VoEVideoSync* voe_sync_interface = VoEVideoSync::GetInterface(voice_engine);
321 vie_channel_->SetVoiceChannel(audio_channel_id, voe_sync_interface);
322 voe_sync_interface->Release();
323 } else {
324 vie_channel_->SetVoiceChannel(-1, nullptr);
325 }
326 }
327
GetStats() const328 VideoReceiveStream::Stats VideoReceiveStream::GetStats() const {
329 return stats_proxy_->GetStats();
330 }
331
DeliverRtcp(const uint8_t * packet,size_t length)332 bool VideoReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) {
333 return vie_channel_->ReceivedRTCPPacket(packet, length) == 0;
334 }
335
DeliverRtp(const uint8_t * packet,size_t length,const PacketTime & packet_time)336 bool VideoReceiveStream::DeliverRtp(const uint8_t* packet,
337 size_t length,
338 const PacketTime& packet_time) {
339 return vie_channel_->ReceivedRTPPacket(packet, length, packet_time) == 0;
340 }
341
FrameCallback(VideoFrame * video_frame)342 void VideoReceiveStream::FrameCallback(VideoFrame* video_frame) {
343 stats_proxy_->OnDecodedFrame();
344
345 // Post processing is not supported if the frame is backed by a texture.
346 if (video_frame->native_handle() == NULL) {
347 if (config_.pre_render_callback)
348 config_.pre_render_callback->FrameCallback(video_frame);
349 }
350 }
351
RenderFrame(const uint32_t,const VideoFrame & video_frame)352 int VideoReceiveStream::RenderFrame(const uint32_t /*stream_id*/,
353 const VideoFrame& video_frame) {
354 // TODO(pbos): Wire up config_.render->IsTextureSupported() and convert if not
355 // supported. Or provide methods for converting a texture frame in
356 // VideoFrame.
357
358 if (config_.renderer != nullptr)
359 config_.renderer->RenderFrame(
360 video_frame,
361 video_frame.render_time_ms() - clock_->TimeInMilliseconds());
362
363 stats_proxy_->OnRenderedFrame(video_frame.width(), video_frame.height());
364
365 return 0;
366 }
367
368 // TODO(asapersson): Consider moving callback from video_encoder.h or
369 // creating a different callback.
Encoded(const EncodedImage & encoded_image,const CodecSpecificInfo * codec_specific_info,const RTPFragmentationHeader * fragmentation)370 int32_t VideoReceiveStream::Encoded(
371 const EncodedImage& encoded_image,
372 const CodecSpecificInfo* codec_specific_info,
373 const RTPFragmentationHeader* fragmentation) {
374 stats_proxy_->OnPreDecode(encoded_image, codec_specific_info);
375 if (config_.pre_decode_callback) {
376 // TODO(asapersson): Remove EncodedFrameCallbackAdapter.
377 encoded_frame_proxy_.Encoded(
378 encoded_image, codec_specific_info, fragmentation);
379 }
380 return 0;
381 }
382
SignalNetworkState(NetworkState state)383 void VideoReceiveStream::SignalNetworkState(NetworkState state) {
384 vie_channel_->SetRTCPMode(state == kNetworkUp ? config_.rtp.rtcp_mode
385 : RtcpMode::kOff);
386 }
387
388 } // namespace internal
389 } // namespace webrtc
390