1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_VIDEO_VIE_CHANNEL_H_ 12 #define WEBRTC_VIDEO_VIE_CHANNEL_H_ 13 14 #include <list> 15 #include <map> 16 #include <vector> 17 18 #include "webrtc/base/platform_thread.h" 19 #include "webrtc/base/scoped_ptr.h" 20 #include "webrtc/base/scoped_ref_ptr.h" 21 #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" 22 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" 23 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h" 24 #include "webrtc/modules/video_coding/include/video_coding_defines.h" 25 #include "webrtc/system_wrappers/include/critical_section_wrapper.h" 26 #include "webrtc/system_wrappers/include/tick_util.h" 27 #include "webrtc/typedefs.h" 28 #include "webrtc/video/vie_receiver.h" 29 #include "webrtc/video/vie_sync_module.h" 30 31 namespace webrtc { 32 33 class CallStatsObserver; 34 class ChannelStatsObserver; 35 class Config; 36 class CriticalSectionWrapper; 37 class EncodedImageCallback; 38 class I420FrameCallback; 39 class IncomingVideoStream; 40 class PacedSender; 41 class PacketRouter; 42 class PayloadRouter; 43 class ProcessThread; 44 class ReceiveStatisticsProxy; 45 class ReportBlockStats; 46 class RtcpRttStats; 47 class ViEChannelProtectionCallback; 48 class ViERTPObserver; 49 class VideoCodingModule; 50 class VideoDecoder; 51 class VideoRenderCallback; 52 class VoEVideoSync; 53 54 enum StreamType { 55 kViEStreamTypeNormal = 0, // Normal media stream 56 kViEStreamTypeRtx = 1 // Retransmission media stream 57 }; 58 59 class ViEChannel : public VCMFrameTypeCallback, 60 public VCMReceiveCallback, 61 public VCMReceiveStatisticsCallback, 62 public VCMDecoderTimingCallback, 63 public VCMPacketRequestCallback, 64 public RtpFeedback { 65 public: 66 friend class ChannelStatsObserver; 67 friend class ViEChannelProtectionCallback; 68 69 ViEChannel(uint32_t number_of_cores, 70 Transport* transport, 71 ProcessThread* module_process_thread, 72 RtcpIntraFrameObserver* intra_frame_observer, 73 RtcpBandwidthObserver* bandwidth_observer, 74 TransportFeedbackObserver* transport_feedback_observer, 75 RemoteBitrateEstimator* remote_bitrate_estimator, 76 RtcpRttStats* rtt_stats, 77 PacedSender* paced_sender, 78 PacketRouter* packet_router, 79 size_t max_rtp_streams, 80 bool sender); 81 ~ViEChannel(); 82 83 int32_t Init(); 84 85 // Sets the encoder to use for the channel. |new_stream| indicates the encoder 86 // type has changed and we should start a new RTP stream. 87 int32_t SetSendCodec(const VideoCodec& video_codec, bool new_stream = true); 88 int32_t SetReceiveCodec(const VideoCodec& video_codec); 89 // Registers an external decoder. 90 void RegisterExternalDecoder(const uint8_t pl_type, VideoDecoder* decoder); 91 int32_t ReceiveCodecStatistics(uint32_t* num_key_frames, 92 uint32_t* num_delta_frames); 93 uint32_t DiscardedPackets() const; 94 95 // Returns the estimated delay in milliseconds. 96 int ReceiveDelay() const; 97 98 void SetExpectedRenderDelay(int delay_ms); 99 100 void SetRTCPMode(const RtcpMode rtcp_mode); 101 void SetProtectionMode(bool enable_nack, 102 bool enable_fec, 103 int payload_type_red, 104 int payload_type_fec); 105 bool IsSendingFecEnabled(); 106 int SetSenderBufferingMode(int target_delay_ms); 107 int SetSendTimestampOffsetStatus(bool enable, int id); 108 int SetReceiveTimestampOffsetStatus(bool enable, int id); 109 int SetSendAbsoluteSendTimeStatus(bool enable, int id); 110 int SetReceiveAbsoluteSendTimeStatus(bool enable, int id); 111 int SetSendVideoRotationStatus(bool enable, int id); 112 int SetReceiveVideoRotationStatus(bool enable, int id); 113 int SetSendTransportSequenceNumber(bool enable, int id); 114 int SetReceiveTransportSequenceNumber(bool enable, int id); 115 void SetRtcpXrRrtrStatus(bool enable); 116 void EnableTMMBR(bool enable); 117 118 // Sets SSRC for outgoing stream. 119 int32_t SetSSRC(const uint32_t SSRC, 120 const StreamType usage, 121 const unsigned char simulcast_idx); 122 123 // Gets SSRC for outgoing stream number |idx|. 124 int32_t GetLocalSSRC(uint8_t idx, unsigned int* ssrc); 125 126 // Gets SSRC for the incoming stream. 127 uint32_t GetRemoteSSRC(); 128 129 int SetRtxSendPayloadType(int payload_type, int associated_payload_type); 130 void SetRtxReceivePayloadType(int payload_type, int associated_payload_type); 131 // If set to true, the RTX payload type mapping supplied in 132 // |SetRtxReceivePayloadType| will be used when restoring RTX packets. Without 133 // it, RTX packets will always be restored to the last non-RTX packet payload 134 // type received. 135 void SetUseRtxPayloadMappingOnRestore(bool val); 136 137 void SetRtpStateForSsrc(uint32_t ssrc, const RtpState& rtp_state); 138 RtpState GetRtpStateForSsrc(uint32_t ssrc); 139 140 // Sets the CName for the outgoing stream on the channel. 141 int32_t SetRTCPCName(const char* rtcp_cname); 142 143 // Gets the CName of the incoming stream. 144 int32_t GetRemoteRTCPCName(char rtcp_cname[]); 145 146 // Returns statistics reported by the remote client in an RTCP packet. 147 // TODO(pbos): Remove this along with VideoSendStream::GetRtt(). 148 int32_t GetSendRtcpStatistics(uint16_t* fraction_lost, 149 uint32_t* cumulative_lost, 150 uint32_t* extended_max, 151 uint32_t* jitter_samples, 152 int64_t* rtt_ms); 153 154 // Called on receipt of RTCP report block from remote side. 155 void RegisterSendChannelRtcpStatisticsCallback( 156 RtcpStatisticsCallback* callback); 157 158 // Called on generation of RTCP stats 159 void RegisterReceiveChannelRtcpStatisticsCallback( 160 RtcpStatisticsCallback* callback); 161 162 // Gets send statistics for the rtp and rtx stream. 163 void GetSendStreamDataCounters(StreamDataCounters* rtp_counters, 164 StreamDataCounters* rtx_counters) const; 165 166 // Gets received stream data counters. 167 void GetReceiveStreamDataCounters(StreamDataCounters* rtp_counters, 168 StreamDataCounters* rtx_counters) const; 169 170 // Called on update of RTP statistics. 171 void RegisterSendChannelRtpStatisticsCallback( 172 StreamDataCountersCallback* callback); 173 174 // Called on update of RTP statistics. 175 void RegisterReceiveChannelRtpStatisticsCallback( 176 StreamDataCountersCallback* callback); 177 178 void GetSendRtcpPacketTypeCounter( 179 RtcpPacketTypeCounter* packet_counter) const; 180 181 void GetReceiveRtcpPacketTypeCounter( 182 RtcpPacketTypeCounter* packet_counter) const; 183 184 void RegisterSendSideDelayObserver(SendSideDelayObserver* observer); 185 186 // Called on any new send bitrate estimate. 187 void RegisterSendBitrateObserver(BitrateStatisticsObserver* observer); 188 189 // Implements RtpFeedback. 190 int32_t OnInitializeDecoder(const int8_t payload_type, 191 const char payload_name[RTP_PAYLOAD_NAME_SIZE], 192 const int frequency, 193 const size_t channels, 194 const uint32_t rate) override; 195 void OnIncomingSSRCChanged(const uint32_t ssrc) override; 196 void OnIncomingCSRCChanged(const uint32_t CSRC, const bool added) override; 197 198 int32_t SetRemoteSSRCType(const StreamType usage, const uint32_t SSRC); 199 200 int32_t StartSend(); 201 int32_t StopSend(); 202 bool Sending(); 203 void StartReceive(); 204 void StopReceive(); 205 206 int32_t ReceivedRTPPacket(const void* rtp_packet, 207 const size_t rtp_packet_length, 208 const PacketTime& packet_time); 209 int32_t ReceivedRTCPPacket(const void* rtcp_packet, 210 const size_t rtcp_packet_length); 211 212 // Sets the maximum transfer unit size for the network link, i.e. including 213 // IP, UDP and RTP headers. 214 int32_t SetMTU(uint16_t mtu); 215 216 // Gets the modules used by the channel. 217 RtpRtcp* rtp_rtcp(); 218 rtc::scoped_refptr<PayloadRouter> send_payload_router(); 219 VCMProtectionCallback* vcm_protection_callback(); 220 221 222 CallStatsObserver* GetStatsObserver(); 223 224 // Implements VCMReceiveCallback. 225 virtual int32_t FrameToRender(VideoFrame& video_frame); // NOLINT 226 227 // Implements VCMReceiveCallback. 228 virtual int32_t ReceivedDecodedReferenceFrame( 229 const uint64_t picture_id); 230 231 // Implements VCMReceiveCallback. 232 void OnIncomingPayloadType(int payload_type) override; 233 void OnDecoderImplementationName(const char* implementation_name) override; 234 235 // Implements VCMReceiveStatisticsCallback. 236 void OnReceiveRatesUpdated(uint32_t bit_rate, uint32_t frame_rate) override; 237 void OnDiscardedPacketsUpdated(int discarded_packets) override; 238 void OnFrameCountsUpdated(const FrameCounts& frame_counts) override; 239 240 // Implements VCMDecoderTimingCallback. 241 virtual void OnDecoderTiming(int decode_ms, 242 int max_decode_ms, 243 int current_delay_ms, 244 int target_delay_ms, 245 int jitter_buffer_ms, 246 int min_playout_delay_ms, 247 int render_delay_ms); 248 249 // Implements FrameTypeCallback. 250 virtual int32_t RequestKeyFrame(); 251 252 // Implements FrameTypeCallback. 253 virtual int32_t SliceLossIndicationRequest( 254 const uint64_t picture_id); 255 256 // Implements VideoPacketRequestCallback. 257 int32_t ResendPackets(const uint16_t* sequence_numbers, 258 uint16_t length) override; 259 260 int32_t SetVoiceChannel(int32_t ve_channel_id, 261 VoEVideoSync* ve_sync_interface); 262 int32_t VoiceChannel(); 263 264 // New-style callbacks, used by VideoReceiveStream. 265 void RegisterPreRenderCallback(I420FrameCallback* pre_render_callback); 266 void RegisterPreDecodeImageCallback( 267 EncodedImageCallback* pre_decode_callback); 268 269 void RegisterSendFrameCountObserver(FrameCountObserver* observer); 270 void RegisterRtcpPacketTypeCounterObserver( 271 RtcpPacketTypeCounterObserver* observer); 272 void RegisterReceiveStatisticsProxy( 273 ReceiveStatisticsProxy* receive_statistics_proxy); 274 void SetIncomingVideoStream(IncomingVideoStream* incoming_video_stream); 275 276 protected: 277 static bool ChannelDecodeThreadFunction(void* obj); 278 bool ChannelDecodeProcess(); 279 280 void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms); 281 282 int ProtectionRequest(const FecProtectionParams* delta_fec_params, 283 const FecProtectionParams* key_fec_params, 284 uint32_t* sent_video_rate_bps, 285 uint32_t* sent_nack_rate_bps, 286 uint32_t* sent_fec_rate_bps); 287 288 private: 289 static std::vector<RtpRtcp*> CreateRtpRtcpModules( 290 bool receiver_only, 291 ReceiveStatistics* receive_statistics, 292 Transport* outgoing_transport, 293 RtcpIntraFrameObserver* intra_frame_callback, 294 RtcpBandwidthObserver* bandwidth_callback, 295 TransportFeedbackObserver* transport_feedback_callback, 296 RtcpRttStats* rtt_stats, 297 RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, 298 RemoteBitrateEstimator* remote_bitrate_estimator, 299 RtpPacketSender* paced_sender, 300 TransportSequenceNumberAllocator* transport_sequence_number_allocator, 301 BitrateStatisticsObserver* send_bitrate_observer, 302 FrameCountObserver* send_frame_count_observer, 303 SendSideDelayObserver* send_side_delay_observer, 304 size_t num_modules); 305 306 // Assumed to be protected. 307 void StartDecodeThread(); 308 void StopDecodeThread(); 309 310 void ProcessNACKRequest(const bool enable); 311 // Compute NACK list parameters for the buffering mode. 312 int GetRequiredNackListSize(int target_delay_ms); 313 void SetRtxSendStatus(bool enable); 314 315 void UpdateHistograms(); 316 317 // ViEChannel exposes methods that allow to modify observers and callbacks 318 // to be modified. Such an API-style is cumbersome to implement and maintain 319 // at all the levels when comparing to only setting them at construction. As 320 // so this class instantiates its children with a wrapper that can be modified 321 // at a later time. 322 template <class T> 323 class RegisterableCallback : public T { 324 public: RegisterableCallback()325 RegisterableCallback() 326 : critsect_(CriticalSectionWrapper::CreateCriticalSection()), 327 callback_(NULL) {} 328 Set(T * callback)329 void Set(T* callback) { 330 CriticalSectionScoped cs(critsect_.get()); 331 callback_ = callback; 332 } 333 334 protected: 335 // Note: this should be implemented with a RW-lock to allow simultaneous 336 // calls into the callback. However that doesn't seem to be needed for the 337 // current type of callbacks covered by this class. 338 rtc::scoped_ptr<CriticalSectionWrapper> critsect_; 339 T* callback_ GUARDED_BY(critsect_); 340 341 private: 342 RTC_DISALLOW_COPY_AND_ASSIGN(RegisterableCallback); 343 }; 344 345 class RegisterableBitrateStatisticsObserver: 346 public RegisterableCallback<BitrateStatisticsObserver> { Notify(const BitrateStatistics & total_stats,const BitrateStatistics & retransmit_stats,uint32_t ssrc)347 virtual void Notify(const BitrateStatistics& total_stats, 348 const BitrateStatistics& retransmit_stats, 349 uint32_t ssrc) { 350 CriticalSectionScoped cs(critsect_.get()); 351 if (callback_) 352 callback_->Notify(total_stats, retransmit_stats, ssrc); 353 } 354 } send_bitrate_observer_; 355 356 class RegisterableFrameCountObserver 357 : public RegisterableCallback<FrameCountObserver> { 358 public: FrameCountUpdated(const FrameCounts & frame_counts,uint32_t ssrc)359 virtual void FrameCountUpdated(const FrameCounts& frame_counts, 360 uint32_t ssrc) { 361 CriticalSectionScoped cs(critsect_.get()); 362 if (callback_) 363 callback_->FrameCountUpdated(frame_counts, ssrc); 364 } 365 366 private: 367 } send_frame_count_observer_; 368 369 class RegisterableSendSideDelayObserver : 370 public RegisterableCallback<SendSideDelayObserver> { SendSideDelayUpdated(int avg_delay_ms,int max_delay_ms,uint32_t ssrc)371 void SendSideDelayUpdated(int avg_delay_ms, 372 int max_delay_ms, 373 uint32_t ssrc) override { 374 CriticalSectionScoped cs(critsect_.get()); 375 if (callback_) 376 callback_->SendSideDelayUpdated(avg_delay_ms, max_delay_ms, ssrc); 377 } 378 } send_side_delay_observer_; 379 380 class RegisterableRtcpPacketTypeCounterObserver 381 : public RegisterableCallback<RtcpPacketTypeCounterObserver> { 382 public: RtcpPacketTypesCounterUpdated(uint32_t ssrc,const RtcpPacketTypeCounter & packet_counter)383 void RtcpPacketTypesCounterUpdated( 384 uint32_t ssrc, 385 const RtcpPacketTypeCounter& packet_counter) override { 386 CriticalSectionScoped cs(critsect_.get()); 387 if (callback_) 388 callback_->RtcpPacketTypesCounterUpdated(ssrc, packet_counter); 389 counter_map_[ssrc] = packet_counter; 390 } 391 GetPacketTypeCounterMap()392 virtual std::map<uint32_t, RtcpPacketTypeCounter> GetPacketTypeCounterMap() 393 const { 394 CriticalSectionScoped cs(critsect_.get()); 395 return counter_map_; 396 } 397 398 private: 399 std::map<uint32_t, RtcpPacketTypeCounter> counter_map_ 400 GUARDED_BY(critsect_); 401 } rtcp_packet_type_counter_observer_; 402 403 const uint32_t number_of_cores_; 404 const bool sender_; 405 406 ProcessThread* const module_process_thread_; 407 408 // Used for all registered callbacks except rendering. 409 rtc::scoped_ptr<CriticalSectionWrapper> crit_; 410 411 // Owned modules/classes. 412 rtc::scoped_refptr<PayloadRouter> send_payload_router_; 413 rtc::scoped_ptr<ViEChannelProtectionCallback> vcm_protection_callback_; 414 415 VideoCodingModule* const vcm_; 416 ViEReceiver vie_receiver_; 417 ViESyncModule vie_sync_; 418 419 // Helper to report call statistics. 420 rtc::scoped_ptr<ChannelStatsObserver> stats_observer_; 421 422 // Not owned. 423 ReceiveStatisticsProxy* receive_stats_callback_ GUARDED_BY(crit_); 424 FrameCounts receive_frame_counts_ GUARDED_BY(crit_); 425 IncomingVideoStream* incoming_video_stream_ GUARDED_BY(crit_); 426 RtcpIntraFrameObserver* const intra_frame_observer_; 427 RtcpRttStats* const rtt_stats_; 428 PacedSender* const paced_sender_; 429 PacketRouter* const packet_router_; 430 431 const rtc::scoped_ptr<RtcpBandwidthObserver> bandwidth_observer_; 432 TransportFeedbackObserver* const transport_feedback_observer_; 433 434 rtc::PlatformThread decode_thread_; 435 436 int nack_history_size_sender_; 437 int max_nack_reordering_threshold_; 438 I420FrameCallback* pre_render_callback_ GUARDED_BY(crit_); 439 440 const rtc::scoped_ptr<ReportBlockStats> report_block_stats_sender_; 441 442 int64_t time_of_first_rtt_ms_ GUARDED_BY(crit_); 443 int64_t rtt_sum_ms_ GUARDED_BY(crit_); 444 int64_t last_rtt_ms_ GUARDED_BY(crit_); 445 size_t num_rtts_ GUARDED_BY(crit_); 446 447 // RtpRtcp modules, declared last as they use other members on construction. 448 const std::vector<RtpRtcp*> rtp_rtcp_modules_; 449 size_t num_active_rtp_rtcp_modules_ GUARDED_BY(crit_); 450 }; 451 452 } // namespace webrtc 453 454 #endif // WEBRTC_VIDEO_VIE_CHANNEL_H_ 455