1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_VIDEO_RECEIVE_STREAM_H_ 12 #define VIDEO_VIDEO_RECEIVE_STREAM_H_ 13 14 #include <memory> 15 #include <vector> 16 17 #include "api/task_queue/task_queue_factory.h" 18 #include "api/video/recordable_encoded_frame.h" 19 #include "call/rtp_packet_sink_interface.h" 20 #include "call/syncable.h" 21 #include "call/video_receive_stream.h" 22 #include "modules/rtp_rtcp/include/flexfec_receiver.h" 23 #include "modules/rtp_rtcp/source/source_tracker.h" 24 #include "modules/video_coding/frame_buffer2.h" 25 #include "modules/video_coding/video_receiver2.h" 26 #include "rtc_base/synchronization/mutex.h" 27 #include "rtc_base/synchronization/sequence_checker.h" 28 #include "rtc_base/task_queue.h" 29 #include "system_wrappers/include/clock.h" 30 #include "video/receive_statistics_proxy.h" 31 #include "video/rtp_streams_synchronizer.h" 32 #include "video/rtp_video_stream_receiver.h" 33 #include "video/transport_adapter.h" 34 #include "video/video_stream_decoder.h" 35 36 namespace webrtc { 37 38 class CallStats; 39 class ProcessThread; 40 class RTPFragmentationHeader; 41 class RtpStreamReceiverInterface; 42 class RtpStreamReceiverControllerInterface; 43 class RtxReceiveStream; 44 class VCMTiming; 45 46 namespace internal { 47 48 class VideoReceiveStream : public webrtc::VideoReceiveStream, 49 public rtc::VideoSinkInterface<VideoFrame>, 50 public NackSender, 51 public video_coding::OnCompleteFrameCallback, 52 public Syncable, 53 public CallStatsObserver { 54 public: 55 // The default number of milliseconds to pass before re-requesting a key frame 56 // to be sent. 57 static constexpr int kMaxWaitForKeyFrameMs = 200; 58 59 VideoReceiveStream(TaskQueueFactory* task_queue_factory, 60 RtpStreamReceiverControllerInterface* receiver_controller, 61 int num_cpu_cores, 62 PacketRouter* packet_router, 63 VideoReceiveStream::Config config, 64 ProcessThread* process_thread, 65 CallStats* call_stats, 66 Clock* clock, 67 VCMTiming* timing); 68 VideoReceiveStream(TaskQueueFactory* task_queue_factory, 69 RtpStreamReceiverControllerInterface* receiver_controller, 70 int num_cpu_cores, 71 PacketRouter* packet_router, 72 VideoReceiveStream::Config config, 73 ProcessThread* process_thread, 74 CallStats* call_stats, 75 Clock* clock); 76 ~VideoReceiveStream() override; 77 config()78 const Config& config() const { return config_; } 79 80 void SignalNetworkState(NetworkState state); 81 bool DeliverRtcp(const uint8_t* packet, size_t length); 82 83 void SetSync(Syncable* audio_syncable); 84 85 // Implements webrtc::VideoReceiveStream. 86 void Start() override; 87 void Stop() override; 88 89 webrtc::VideoReceiveStream::Stats GetStats() const override; 90 91 void AddSecondarySink(RtpPacketSinkInterface* sink) override; 92 void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override; 93 94 // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called 95 // from webrtc/api level and requested by user code. For e.g. blink/js layer 96 // in Chromium. 97 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; 98 int GetBaseMinimumPlayoutDelayMs() const override; 99 100 void SetFrameDecryptor( 101 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) override; 102 void SetDepacketizerToDecoderFrameTransformer( 103 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override; 104 105 // Implements rtc::VideoSinkInterface<VideoFrame>. 106 void OnFrame(const VideoFrame& video_frame) override; 107 108 // Implements NackSender. 109 // For this particular override of the interface, 110 // only (buffering_allowed == true) is acceptable. 111 void SendNack(const std::vector<uint16_t>& sequence_numbers, 112 bool buffering_allowed) override; 113 114 // Implements video_coding::OnCompleteFrameCallback. 115 void OnCompleteFrame( 116 std::unique_ptr<video_coding::EncodedFrame> frame) override; 117 118 // Implements CallStatsObserver::OnRttUpdate 119 void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; 120 121 // Implements Syncable. 122 uint32_t id() const override; 123 absl::optional<Syncable::Info> GetInfo() const override; 124 bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, 125 int64_t* time_ms) const override; 126 void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, 127 int64_t time_ms) override; 128 129 // SetMinimumPlayoutDelay is only called by A/V sync. 130 void SetMinimumPlayoutDelay(int delay_ms) override; 131 132 std::vector<webrtc::RtpSource> GetSources() const override; 133 134 RecordingState SetAndGetRecordingState(RecordingState state, 135 bool generate_key_frame) override; 136 void GenerateKeyFrame() override; 137 138 private: 139 int64_t GetWaitMs() const; 140 void StartNextDecode() RTC_RUN_ON(decode_queue_); 141 void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame) 142 RTC_RUN_ON(decode_queue_); 143 void HandleFrameBufferTimeout() RTC_RUN_ON(decode_queue_); 144 void UpdatePlayoutDelays() const 145 RTC_EXCLUSIVE_LOCKS_REQUIRED(playout_delay_lock_); 146 void RequestKeyFrame(int64_t timestamp_ms) RTC_RUN_ON(decode_queue_); 147 void HandleKeyFrameGeneration(bool received_frame_is_keyframe, int64_t now_ms) 148 RTC_RUN_ON(decode_queue_); 149 bool IsReceivingKeyFrame(int64_t timestamp_ms) const 150 RTC_RUN_ON(decode_queue_); 151 152 void UpdateHistograms(); 153 154 SequenceChecker worker_sequence_checker_; 155 SequenceChecker module_process_sequence_checker_; 156 SequenceChecker network_sequence_checker_; 157 158 TaskQueueFactory* const task_queue_factory_; 159 160 TransportAdapter transport_adapter_; 161 const VideoReceiveStream::Config config_; 162 const int num_cpu_cores_; 163 ProcessThread* const process_thread_; 164 Clock* const clock_; 165 166 CallStats* const call_stats_; 167 168 bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false; 169 bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true; 170 171 SourceTracker source_tracker_; 172 ReceiveStatisticsProxy stats_proxy_; 173 // Shared by media and rtx stream receivers, since the latter has no RtpRtcp 174 // module of its own. 175 const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; 176 177 std::unique_ptr<VCMTiming> timing_; // Jitter buffer experiment. 178 VideoReceiver2 video_receiver_; 179 std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_; 180 RtpVideoStreamReceiver rtp_video_stream_receiver_; 181 std::unique_ptr<VideoStreamDecoder> video_stream_decoder_; 182 RtpStreamsSynchronizer rtp_stream_sync_; 183 184 // TODO(nisse, philipel): Creation and ownership of video encoders should be 185 // moved to the new VideoStreamDecoder. 186 std::vector<std::unique_ptr<VideoDecoder>> video_decoders_; 187 188 // Members for the new jitter buffer experiment. 189 std::unique_ptr<video_coding::FrameBuffer> frame_buffer_; 190 191 std::unique_ptr<RtpStreamReceiverInterface> media_receiver_; 192 std::unique_ptr<RtxReceiveStream> rtx_receive_stream_; 193 std::unique_ptr<RtpStreamReceiverInterface> rtx_receiver_; 194 195 // Whenever we are in an undecodable state (stream has just started or due to 196 // a decoding error) we require a keyframe to restart the stream. 197 bool keyframe_required_ = true; 198 199 // If we have successfully decoded any frame. 200 bool frame_decoded_ = false; 201 202 int64_t last_keyframe_request_ms_ = 0; 203 int64_t last_complete_frame_time_ms_ = 0; 204 205 // Keyframe request intervals are configurable through field trials. 206 const int max_wait_for_keyframe_ms_; 207 const int max_wait_for_frame_ms_; 208 209 mutable Mutex playout_delay_lock_; 210 211 // All of them tries to change current min_playout_delay on |timing_| but 212 // source of the change request is different in each case. Among them the 213 // biggest delay is used. -1 means use default value from the |timing_|. 214 // 215 // Minimum delay as decided by the RTP playout delay extension. 216 int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; 217 // Minimum delay as decided by the setLatency function in "webrtc/api". 218 int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; 219 // Minimum delay as decided by the A/V synchronization feature. 220 int syncable_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = 221 -1; 222 223 // Maximum delay as decided by the RTP playout delay extension. 224 int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; 225 226 // Function that is triggered with encoded frames, if not empty. 227 std::function<void(const RecordableEncodedFrame&)> 228 encoded_frame_buffer_function_ RTC_GUARDED_BY(decode_queue_); 229 // Set to true while we're requesting keyframes but not yet received one. 230 bool keyframe_generation_requested_ RTC_GUARDED_BY(decode_queue_) = false; 231 232 // Defined last so they are destroyed before all other members. 233 rtc::TaskQueue decode_queue_; 234 }; 235 } // namespace internal 236 } // namespace webrtc 237 238 #endif // VIDEO_VIDEO_RECEIVE_STREAM_H_ 239