1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_VIDEO_RECEIVE_STREAM2_H_ 12 #define VIDEO_VIDEO_RECEIVE_STREAM2_H_ 13 14 #include <memory> 15 #include <vector> 16 17 #include "api/task_queue/task_queue_factory.h" 18 #include "api/units/timestamp.h" 19 #include "api/video/recordable_encoded_frame.h" 20 #include "call/rtp_packet_sink_interface.h" 21 #include "call/syncable.h" 22 #include "call/video_receive_stream.h" 23 #include "modules/rtp_rtcp/include/flexfec_receiver.h" 24 #include "modules/rtp_rtcp/source/source_tracker.h" 25 #include "modules/video_coding/frame_buffer2.h" 26 #include "modules/video_coding/video_receiver2.h" 27 #include "rtc_base/synchronization/sequence_checker.h" 28 #include "rtc_base/task_queue.h" 29 #include "rtc_base/task_utils/pending_task_safety_flag.h" 30 #include "system_wrappers/include/clock.h" 31 #include "video/receive_statistics_proxy2.h" 32 #include "video/rtp_streams_synchronizer2.h" 33 #include "video/rtp_video_stream_receiver2.h" 34 #include "video/transport_adapter.h" 35 #include "video/video_stream_decoder2.h" 36 37 namespace webrtc { 38 39 class ProcessThread; 40 class RTPFragmentationHeader; 41 class RtpStreamReceiverInterface; 42 class RtpStreamReceiverControllerInterface; 43 class RtxReceiveStream; 44 class VCMTiming; 45 46 namespace internal { 47 48 class CallStats; 49 50 // Utility struct for grabbing metadata from a VideoFrame and processing it 51 // asynchronously without needing the actual frame data. 52 // Additionally the caller can bundle information from the current clock 53 // when the metadata is captured, for accurate reporting and not needeing 54 // multiple calls to clock->Now(). 55 struct VideoFrameMetaData { VideoFrameMetaDataVideoFrameMetaData56 VideoFrameMetaData(const webrtc::VideoFrame& frame, Timestamp now) 57 : rtp_timestamp(frame.timestamp()), 58 timestamp_us(frame.timestamp_us()), 59 ntp_time_ms(frame.ntp_time_ms()), 60 width(frame.width()), 61 height(frame.height()), 62 decode_timestamp(now) {} 63 render_time_msVideoFrameMetaData64 int64_t render_time_ms() const { 65 return timestamp_us / rtc::kNumMicrosecsPerMillisec; 66 } 67 68 const uint32_t rtp_timestamp; 69 const int64_t timestamp_us; 70 const int64_t ntp_time_ms; 71 const int width; 72 const int height; 73 74 const Timestamp decode_timestamp; 75 }; 76 77 class VideoReceiveStream2 : public webrtc::VideoReceiveStream, 78 public rtc::VideoSinkInterface<VideoFrame>, 79 public NackSender, 80 public video_coding::OnCompleteFrameCallback, 81 public Syncable, 82 public CallStatsObserver { 83 public: 84 // The default number of milliseconds to pass before re-requesting a key frame 85 // to be sent. 86 static constexpr int kMaxWaitForKeyFrameMs = 200; 87 88 VideoReceiveStream2(TaskQueueFactory* task_queue_factory, 89 TaskQueueBase* current_queue, 90 RtpStreamReceiverControllerInterface* receiver_controller, 91 int num_cpu_cores, 92 PacketRouter* packet_router, 93 VideoReceiveStream::Config config, 94 ProcessThread* process_thread, 95 CallStats* call_stats, 96 Clock* clock, 97 VCMTiming* timing); 98 ~VideoReceiveStream2() override; 99 config()100 const Config& config() const { return config_; } 101 102 void SignalNetworkState(NetworkState state); 103 bool DeliverRtcp(const uint8_t* packet, size_t length); 104 105 void SetSync(Syncable* audio_syncable); 106 107 // Implements webrtc::VideoReceiveStream. 108 void Start() override; 109 void Stop() override; 110 111 webrtc::VideoReceiveStream::Stats GetStats() const override; 112 113 void AddSecondarySink(RtpPacketSinkInterface* sink) override; 114 void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override; 115 116 // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called 117 // from webrtc/api level and requested by user code. For e.g. blink/js layer 118 // in Chromium. 119 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; 120 int GetBaseMinimumPlayoutDelayMs() const override; 121 122 void SetFrameDecryptor( 123 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) override; 124 void SetDepacketizerToDecoderFrameTransformer( 125 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override; 126 127 // Implements rtc::VideoSinkInterface<VideoFrame>. 128 void OnFrame(const VideoFrame& video_frame) override; 129 130 // Implements NackSender. 131 // For this particular override of the interface, 132 // only (buffering_allowed == true) is acceptable. 133 void SendNack(const std::vector<uint16_t>& sequence_numbers, 134 bool buffering_allowed) override; 135 136 // Implements video_coding::OnCompleteFrameCallback. 137 void OnCompleteFrame( 138 std::unique_ptr<video_coding::EncodedFrame> frame) override; 139 140 // Implements CallStatsObserver::OnRttUpdate 141 void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; 142 143 // Implements Syncable. 144 uint32_t id() const override; 145 absl::optional<Syncable::Info> GetInfo() const override; 146 bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, 147 int64_t* time_ms) const override; 148 void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, 149 int64_t time_ms) override; 150 151 // SetMinimumPlayoutDelay is only called by A/V sync. 152 void SetMinimumPlayoutDelay(int delay_ms) override; 153 154 std::vector<webrtc::RtpSource> GetSources() const override; 155 156 RecordingState SetAndGetRecordingState(RecordingState state, 157 bool generate_key_frame) override; 158 void GenerateKeyFrame() override; 159 160 private: 161 int64_t GetMaxWaitMs() const RTC_RUN_ON(decode_queue_); 162 void StartNextDecode() RTC_RUN_ON(decode_queue_); 163 void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame) 164 RTC_RUN_ON(decode_queue_); 165 void HandleFrameBufferTimeout(int64_t now_ms, int64_t wait_ms) 166 RTC_RUN_ON(worker_sequence_checker_); 167 void UpdatePlayoutDelays() const 168 RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_sequence_checker_); 169 void RequestKeyFrame(int64_t timestamp_ms) 170 RTC_RUN_ON(worker_sequence_checker_); 171 void HandleKeyFrameGeneration(bool received_frame_is_keyframe, 172 int64_t now_ms, 173 bool always_request_key_frame, 174 bool keyframe_request_is_due) 175 RTC_RUN_ON(worker_sequence_checker_); 176 bool IsReceivingKeyFrame(int64_t timestamp_ms) const 177 RTC_RUN_ON(worker_sequence_checker_); 178 179 void UpdateHistograms(); 180 181 SequenceChecker worker_sequence_checker_; 182 SequenceChecker module_process_sequence_checker_; 183 184 TaskQueueFactory* const task_queue_factory_; 185 186 TransportAdapter transport_adapter_; 187 const VideoReceiveStream::Config config_; 188 const int num_cpu_cores_; 189 TaskQueueBase* const worker_thread_; 190 Clock* const clock_; 191 192 CallStats* const call_stats_; 193 194 bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false; 195 bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true; 196 197 SourceTracker source_tracker_; 198 ReceiveStatisticsProxy stats_proxy_; 199 // Shared by media and rtx stream receivers, since the latter has no RtpRtcp 200 // module of its own. 201 const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; 202 203 std::unique_ptr<VCMTiming> timing_; // Jitter buffer experiment. 204 VideoReceiver2 video_receiver_; 205 std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_; 206 RtpVideoStreamReceiver2 rtp_video_stream_receiver_; 207 std::unique_ptr<VideoStreamDecoder> video_stream_decoder_; 208 RtpStreamsSynchronizer rtp_stream_sync_; 209 210 // TODO(nisse, philipel): Creation and ownership of video encoders should be 211 // moved to the new VideoStreamDecoder. 212 std::vector<std::unique_ptr<VideoDecoder>> video_decoders_; 213 214 // Members for the new jitter buffer experiment. 215 std::unique_ptr<video_coding::FrameBuffer> frame_buffer_; 216 217 std::unique_ptr<RtpStreamReceiverInterface> media_receiver_; 218 std::unique_ptr<RtxReceiveStream> rtx_receive_stream_; 219 std::unique_ptr<RtpStreamReceiverInterface> rtx_receiver_; 220 221 // Whenever we are in an undecodable state (stream has just started or due to 222 // a decoding error) we require a keyframe to restart the stream. 223 bool keyframe_required_ RTC_GUARDED_BY(decode_queue_) = true; 224 225 // If we have successfully decoded any frame. 226 bool frame_decoded_ RTC_GUARDED_BY(decode_queue_) = false; 227 228 int64_t last_keyframe_request_ms_ RTC_GUARDED_BY(decode_queue_) = 0; 229 int64_t last_complete_frame_time_ms_ 230 RTC_GUARDED_BY(worker_sequence_checker_) = 0; 231 232 // Keyframe request intervals are configurable through field trials. 233 const int max_wait_for_keyframe_ms_; 234 const int max_wait_for_frame_ms_; 235 236 // All of them tries to change current min_playout_delay on |timing_| but 237 // source of the change request is different in each case. Among them the 238 // biggest delay is used. -1 means use default value from the |timing_|. 239 // 240 // Minimum delay as decided by the RTP playout delay extension. 241 int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(worker_sequence_checker_) = 242 -1; 243 // Minimum delay as decided by the setLatency function in "webrtc/api". 244 int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(worker_sequence_checker_) = 245 -1; 246 // Minimum delay as decided by the A/V synchronization feature. 247 int syncable_minimum_playout_delay_ms_ 248 RTC_GUARDED_BY(worker_sequence_checker_) = -1; 249 250 // Maximum delay as decided by the RTP playout delay extension. 251 int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(worker_sequence_checker_) = 252 -1; 253 254 // Function that is triggered with encoded frames, if not empty. 255 std::function<void(const RecordableEncodedFrame&)> 256 encoded_frame_buffer_function_ RTC_GUARDED_BY(decode_queue_); 257 // Set to true while we're requesting keyframes but not yet received one. 258 bool keyframe_generation_requested_ RTC_GUARDED_BY(worker_sequence_checker_) = 259 false; 260 261 // Defined last so they are destroyed before all other members. 262 rtc::TaskQueue decode_queue_; 263 264 // Used to signal destruction to potentially pending tasks. 265 ScopedTaskSafety task_safety_; 266 }; 267 } // namespace internal 268 } // namespace webrtc 269 270 #endif // VIDEO_VIDEO_RECEIVE_STREAM2_H_ 271