1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_VIDEO_RECEIVE_STREAM2_H_ 12 #define VIDEO_VIDEO_RECEIVE_STREAM2_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "api/sequence_checker.h" 20 #include "api/task_queue/pending_task_safety_flag.h" 21 #include "api/task_queue/task_queue_factory.h" 22 #include "api/units/time_delta.h" 23 #include "api/units/timestamp.h" 24 #include "api/video/recordable_encoded_frame.h" 25 #include "call/call.h" 26 #include "call/rtp_packet_sink_interface.h" 27 #include "call/syncable.h" 28 #include "call/video_receive_stream.h" 29 #include "modules/rtp_rtcp/source/source_tracker.h" 30 #include "modules/video_coding/nack_requester.h" 31 #include "modules/video_coding/video_receiver2.h" 32 #include "rtc_base/system/no_unique_address.h" 33 #include "rtc_base/task_queue.h" 34 #include "rtc_base/thread_annotations.h" 35 #include "system_wrappers/include/clock.h" 36 #include "video/receive_statistics_proxy2.h" 37 #include "video/rtp_streams_synchronizer2.h" 38 #include "video/rtp_video_stream_receiver2.h" 39 #include "video/transport_adapter.h" 40 #include "video/video_stream_buffer_controller.h" 41 #include "video/video_stream_decoder2.h" 42 43 namespace webrtc { 44 45 class RtpStreamReceiverInterface; 46 class RtpStreamReceiverControllerInterface; 47 class RtxReceiveStream; 48 class VCMTiming; 49 50 constexpr TimeDelta kMaxWaitForKeyFrame = TimeDelta::Millis(200); 51 constexpr TimeDelta kMaxWaitForFrame = TimeDelta::Seconds(3); 52 53 namespace internal { 54 55 class CallStats; 56 57 // Utility struct for grabbing metadata from a VideoFrame and processing it 58 // asynchronously without needing the actual frame data. 59 // Additionally the caller can bundle information from the current clock 60 // when the metadata is captured, for accurate reporting and not needing 61 // multiple calls to clock->Now(). 62 struct VideoFrameMetaData { VideoFrameMetaDataVideoFrameMetaData63 VideoFrameMetaData(const webrtc::VideoFrame& frame, Timestamp now) 64 : rtp_timestamp(frame.timestamp()), 65 timestamp_us(frame.timestamp_us()), 66 ntp_time_ms(frame.ntp_time_ms()), 67 width(frame.width()), 68 height(frame.height()), 69 decode_timestamp(now) {} 70 render_time_msVideoFrameMetaData71 int64_t render_time_ms() const { 72 return timestamp_us / rtc::kNumMicrosecsPerMillisec; 73 } 74 75 const uint32_t rtp_timestamp; 76 const int64_t timestamp_us; 77 const int64_t ntp_time_ms; 78 const int width; 79 const int height; 80 81 const Timestamp decode_timestamp; 82 }; 83 84 class VideoReceiveStream2 85 : public webrtc::VideoReceiveStreamInterface, 86 public rtc::VideoSinkInterface<VideoFrame>, 87 public RtpVideoStreamReceiver2::OnCompleteFrameCallback, 88 public Syncable, 89 public CallStatsObserver, 90 public FrameSchedulingReceiver { 91 public: 92 // The maximum number of buffered encoded frames when encoded output is 93 // configured. 94 static constexpr size_t kBufferedEncodedFramesMaxSize = 60; 95 96 VideoReceiveStream2(TaskQueueFactory* task_queue_factory, 97 Call* call, 98 int num_cpu_cores, 99 PacketRouter* packet_router, 100 VideoReceiveStreamInterface::Config config, 101 CallStats* call_stats, 102 Clock* clock, 103 std::unique_ptr<VCMTiming> timing, 104 NackPeriodicProcessor* nack_periodic_processor, 105 DecodeSynchronizer* decode_sync, 106 RtcEventLog* event_log); 107 // Destruction happens on the worker thread. Prior to destruction the caller 108 // must ensure that a registration with the transport has been cleared. See 109 // `RegisterWithTransport` for details. 110 // TODO(tommi): As a further improvement to this, performing the full 111 // destruction on the network thread could be made the default. 112 ~VideoReceiveStream2() override; 113 114 // Called on `packet_sequence_checker_` to register/unregister with the 115 // network transport. 116 void RegisterWithTransport( 117 RtpStreamReceiverControllerInterface* receiver_controller); 118 // If registration has previously been done (via `RegisterWithTransport`) then 119 // `UnregisterFromTransport` must be called prior to destruction, on the 120 // network thread. 121 void UnregisterFromTransport(); 122 123 // Accessor for the a/v sync group. This value may change and the caller 124 // must be on the packet delivery thread. 125 const std::string& sync_group() const; 126 127 // Getters for const remote SSRC values that won't change throughout the 128 // object's lifetime. remote_ssrc()129 uint32_t remote_ssrc() const { return config_.rtp.remote_ssrc; } rtx_ssrc()130 uint32_t rtx_ssrc() const { return config_.rtp.rtx_ssrc; } 131 132 void SignalNetworkState(NetworkState state); 133 bool DeliverRtcp(const uint8_t* packet, size_t length); 134 135 void SetSync(Syncable* audio_syncable); 136 137 // Updates the `rtp_video_stream_receiver_`'s `local_ssrc` when the default 138 // sender has been created, changed or removed. 139 void SetLocalSsrc(uint32_t local_ssrc); 140 141 // Implements webrtc::VideoReceiveStreamInterface. 142 void Start() override; 143 void Stop() override; 144 145 void SetRtpExtensions(std::vector<RtpExtension> extensions) override; 146 RtpHeaderExtensionMap GetRtpExtensionMap() const override; 147 bool transport_cc() const override; 148 void SetTransportCc(bool transport_cc) override; 149 void SetRtcpMode(RtcpMode mode) override; 150 void SetFlexFecProtection(RtpPacketSinkInterface* flexfec_sink) override; 151 void SetLossNotificationEnabled(bool enabled) override; 152 void SetNackHistory(TimeDelta history) override; 153 void SetProtectionPayloadTypes(int red_payload_type, 154 int ulpfec_payload_type) override; 155 void SetRtcpXr(Config::Rtp::RtcpXr rtcp_xr) override; 156 void SetAssociatedPayloadTypes( 157 std::map<int, int> associated_payload_types) override; 158 159 webrtc::VideoReceiveStreamInterface::Stats GetStats() const override; 160 161 // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called 162 // from webrtc/api level and requested by user code. For e.g. blink/js layer 163 // in Chromium. 164 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; 165 int GetBaseMinimumPlayoutDelayMs() const override; 166 167 void SetFrameDecryptor( 168 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) override; 169 void SetDepacketizerToDecoderFrameTransformer( 170 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override; 171 172 // Implements rtc::VideoSinkInterface<VideoFrame>. 173 void OnFrame(const VideoFrame& video_frame) override; 174 175 // Implements RtpVideoStreamReceiver2::OnCompleteFrameCallback. 176 void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) override; 177 178 // Implements CallStatsObserver::OnRttUpdate 179 void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; 180 181 // Implements Syncable. 182 uint32_t id() const override; 183 absl::optional<Syncable::Info> GetInfo() const override; 184 bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, 185 int64_t* time_ms) const override; 186 void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, 187 int64_t time_ms) override; 188 189 // SetMinimumPlayoutDelay is only called by A/V sync. 190 bool SetMinimumPlayoutDelay(int delay_ms) override; 191 192 std::vector<webrtc::RtpSource> GetSources() const override; 193 194 RecordingState SetAndGetRecordingState(RecordingState state, 195 bool generate_key_frame) override; 196 void GenerateKeyFrame() override; 197 198 private: 199 // FrameSchedulingReceiver implementation. 200 // Called on packet sequence. 201 void OnEncodedFrame(std::unique_ptr<EncodedFrame> frame) override; 202 // Called on packet sequence. 203 void OnDecodableFrameTimeout(TimeDelta wait) override; 204 205 void CreateAndRegisterExternalDecoder(const Decoder& decoder); 206 207 struct DecodeFrameResult { 208 // True if the decoder returned code WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME, 209 // or if the decoder failed and a keyframe is required. When true, a 210 // keyframe request should be sent even if a keyframe request was sent 211 // recently. 212 bool force_request_key_frame; 213 214 // The picture id of the frame that was decoded, or nullopt if the frame was 215 // not decoded. 216 absl::optional<int64_t> decoded_frame_picture_id; 217 218 // True if the next frame decoded must be a keyframe. This value will set 219 // the value of `keyframe_required_`, which will force the frame buffer to 220 // drop all frames that are not keyframes. 221 bool keyframe_required; 222 }; 223 224 DecodeFrameResult HandleEncodedFrameOnDecodeQueue( 225 std::unique_ptr<EncodedFrame> frame, 226 bool keyframe_request_is_due, 227 bool keyframe_required) RTC_RUN_ON(decode_queue_); 228 void UpdatePlayoutDelays() const 229 RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_sequence_checker_); 230 void RequestKeyFrame(Timestamp now) RTC_RUN_ON(packet_sequence_checker_); 231 void HandleKeyFrameGeneration(bool received_frame_is_keyframe, 232 Timestamp now, 233 bool always_request_key_frame, 234 bool keyframe_request_is_due) 235 RTC_RUN_ON(packet_sequence_checker_); 236 bool IsReceivingKeyFrame(Timestamp timestamp) const 237 RTC_RUN_ON(packet_sequence_checker_); 238 int DecodeAndMaybeDispatchEncodedFrame(std::unique_ptr<EncodedFrame> frame) 239 RTC_RUN_ON(decode_queue_); 240 241 void UpdateHistograms(); 242 243 RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; 244 // TODO(bugs.webrtc.org/11993): This checker conceptually represents 245 // operations that belong to the network thread. The Call class is currently 246 // moving towards handling network packets on the network thread and while 247 // that work is ongoing, this checker may in practice represent the worker 248 // thread, but still serves as a mechanism of grouping together concepts 249 // that belong to the network thread. Once the packets are fully delivered 250 // on the network thread, this comment will be deleted. 251 RTC_NO_UNIQUE_ADDRESS SequenceChecker packet_sequence_checker_; 252 253 TaskQueueFactory* const task_queue_factory_; 254 255 TransportAdapter transport_adapter_; 256 const VideoReceiveStreamInterface::Config config_; 257 const int num_cpu_cores_; 258 Call* const call_; 259 Clock* const clock_; 260 261 CallStats* const call_stats_; 262 263 bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false; 264 bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true; 265 266 SourceTracker source_tracker_; 267 ReceiveStatisticsProxy stats_proxy_; 268 // Shared by media and rtx stream receivers, since the latter has no RtpRtcp 269 // module of its own. 270 const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; 271 272 std::unique_ptr<VCMTiming> timing_; // Jitter buffer experiment. 273 VideoReceiver2 video_receiver_; 274 std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_; 275 RtpVideoStreamReceiver2 rtp_video_stream_receiver_; 276 std::unique_ptr<VideoStreamDecoder> video_stream_decoder_; 277 RtpStreamsSynchronizer rtp_stream_sync_; 278 279 std::unique_ptr<VideoStreamBufferController> buffer_; 280 281 std::unique_ptr<RtpStreamReceiverInterface> media_receiver_ 282 RTC_GUARDED_BY(packet_sequence_checker_); 283 std::unique_ptr<RtxReceiveStream> rtx_receive_stream_ 284 RTC_GUARDED_BY(packet_sequence_checker_); 285 std::unique_ptr<RtpStreamReceiverInterface> rtx_receiver_ 286 RTC_GUARDED_BY(packet_sequence_checker_); 287 288 // Whenever we are in an undecodable state (stream has just started or due to 289 // a decoding error) we require a keyframe to restart the stream. 290 bool keyframe_required_ RTC_GUARDED_BY(packet_sequence_checker_) = true; 291 292 // If we have successfully decoded any frame. 293 bool frame_decoded_ RTC_GUARDED_BY(decode_queue_) = false; 294 295 absl::optional<Timestamp> last_keyframe_request_ 296 RTC_GUARDED_BY(packet_sequence_checker_); 297 298 // Keyframe request intervals are configurable through field trials. 299 TimeDelta max_wait_for_keyframe_ RTC_GUARDED_BY(packet_sequence_checker_); 300 TimeDelta max_wait_for_frame_ RTC_GUARDED_BY(packet_sequence_checker_); 301 302 // All of them tries to change current min_playout_delay on `timing_` but 303 // source of the change request is different in each case. Among them the 304 // biggest delay is used. -1 means use default value from the `timing_`. 305 // 306 // Minimum delay as decided by the RTP playout delay extension. 307 absl::optional<TimeDelta> frame_minimum_playout_delay_ 308 RTC_GUARDED_BY(worker_sequence_checker_); 309 // Minimum delay as decided by the setLatency function in "webrtc/api". 310 absl::optional<TimeDelta> base_minimum_playout_delay_ 311 RTC_GUARDED_BY(worker_sequence_checker_); 312 // Minimum delay as decided by the A/V synchronization feature. 313 absl::optional<TimeDelta> syncable_minimum_playout_delay_ 314 RTC_GUARDED_BY(worker_sequence_checker_); 315 316 // Maximum delay as decided by the RTP playout delay extension. 317 absl::optional<TimeDelta> frame_maximum_playout_delay_ 318 RTC_GUARDED_BY(worker_sequence_checker_); 319 320 // Function that is triggered with encoded frames, if not empty. 321 std::function<void(const RecordableEncodedFrame&)> 322 encoded_frame_buffer_function_ RTC_GUARDED_BY(decode_queue_); 323 // Set to true while we're requesting keyframes but not yet received one. 324 bool keyframe_generation_requested_ RTC_GUARDED_BY(packet_sequence_checker_) = 325 false; 326 // Lock to avoid unnecessary per-frame idle wakeups in the code. 327 webrtc::Mutex pending_resolution_mutex_; 328 // Signal from decode queue to OnFrame callback to fill pending_resolution_. 329 // absl::nullopt - no resolution needed. 0x0 - next OnFrame to fill with 330 // received resolution. Not 0x0 - OnFrame has filled a resolution. 331 absl::optional<RecordableEncodedFrame::EncodedResolution> pending_resolution_ 332 RTC_GUARDED_BY(pending_resolution_mutex_); 333 // Buffered encoded frames held while waiting for decoded resolution. 334 std::vector<std::unique_ptr<EncodedFrame>> buffered_encoded_frames_ 335 RTC_GUARDED_BY(decode_queue_); 336 337 // Defined last so they are destroyed before all other members. 338 rtc::TaskQueue decode_queue_; 339 340 // Used to signal destruction to potentially pending tasks. 341 ScopedTaskSafety task_safety_; 342 }; 343 344 } // namespace internal 345 } // namespace webrtc 346 347 #endif // VIDEO_VIDEO_RECEIVE_STREAM2_H_ 348