1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/video_receive_stream2.h"
12
13 #include <stdlib.h>
14 #include <string.h>
15
16 #include <algorithm>
17 #include <memory>
18 #include <set>
19 #include <string>
20 #include <utility>
21
22 #include "absl/algorithm/container.h"
23 #include "absl/types/optional.h"
24 #include "api/array_view.h"
25 #include "api/crypto/frame_decryptor_interface.h"
26 #include "api/video/encoded_image.h"
27 #include "api/video_codecs/sdp_video_format.h"
28 #include "api/video_codecs/video_codec.h"
29 #include "api/video_codecs/video_decoder_factory.h"
30 #include "api/video_codecs/video_encoder.h"
31 #include "call/rtp_stream_receiver_controller_interface.h"
32 #include "call/rtx_receive_stream.h"
33 #include "common_video/include/incoming_video_stream.h"
34 #include "media/base/h264_profile_level_id.h"
35 #include "modules/video_coding/include/video_codec_interface.h"
36 #include "modules/video_coding/include/video_coding_defines.h"
37 #include "modules/video_coding/include/video_error_codes.h"
38 #include "modules/video_coding/timing.h"
39 #include "modules/video_coding/utility/vp8_header_parser.h"
40 #include "rtc_base/checks.h"
41 #include "rtc_base/experiments/keyframe_interval_settings.h"
42 #include "rtc_base/location.h"
43 #include "rtc_base/logging.h"
44 #include "rtc_base/strings/string_builder.h"
45 #include "rtc_base/system/thread_registry.h"
46 #include "rtc_base/time_utils.h"
47 #include "rtc_base/trace_event.h"
48 #include "system_wrappers/include/clock.h"
49 #include "system_wrappers/include/field_trial.h"
50 #include "video/call_stats2.h"
51 #include "video/frame_dumping_decoder.h"
52 #include "video/receive_statistics_proxy2.h"
53
54 namespace webrtc {
55
56 namespace internal {
57 constexpr int VideoReceiveStream2::kMaxWaitForKeyFrameMs;
58
59 namespace {
60
61 using video_coding::EncodedFrame;
62 using ReturnReason = video_coding::FrameBuffer::ReturnReason;
63
64 constexpr int kMinBaseMinimumDelayMs = 0;
65 constexpr int kMaxBaseMinimumDelayMs = 10000;
66
67 constexpr int kMaxWaitForFrameMs = 3000;
68
69 // Concrete instance of RecordableEncodedFrame wrapping needed content
70 // from video_coding::EncodedFrame.
71 class WebRtcRecordableEncodedFrame : public RecordableEncodedFrame {
72 public:
WebRtcRecordableEncodedFrame(const EncodedFrame & frame)73 explicit WebRtcRecordableEncodedFrame(const EncodedFrame& frame)
74 : buffer_(frame.GetEncodedData()),
75 render_time_ms_(frame.RenderTime()),
76 codec_(frame.CodecSpecific()->codecType),
77 is_key_frame_(frame.FrameType() == VideoFrameType::kVideoFrameKey),
78 resolution_{frame.EncodedImage()._encodedWidth,
79 frame.EncodedImage()._encodedHeight} {
80 if (frame.ColorSpace()) {
81 color_space_ = *frame.ColorSpace();
82 }
83 }
84
85 // VideoEncodedSinkInterface::FrameBuffer
encoded_buffer() const86 rtc::scoped_refptr<const EncodedImageBufferInterface> encoded_buffer()
87 const override {
88 return buffer_;
89 }
90
color_space() const91 absl::optional<webrtc::ColorSpace> color_space() const override {
92 return color_space_;
93 }
94
codec() const95 VideoCodecType codec() const override { return codec_; }
96
is_key_frame() const97 bool is_key_frame() const override { return is_key_frame_; }
98
resolution() const99 EncodedResolution resolution() const override { return resolution_; }
100
render_time() const101 Timestamp render_time() const override {
102 return Timestamp::Millis(render_time_ms_);
103 }
104
105 private:
106 rtc::scoped_refptr<EncodedImageBufferInterface> buffer_;
107 int64_t render_time_ms_;
108 VideoCodecType codec_;
109 bool is_key_frame_;
110 EncodedResolution resolution_;
111 absl::optional<webrtc::ColorSpace> color_space_;
112 };
113
CreateDecoderVideoCodec(const VideoReceiveStream::Decoder & decoder)114 VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) {
115 VideoCodec codec;
116 memset(&codec, 0, sizeof(codec));
117
118 codec.plType = decoder.payload_type;
119 codec.codecType = PayloadStringToCodecType(decoder.video_format.name);
120
121 if (codec.codecType == kVideoCodecVP8) {
122 *(codec.VP8()) = VideoEncoder::GetDefaultVp8Settings();
123 } else if (codec.codecType == kVideoCodecVP9) {
124 *(codec.VP9()) = VideoEncoder::GetDefaultVp9Settings();
125 } else if (codec.codecType == kVideoCodecH264) {
126 *(codec.H264()) = VideoEncoder::GetDefaultH264Settings();
127 } else if (codec.codecType == kVideoCodecMultiplex) {
128 VideoReceiveStream::Decoder associated_decoder = decoder;
129 associated_decoder.video_format =
130 SdpVideoFormat(CodecTypeToPayloadString(kVideoCodecVP9));
131 VideoCodec associated_codec = CreateDecoderVideoCodec(associated_decoder);
132 associated_codec.codecType = kVideoCodecMultiplex;
133 return associated_codec;
134 }
135
136 codec.width = 320;
137 codec.height = 180;
138 const int kDefaultStartBitrate = 300;
139 codec.startBitrate = codec.minBitrate = codec.maxBitrate =
140 kDefaultStartBitrate;
141
142 return codec;
143 }
144
145 // Video decoder class to be used for unknown codecs. Doesn't support decoding
146 // but logs messages to LS_ERROR.
147 class NullVideoDecoder : public webrtc::VideoDecoder {
148 public:
InitDecode(const webrtc::VideoCodec * codec_settings,int32_t number_of_cores)149 int32_t InitDecode(const webrtc::VideoCodec* codec_settings,
150 int32_t number_of_cores) override {
151 RTC_LOG(LS_ERROR) << "Can't initialize NullVideoDecoder.";
152 return WEBRTC_VIDEO_CODEC_OK;
153 }
154
Decode(const webrtc::EncodedImage & input_image,bool missing_frames,int64_t render_time_ms)155 int32_t Decode(const webrtc::EncodedImage& input_image,
156 bool missing_frames,
157 int64_t render_time_ms) override {
158 RTC_LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding.";
159 return WEBRTC_VIDEO_CODEC_OK;
160 }
161
RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback)162 int32_t RegisterDecodeCompleteCallback(
163 webrtc::DecodedImageCallback* callback) override {
164 RTC_LOG(LS_ERROR)
165 << "Can't register decode complete callback on NullVideoDecoder.";
166 return WEBRTC_VIDEO_CODEC_OK;
167 }
168
Release()169 int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; }
170
ImplementationName() const171 const char* ImplementationName() const override { return "NullVideoDecoder"; }
172 };
173
174 // TODO(https://bugs.webrtc.org/9974): Consider removing this workaround.
175 // Maximum time between frames before resetting the FrameBuffer to avoid RTP
176 // timestamps wraparound to affect FrameBuffer.
177 constexpr int kInactiveStreamThresholdMs = 600000; // 10 minutes.
178
179 } // namespace
180
VideoReceiveStream2(TaskQueueFactory * task_queue_factory,TaskQueueBase * current_queue,RtpStreamReceiverControllerInterface * receiver_controller,int num_cpu_cores,PacketRouter * packet_router,VideoReceiveStream::Config config,ProcessThread * process_thread,CallStats * call_stats,Clock * clock,VCMTiming * timing)181 VideoReceiveStream2::VideoReceiveStream2(
182 TaskQueueFactory* task_queue_factory,
183 TaskQueueBase* current_queue,
184 RtpStreamReceiverControllerInterface* receiver_controller,
185 int num_cpu_cores,
186 PacketRouter* packet_router,
187 VideoReceiveStream::Config config,
188 ProcessThread* process_thread,
189 CallStats* call_stats,
190 Clock* clock,
191 VCMTiming* timing)
192 : task_queue_factory_(task_queue_factory),
193 transport_adapter_(config.rtcp_send_transport),
194 config_(std::move(config)),
195 num_cpu_cores_(num_cpu_cores),
196 worker_thread_(current_queue),
197 clock_(clock),
198 call_stats_(call_stats),
199 source_tracker_(clock_),
200 stats_proxy_(&config_, clock_, worker_thread_),
201 rtp_receive_statistics_(ReceiveStatistics::Create(clock_)),
202 timing_(timing),
203 video_receiver_(clock_, timing_.get()),
204 rtp_video_stream_receiver_(worker_thread_,
205 clock_,
206 &transport_adapter_,
207 call_stats->AsRtcpRttStats(),
208 packet_router,
209 &config_,
210 rtp_receive_statistics_.get(),
211 &stats_proxy_,
212 &stats_proxy_,
213 process_thread,
214 this, // NackSender
215 nullptr, // Use default KeyFrameRequestSender
216 this, // OnCompleteFrameCallback
217 config_.frame_decryptor,
218 config_.frame_transformer),
219 rtp_stream_sync_(current_queue, this),
220 max_wait_for_keyframe_ms_(KeyframeIntervalSettings::ParseFromFieldTrials()
221 .MaxWaitForKeyframeMs()
222 .value_or(kMaxWaitForKeyFrameMs)),
223 max_wait_for_frame_ms_(KeyframeIntervalSettings::ParseFromFieldTrials()
224 .MaxWaitForFrameMs()
225 .value_or(kMaxWaitForFrameMs)),
226 decode_queue_(task_queue_factory_->CreateTaskQueue(
227 "DecodingQueue",
228 TaskQueueFactory::Priority::HIGH)) {
229 RTC_LOG(LS_INFO) << "VideoReceiveStream2: " << config_.ToString();
230
231 RTC_DCHECK(worker_thread_);
232 RTC_DCHECK(config_.renderer);
233 RTC_DCHECK(call_stats_);
234
235 module_process_sequence_checker_.Detach();
236
237 RTC_DCHECK(!config_.decoders.empty());
238 std::set<int> decoder_payload_types;
239 for (const Decoder& decoder : config_.decoders) {
240 RTC_CHECK(decoder.decoder_factory);
241 RTC_CHECK(decoder_payload_types.find(decoder.payload_type) ==
242 decoder_payload_types.end())
243 << "Duplicate payload type (" << decoder.payload_type
244 << ") for different decoders.";
245 decoder_payload_types.insert(decoder.payload_type);
246 }
247
248 timing_->set_render_delay(config_.render_delay_ms);
249
250 frame_buffer_.reset(
251 new video_coding::FrameBuffer(clock_, timing_.get(), &stats_proxy_));
252
253 // Register with RtpStreamReceiverController.
254 media_receiver_ = receiver_controller->CreateReceiver(
255 config_.rtp.remote_ssrc, &rtp_video_stream_receiver_);
256 if (config_.rtp.rtx_ssrc) {
257 rtx_receive_stream_ = std::make_unique<RtxReceiveStream>(
258 &rtp_video_stream_receiver_, config.rtp.rtx_associated_payload_types,
259 config_.rtp.remote_ssrc, rtp_receive_statistics_.get());
260 rtx_receiver_ = receiver_controller->CreateReceiver(
261 config_.rtp.rtx_ssrc, rtx_receive_stream_.get());
262 } else {
263 rtp_receive_statistics_->EnableRetransmitDetection(config.rtp.remote_ssrc,
264 true);
265 }
266 }
267
~VideoReceiveStream2()268 VideoReceiveStream2::~VideoReceiveStream2() {
269 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
270 RTC_LOG(LS_INFO) << "~VideoReceiveStream2: " << config_.ToString();
271 Stop();
272 }
273
SignalNetworkState(NetworkState state)274 void VideoReceiveStream2::SignalNetworkState(NetworkState state) {
275 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
276 rtp_video_stream_receiver_.SignalNetworkState(state);
277 }
278
DeliverRtcp(const uint8_t * packet,size_t length)279 bool VideoReceiveStream2::DeliverRtcp(const uint8_t* packet, size_t length) {
280 return rtp_video_stream_receiver_.DeliverRtcp(packet, length);
281 }
282
SetSync(Syncable * audio_syncable)283 void VideoReceiveStream2::SetSync(Syncable* audio_syncable) {
284 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
285 rtp_stream_sync_.ConfigureSync(audio_syncable);
286 }
287
Start()288 void VideoReceiveStream2::Start() {
289 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
290
291 if (decoder_running_) {
292 return;
293 }
294
295 const bool protected_by_fec = config_.rtp.protected_by_flexfec ||
296 rtp_video_stream_receiver_.IsUlpfecEnabled();
297
298 frame_buffer_->Start();
299
300 if (rtp_video_stream_receiver_.IsRetransmissionsEnabled() &&
301 protected_by_fec) {
302 frame_buffer_->SetProtectionMode(kProtectionNackFEC);
303 }
304
305 transport_adapter_.Enable();
306 rtc::VideoSinkInterface<VideoFrame>* renderer = nullptr;
307 if (config_.enable_prerenderer_smoothing) {
308 incoming_video_stream_.reset(new IncomingVideoStream(
309 task_queue_factory_, config_.render_delay_ms, this));
310 renderer = incoming_video_stream_.get();
311 } else {
312 renderer = this;
313 }
314
315 for (const Decoder& decoder : config_.decoders) {
316 std::unique_ptr<VideoDecoder> video_decoder =
317 decoder.decoder_factory->LegacyCreateVideoDecoder(decoder.video_format,
318 config_.stream_id);
319 // If we still have no valid decoder, we have to create a "Null" decoder
320 // that ignores all calls. The reason we can get into this state is that the
321 // old decoder factory interface doesn't have a way to query supported
322 // codecs.
323 if (!video_decoder) {
324 video_decoder = std::make_unique<NullVideoDecoder>();
325 }
326
327 std::string decoded_output_file =
328 field_trial::FindFullName("WebRTC-DecoderDataDumpDirectory");
329 // Because '/' can't be used inside a field trial parameter, we use ';'
330 // instead.
331 // This is only relevant to WebRTC-DecoderDataDumpDirectory
332 // field trial. ';' is chosen arbitrary. Even though it's a legal character
333 // in some file systems, we can sacrifice ability to use it in the path to
334 // dumped video, since it's developers-only feature for debugging.
335 absl::c_replace(decoded_output_file, ';', '/');
336 if (!decoded_output_file.empty()) {
337 char filename_buffer[256];
338 rtc::SimpleStringBuilder ssb(filename_buffer);
339 ssb << decoded_output_file << "/webrtc_receive_stream_"
340 << this->config_.rtp.remote_ssrc << "-" << rtc::TimeMicros()
341 << ".ivf";
342 video_decoder = CreateFrameDumpingDecoderWrapper(
343 std::move(video_decoder), FileWrapper::OpenWriteOnly(ssb.str()));
344 }
345
346 video_decoders_.push_back(std::move(video_decoder));
347
348 video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(),
349 decoder.payload_type);
350 VideoCodec codec = CreateDecoderVideoCodec(decoder);
351
352 const bool raw_payload =
353 config_.rtp.raw_payload_types.count(codec.plType) > 0;
354 rtp_video_stream_receiver_.AddReceiveCodec(
355 codec, decoder.video_format.parameters, raw_payload);
356 RTC_CHECK_EQ(VCM_OK, video_receiver_.RegisterReceiveCodec(
357 &codec, num_cpu_cores_, false));
358 }
359
360 RTC_DCHECK(renderer != nullptr);
361 video_stream_decoder_.reset(
362 new VideoStreamDecoder(&video_receiver_, &stats_proxy_, renderer));
363
364 // Make sure we register as a stats observer *after* we've prepared the
365 // |video_stream_decoder_|.
366 call_stats_->RegisterStatsObserver(this);
367
368 // Start decoding on task queue.
369 video_receiver_.DecoderThreadStarting();
370 stats_proxy_.DecoderThreadStarting();
371 decode_queue_.PostTask([this] {
372 RTC_DCHECK_RUN_ON(&decode_queue_);
373 decoder_stopped_ = false;
374 StartNextDecode();
375 });
376 decoder_running_ = true;
377 rtp_video_stream_receiver_.StartReceive();
378 }
379
Stop()380 void VideoReceiveStream2::Stop() {
381 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
382 rtp_video_stream_receiver_.StopReceive();
383
384 stats_proxy_.OnUniqueFramesCounted(
385 rtp_video_stream_receiver_.GetUniqueFramesSeen());
386
387 decode_queue_.PostTask([this] { frame_buffer_->Stop(); });
388
389 call_stats_->DeregisterStatsObserver(this);
390
391 if (decoder_running_) {
392 rtc::Event done;
393 decode_queue_.PostTask([this, &done] {
394 RTC_DCHECK_RUN_ON(&decode_queue_);
395 decoder_stopped_ = true;
396 done.Set();
397 });
398 done.Wait(rtc::Event::kForever);
399
400 decoder_running_ = false;
401 video_receiver_.DecoderThreadStopped();
402 stats_proxy_.DecoderThreadStopped();
403 // Deregister external decoders so they are no longer running during
404 // destruction. This effectively stops the VCM since the decoder thread is
405 // stopped, the VCM is deregistered and no asynchronous decoder threads are
406 // running.
407 for (const Decoder& decoder : config_.decoders)
408 video_receiver_.RegisterExternalDecoder(nullptr, decoder.payload_type);
409
410 UpdateHistograms();
411 }
412
413 video_stream_decoder_.reset();
414 incoming_video_stream_.reset();
415 transport_adapter_.Disable();
416 }
417
GetStats() const418 VideoReceiveStream::Stats VideoReceiveStream2::GetStats() const {
419 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
420 VideoReceiveStream2::Stats stats = stats_proxy_.GetStats();
421 stats.total_bitrate_bps = 0;
422 StreamStatistician* statistician =
423 rtp_receive_statistics_->GetStatistician(stats.ssrc);
424 if (statistician) {
425 stats.rtp_stats = statistician->GetStats();
426 stats.total_bitrate_bps = statistician->BitrateReceived();
427 }
428 if (config_.rtp.rtx_ssrc) {
429 StreamStatistician* rtx_statistician =
430 rtp_receive_statistics_->GetStatistician(config_.rtp.rtx_ssrc);
431 if (rtx_statistician)
432 stats.total_bitrate_bps += rtx_statistician->BitrateReceived();
433 }
434 return stats;
435 }
436
UpdateHistograms()437 void VideoReceiveStream2::UpdateHistograms() {
438 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
439 absl::optional<int> fraction_lost;
440 StreamDataCounters rtp_stats;
441 StreamStatistician* statistician =
442 rtp_receive_statistics_->GetStatistician(config_.rtp.remote_ssrc);
443 if (statistician) {
444 fraction_lost = statistician->GetFractionLostInPercent();
445 rtp_stats = statistician->GetReceiveStreamDataCounters();
446 }
447 if (config_.rtp.rtx_ssrc) {
448 StreamStatistician* rtx_statistician =
449 rtp_receive_statistics_->GetStatistician(config_.rtp.rtx_ssrc);
450 if (rtx_statistician) {
451 StreamDataCounters rtx_stats =
452 rtx_statistician->GetReceiveStreamDataCounters();
453 stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, &rtx_stats);
454 return;
455 }
456 }
457 stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, nullptr);
458 }
459
AddSecondarySink(RtpPacketSinkInterface * sink)460 void VideoReceiveStream2::AddSecondarySink(RtpPacketSinkInterface* sink) {
461 rtp_video_stream_receiver_.AddSecondarySink(sink);
462 }
463
RemoveSecondarySink(const RtpPacketSinkInterface * sink)464 void VideoReceiveStream2::RemoveSecondarySink(
465 const RtpPacketSinkInterface* sink) {
466 rtp_video_stream_receiver_.RemoveSecondarySink(sink);
467 }
468
SetBaseMinimumPlayoutDelayMs(int delay_ms)469 bool VideoReceiveStream2::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
470 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
471 if (delay_ms < kMinBaseMinimumDelayMs || delay_ms > kMaxBaseMinimumDelayMs) {
472 return false;
473 }
474
475 base_minimum_playout_delay_ms_ = delay_ms;
476 UpdatePlayoutDelays();
477 return true;
478 }
479
GetBaseMinimumPlayoutDelayMs() const480 int VideoReceiveStream2::GetBaseMinimumPlayoutDelayMs() const {
481 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
482 return base_minimum_playout_delay_ms_;
483 }
484
OnFrame(const VideoFrame & video_frame)485 void VideoReceiveStream2::OnFrame(const VideoFrame& video_frame) {
486 VideoFrameMetaData frame_meta(video_frame, clock_->CurrentTime());
487
488 worker_thread_->PostTask(
489 ToQueuedTask(task_safety_, [frame_meta, this]() {
490 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
491 int64_t video_playout_ntp_ms;
492 int64_t sync_offset_ms;
493 double estimated_freq_khz;
494 if (rtp_stream_sync_.GetStreamSyncOffsetInMs(
495 frame_meta.rtp_timestamp, frame_meta.render_time_ms(),
496 &video_playout_ntp_ms, &sync_offset_ms, &estimated_freq_khz)) {
497 stats_proxy_.OnSyncOffsetUpdated(video_playout_ntp_ms, sync_offset_ms,
498 estimated_freq_khz);
499 }
500 stats_proxy_.OnRenderedFrame(frame_meta);
501 }));
502
503 source_tracker_.OnFrameDelivered(video_frame.packet_infos());
504 config_.renderer->OnFrame(video_frame);
505 }
506
SetFrameDecryptor(rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor)507 void VideoReceiveStream2::SetFrameDecryptor(
508 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) {
509 rtp_video_stream_receiver_.SetFrameDecryptor(std::move(frame_decryptor));
510 }
511
SetDepacketizerToDecoderFrameTransformer(rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)512 void VideoReceiveStream2::SetDepacketizerToDecoderFrameTransformer(
513 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
514 rtp_video_stream_receiver_.SetDepacketizerToDecoderFrameTransformer(
515 std::move(frame_transformer));
516 }
517
SendNack(const std::vector<uint16_t> & sequence_numbers,bool buffering_allowed)518 void VideoReceiveStream2::SendNack(
519 const std::vector<uint16_t>& sequence_numbers,
520 bool buffering_allowed) {
521 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
522 RTC_DCHECK(buffering_allowed);
523 rtp_video_stream_receiver_.RequestPacketRetransmit(sequence_numbers);
524 }
525
RequestKeyFrame(int64_t timestamp_ms)526 void VideoReceiveStream2::RequestKeyFrame(int64_t timestamp_ms) {
527 // Running on worker_sequence_checker_.
528 // Called from RtpVideoStreamReceiver (rtp_video_stream_receiver_ is
529 // ultimately responsible).
530 rtp_video_stream_receiver_.RequestKeyFrame();
531 decode_queue_.PostTask([this, timestamp_ms]() {
532 RTC_DCHECK_RUN_ON(&decode_queue_);
533 last_keyframe_request_ms_ = timestamp_ms;
534 });
535 }
536
OnCompleteFrame(std::unique_ptr<video_coding::EncodedFrame> frame)537 void VideoReceiveStream2::OnCompleteFrame(
538 std::unique_ptr<video_coding::EncodedFrame> frame) {
539 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
540
541 // TODO(https://bugs.webrtc.org/9974): Consider removing this workaround.
542 int64_t time_now_ms = clock_->TimeInMilliseconds();
543 if (last_complete_frame_time_ms_ > 0 &&
544 time_now_ms - last_complete_frame_time_ms_ > kInactiveStreamThresholdMs) {
545 frame_buffer_->Clear();
546 }
547 last_complete_frame_time_ms_ = time_now_ms;
548
549 const PlayoutDelay& playout_delay = frame->EncodedImage().playout_delay_;
550 if (playout_delay.min_ms >= 0) {
551 frame_minimum_playout_delay_ms_ = playout_delay.min_ms;
552 UpdatePlayoutDelays();
553 }
554
555 if (playout_delay.max_ms >= 0) {
556 frame_maximum_playout_delay_ms_ = playout_delay.max_ms;
557 UpdatePlayoutDelays();
558 }
559
560 int64_t last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame));
561 if (last_continuous_pid != -1)
562 rtp_video_stream_receiver_.FrameContinuous(last_continuous_pid);
563 }
564
OnRttUpdate(int64_t avg_rtt_ms,int64_t max_rtt_ms)565 void VideoReceiveStream2::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) {
566 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
567 frame_buffer_->UpdateRtt(max_rtt_ms);
568 rtp_video_stream_receiver_.UpdateRtt(max_rtt_ms);
569 stats_proxy_.OnRttUpdate(avg_rtt_ms);
570 }
571
id() const572 uint32_t VideoReceiveStream2::id() const {
573 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
574 return config_.rtp.remote_ssrc;
575 }
576
GetInfo() const577 absl::optional<Syncable::Info> VideoReceiveStream2::GetInfo() const {
578 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
579 absl::optional<Syncable::Info> info =
580 rtp_video_stream_receiver_.GetSyncInfo();
581
582 if (!info)
583 return absl::nullopt;
584
585 info->current_delay_ms = timing_->TargetVideoDelay();
586 return info;
587 }
588
GetPlayoutRtpTimestamp(uint32_t * rtp_timestamp,int64_t * time_ms) const589 bool VideoReceiveStream2::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp,
590 int64_t* time_ms) const {
591 RTC_NOTREACHED();
592 return 0;
593 }
594
SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms,int64_t time_ms)595 void VideoReceiveStream2::SetEstimatedPlayoutNtpTimestampMs(
596 int64_t ntp_timestamp_ms,
597 int64_t time_ms) {
598 RTC_NOTREACHED();
599 }
600
SetMinimumPlayoutDelay(int delay_ms)601 void VideoReceiveStream2::SetMinimumPlayoutDelay(int delay_ms) {
602 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
603 syncable_minimum_playout_delay_ms_ = delay_ms;
604 UpdatePlayoutDelays();
605 }
606
GetMaxWaitMs() const607 int64_t VideoReceiveStream2::GetMaxWaitMs() const {
608 return keyframe_required_ ? max_wait_for_keyframe_ms_
609 : max_wait_for_frame_ms_;
610 }
611
StartNextDecode()612 void VideoReceiveStream2::StartNextDecode() {
613 // Running on the decode thread.
614 TRACE_EVENT0("webrtc", "VideoReceiveStream2::StartNextDecode");
615 frame_buffer_->NextFrame(
616 GetMaxWaitMs(), keyframe_required_, &decode_queue_,
617 /* encoded frame handler */
618 [this](std::unique_ptr<EncodedFrame> frame, ReturnReason res) {
619 RTC_DCHECK_EQ(frame == nullptr, res == ReturnReason::kTimeout);
620 RTC_DCHECK_EQ(frame != nullptr, res == ReturnReason::kFrameFound);
621 decode_queue_.PostTask([this, frame = std::move(frame)]() mutable {
622 RTC_DCHECK_RUN_ON(&decode_queue_);
623 if (decoder_stopped_)
624 return;
625 if (frame) {
626 HandleEncodedFrame(std::move(frame));
627 } else {
628 int64_t now_ms = clock_->TimeInMilliseconds();
629 worker_thread_->PostTask(ToQueuedTask(
630 task_safety_, [this, now_ms, wait_ms = GetMaxWaitMs()]() {
631 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
632 HandleFrameBufferTimeout(now_ms, wait_ms);
633 }));
634 }
635 StartNextDecode();
636 });
637 });
638 }
639
HandleEncodedFrame(std::unique_ptr<EncodedFrame> frame)640 void VideoReceiveStream2::HandleEncodedFrame(
641 std::unique_ptr<EncodedFrame> frame) {
642 // Running on |decode_queue_|.
643 int64_t now_ms = clock_->TimeInMilliseconds();
644
645 // Current OnPreDecode only cares about QP for VP8.
646 int qp = -1;
647 if (frame->CodecSpecific()->codecType == kVideoCodecVP8) {
648 if (!vp8::GetQp(frame->data(), frame->size(), &qp)) {
649 RTC_LOG(LS_WARNING) << "Failed to extract QP from VP8 video frame";
650 }
651 }
652 stats_proxy_.OnPreDecode(frame->CodecSpecific()->codecType, qp);
653
654 bool force_request_key_frame = false;
655 int64_t decoded_frame_picture_id = -1;
656
657 const bool keyframe_request_is_due =
658 now_ms >= (last_keyframe_request_ms_ + max_wait_for_keyframe_ms_);
659
660 int decode_result = video_receiver_.Decode(frame.get());
661 if (decode_result == WEBRTC_VIDEO_CODEC_OK ||
662 decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) {
663 keyframe_required_ = false;
664 frame_decoded_ = true;
665
666 decoded_frame_picture_id = frame->id.picture_id;
667
668 if (decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME)
669 force_request_key_frame = true;
670 } else if (!frame_decoded_ || !keyframe_required_ ||
671 keyframe_request_is_due) {
672 keyframe_required_ = true;
673 // TODO(philipel): Remove this keyframe request when downstream project
674 // has been fixed.
675 force_request_key_frame = true;
676 }
677
678 bool received_frame_is_keyframe =
679 frame->FrameType() == VideoFrameType::kVideoFrameKey;
680
681 worker_thread_->PostTask(ToQueuedTask(
682 task_safety_,
683 [this, now_ms, received_frame_is_keyframe, force_request_key_frame,
684 decoded_frame_picture_id, keyframe_request_is_due]() {
685 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
686
687 if (decoded_frame_picture_id != -1)
688 rtp_video_stream_receiver_.FrameDecoded(decoded_frame_picture_id);
689
690 HandleKeyFrameGeneration(received_frame_is_keyframe, now_ms,
691 force_request_key_frame,
692 keyframe_request_is_due);
693 }));
694
695 if (encoded_frame_buffer_function_) {
696 frame->Retain();
697 encoded_frame_buffer_function_(WebRtcRecordableEncodedFrame(*frame));
698 }
699 }
700
HandleKeyFrameGeneration(bool received_frame_is_keyframe,int64_t now_ms,bool always_request_key_frame,bool keyframe_request_is_due)701 void VideoReceiveStream2::HandleKeyFrameGeneration(
702 bool received_frame_is_keyframe,
703 int64_t now_ms,
704 bool always_request_key_frame,
705 bool keyframe_request_is_due) {
706 // Running on worker_sequence_checker_.
707
708 bool request_key_frame = always_request_key_frame;
709
710 // Repeat sending keyframe requests if we've requested a keyframe.
711 if (keyframe_generation_requested_) {
712 if (received_frame_is_keyframe) {
713 keyframe_generation_requested_ = false;
714 } else if (keyframe_request_is_due) {
715 if (!IsReceivingKeyFrame(now_ms)) {
716 request_key_frame = true;
717 }
718 } else {
719 // It hasn't been long enough since the last keyframe request, do nothing.
720 }
721 }
722
723 if (request_key_frame) {
724 // HandleKeyFrameGeneration is initated from the decode thread -
725 // RequestKeyFrame() triggers a call back to the decode thread.
726 // Perhaps there's a way to avoid that.
727 RequestKeyFrame(now_ms);
728 }
729 }
730
HandleFrameBufferTimeout(int64_t now_ms,int64_t wait_ms)731 void VideoReceiveStream2::HandleFrameBufferTimeout(int64_t now_ms,
732 int64_t wait_ms) {
733 // Running on |worker_sequence_checker_|.
734 absl::optional<int64_t> last_packet_ms =
735 rtp_video_stream_receiver_.LastReceivedPacketMs();
736
737 // To avoid spamming keyframe requests for a stream that is not active we
738 // check if we have received a packet within the last 5 seconds.
739 const bool stream_is_active =
740 last_packet_ms && now_ms - *last_packet_ms < 5000;
741 if (!stream_is_active)
742 stats_proxy_.OnStreamInactive();
743
744 if (stream_is_active && !IsReceivingKeyFrame(now_ms) &&
745 (!config_.crypto_options.sframe.require_frame_encryption ||
746 rtp_video_stream_receiver_.IsDecryptable())) {
747 RTC_LOG(LS_WARNING) << "No decodable frame in " << wait_ms
748 << " ms, requesting keyframe.";
749 RequestKeyFrame(now_ms);
750 }
751 }
752
IsReceivingKeyFrame(int64_t timestamp_ms) const753 bool VideoReceiveStream2::IsReceivingKeyFrame(int64_t timestamp_ms) const {
754 // Running on worker_sequence_checker_.
755 absl::optional<int64_t> last_keyframe_packet_ms =
756 rtp_video_stream_receiver_.LastReceivedKeyframePacketMs();
757
758 // If we recently have been receiving packets belonging to a keyframe then
759 // we assume a keyframe is currently being received.
760 bool receiving_keyframe =
761 last_keyframe_packet_ms &&
762 timestamp_ms - *last_keyframe_packet_ms < max_wait_for_keyframe_ms_;
763 return receiving_keyframe;
764 }
765
UpdatePlayoutDelays() const766 void VideoReceiveStream2::UpdatePlayoutDelays() const {
767 // Running on worker_sequence_checker_.
768 const int minimum_delay_ms =
769 std::max({frame_minimum_playout_delay_ms_, base_minimum_playout_delay_ms_,
770 syncable_minimum_playout_delay_ms_});
771 if (minimum_delay_ms >= 0) {
772 timing_->set_min_playout_delay(minimum_delay_ms);
773 }
774
775 const int maximum_delay_ms = frame_maximum_playout_delay_ms_;
776 if (maximum_delay_ms >= 0) {
777 timing_->set_max_playout_delay(maximum_delay_ms);
778 }
779 }
780
GetSources() const781 std::vector<webrtc::RtpSource> VideoReceiveStream2::GetSources() const {
782 return source_tracker_.GetSources();
783 }
784
785 VideoReceiveStream2::RecordingState
SetAndGetRecordingState(RecordingState state,bool generate_key_frame)786 VideoReceiveStream2::SetAndGetRecordingState(RecordingState state,
787 bool generate_key_frame) {
788 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
789 rtc::Event event;
790
791 // Save old state, set the new state.
792 RecordingState old_state;
793
794 decode_queue_.PostTask(
795 [this, &event, &old_state, callback = std::move(state.callback),
796 generate_key_frame,
797 last_keyframe_request = state.last_keyframe_request_ms.value_or(0)] {
798 RTC_DCHECK_RUN_ON(&decode_queue_);
799 old_state.callback = std::move(encoded_frame_buffer_function_);
800 encoded_frame_buffer_function_ = std::move(callback);
801
802 old_state.last_keyframe_request_ms = last_keyframe_request_ms_;
803 last_keyframe_request_ms_ = generate_key_frame
804 ? clock_->TimeInMilliseconds()
805 : last_keyframe_request;
806
807 event.Set();
808 });
809
810 old_state.keyframe_needed = keyframe_generation_requested_;
811
812 if (generate_key_frame) {
813 rtp_video_stream_receiver_.RequestKeyFrame();
814 keyframe_generation_requested_ = true;
815 } else {
816 keyframe_generation_requested_ = state.keyframe_needed;
817 }
818
819 event.Wait(rtc::Event::kForever);
820 return old_state;
821 }
822
GenerateKeyFrame()823 void VideoReceiveStream2::GenerateKeyFrame() {
824 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
825 RequestKeyFrame(clock_->TimeInMilliseconds());
826 keyframe_generation_requested_ = true;
827 }
828
829 } // namespace internal
830 } // namespace webrtc
831