1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/base/checks.h"
12 #include "webrtc/base/logging.h"
13 #include "webrtc/base/trace_event.h"
14 #include "webrtc/common_types.h"
15 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
16 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
17 #include "webrtc/modules/video_coding/encoded_frame.h"
18 #include "webrtc/modules/video_coding/jitter_buffer.h"
19 #include "webrtc/modules/video_coding/packet.h"
20 #include "webrtc/modules/video_coding/video_coding_impl.h"
21 #include "webrtc/system_wrappers/include/clock.h"
22
23 // #define DEBUG_DECODER_BIT_STREAM
24
25 namespace webrtc {
26 namespace vcm {
27
VideoReceiver(Clock * clock,EventFactory * event_factory)28 VideoReceiver::VideoReceiver(Clock* clock, EventFactory* event_factory)
29 : clock_(clock),
30 process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
31 _receiveCritSect(CriticalSectionWrapper::CreateCriticalSection()),
32 _timing(clock_),
33 _receiver(&_timing, clock_, event_factory),
34 _decodedFrameCallback(&_timing, clock_),
35 _frameTypeCallback(NULL),
36 _receiveStatsCallback(NULL),
37 _decoderTimingCallback(NULL),
38 _packetRequestCallback(NULL),
39 render_buffer_callback_(NULL),
40 _decoder(NULL),
41 #ifdef DEBUG_DECODER_BIT_STREAM
42 _bitStreamBeforeDecoder(NULL),
43 #endif
44 _frameFromFile(),
45 _scheduleKeyRequest(false),
46 max_nack_list_size_(0),
47 pre_decode_image_callback_(NULL),
48 _codecDataBase(nullptr, nullptr),
49 _receiveStatsTimer(1000, clock_),
50 _retransmissionTimer(10, clock_),
51 _keyRequestTimer(500, clock_) {
52 assert(clock_);
53 #ifdef DEBUG_DECODER_BIT_STREAM
54 _bitStreamBeforeDecoder = fopen("decoderBitStream.bit", "wb");
55 #endif
56 }
57
~VideoReceiver()58 VideoReceiver::~VideoReceiver() {
59 delete _receiveCritSect;
60 #ifdef DEBUG_DECODER_BIT_STREAM
61 fclose(_bitStreamBeforeDecoder);
62 #endif
63 }
64
Process()65 int32_t VideoReceiver::Process() {
66 int32_t returnValue = VCM_OK;
67
68 // Receive-side statistics
69 if (_receiveStatsTimer.TimeUntilProcess() == 0) {
70 _receiveStatsTimer.Processed();
71 CriticalSectionScoped cs(process_crit_sect_.get());
72 if (_receiveStatsCallback != NULL) {
73 uint32_t bitRate;
74 uint32_t frameRate;
75 _receiver.ReceiveStatistics(&bitRate, &frameRate);
76 _receiveStatsCallback->OnReceiveRatesUpdated(bitRate, frameRate);
77 }
78
79 if (_decoderTimingCallback != NULL) {
80 int decode_ms;
81 int max_decode_ms;
82 int current_delay_ms;
83 int target_delay_ms;
84 int jitter_buffer_ms;
85 int min_playout_delay_ms;
86 int render_delay_ms;
87 _timing.GetTimings(&decode_ms, &max_decode_ms, ¤t_delay_ms,
88 &target_delay_ms, &jitter_buffer_ms,
89 &min_playout_delay_ms, &render_delay_ms);
90 _decoderTimingCallback->OnDecoderTiming(
91 decode_ms, max_decode_ms, current_delay_ms, target_delay_ms,
92 jitter_buffer_ms, min_playout_delay_ms, render_delay_ms);
93 }
94
95 // Size of render buffer.
96 if (render_buffer_callback_) {
97 int buffer_size_ms = _receiver.RenderBufferSizeMs();
98 render_buffer_callback_->RenderBufferSizeMs(buffer_size_ms);
99 }
100 }
101
102 // Key frame requests
103 if (_keyRequestTimer.TimeUntilProcess() == 0) {
104 _keyRequestTimer.Processed();
105 bool request_key_frame = false;
106 {
107 CriticalSectionScoped cs(process_crit_sect_.get());
108 request_key_frame = _scheduleKeyRequest && _frameTypeCallback != NULL;
109 }
110 if (request_key_frame) {
111 const int32_t ret = RequestKeyFrame();
112 if (ret != VCM_OK && returnValue == VCM_OK) {
113 returnValue = ret;
114 }
115 }
116 }
117
118 // Packet retransmission requests
119 // TODO(holmer): Add API for changing Process interval and make sure it's
120 // disabled when NACK is off.
121 if (_retransmissionTimer.TimeUntilProcess() == 0) {
122 _retransmissionTimer.Processed();
123 bool callback_registered = false;
124 uint16_t length;
125 {
126 CriticalSectionScoped cs(process_crit_sect_.get());
127 length = max_nack_list_size_;
128 callback_registered = _packetRequestCallback != NULL;
129 }
130 if (callback_registered && length > 0) {
131 // Collect sequence numbers from the default receiver.
132 bool request_key_frame = false;
133 std::vector<uint16_t> nackList = _receiver.NackList(&request_key_frame);
134 int32_t ret = VCM_OK;
135 if (request_key_frame) {
136 ret = RequestKeyFrame();
137 if (ret != VCM_OK && returnValue == VCM_OK) {
138 returnValue = ret;
139 }
140 }
141 if (ret == VCM_OK && !nackList.empty()) {
142 CriticalSectionScoped cs(process_crit_sect_.get());
143 if (_packetRequestCallback != NULL) {
144 _packetRequestCallback->ResendPackets(&nackList[0], nackList.size());
145 }
146 }
147 }
148 }
149
150 return returnValue;
151 }
152
TimeUntilNextProcess()153 int64_t VideoReceiver::TimeUntilNextProcess() {
154 int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess();
155 if (_receiver.NackMode() != kNoNack) {
156 // We need a Process call more often if we are relying on
157 // retransmissions
158 timeUntilNextProcess =
159 VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess());
160 }
161 timeUntilNextProcess =
162 VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess());
163
164 return timeUntilNextProcess;
165 }
166
SetReceiveChannelParameters(int64_t rtt)167 int32_t VideoReceiver::SetReceiveChannelParameters(int64_t rtt) {
168 CriticalSectionScoped receiveCs(_receiveCritSect);
169 _receiver.UpdateRtt(rtt);
170 return 0;
171 }
172
173 // Enable or disable a video protection method.
174 // Note: This API should be deprecated, as it does not offer a distinction
175 // between the protection method and decoding with or without errors. If such a
176 // behavior is desired, use the following API: SetReceiverRobustnessMode.
SetVideoProtection(VCMVideoProtection videoProtection,bool enable)177 int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection,
178 bool enable) {
179 // By default, do not decode with errors.
180 _receiver.SetDecodeErrorMode(kNoErrors);
181 switch (videoProtection) {
182 case kProtectionNack: {
183 RTC_DCHECK(enable);
184 _receiver.SetNackMode(kNack, -1, -1);
185 break;
186 }
187
188 case kProtectionNackFEC: {
189 CriticalSectionScoped cs(_receiveCritSect);
190 RTC_DCHECK(enable);
191 _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
192 _receiver.SetDecodeErrorMode(kNoErrors);
193 break;
194 }
195 case kProtectionFEC:
196 case kProtectionNone:
197 // No receiver-side protection.
198 RTC_DCHECK(enable);
199 _receiver.SetNackMode(kNoNack, -1, -1);
200 _receiver.SetDecodeErrorMode(kWithErrors);
201 break;
202 }
203 return VCM_OK;
204 }
205
206 // Register a receive callback. Will be called whenever there is a new frame
207 // ready for rendering.
RegisterReceiveCallback(VCMReceiveCallback * receiveCallback)208 int32_t VideoReceiver::RegisterReceiveCallback(
209 VCMReceiveCallback* receiveCallback) {
210 CriticalSectionScoped cs(_receiveCritSect);
211 _decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
212 return VCM_OK;
213 }
214
RegisterReceiveStatisticsCallback(VCMReceiveStatisticsCallback * receiveStats)215 int32_t VideoReceiver::RegisterReceiveStatisticsCallback(
216 VCMReceiveStatisticsCallback* receiveStats) {
217 CriticalSectionScoped cs(process_crit_sect_.get());
218 _receiver.RegisterStatsCallback(receiveStats);
219 _receiveStatsCallback = receiveStats;
220 return VCM_OK;
221 }
222
RegisterDecoderTimingCallback(VCMDecoderTimingCallback * decoderTiming)223 int32_t VideoReceiver::RegisterDecoderTimingCallback(
224 VCMDecoderTimingCallback* decoderTiming) {
225 CriticalSectionScoped cs(process_crit_sect_.get());
226 _decoderTimingCallback = decoderTiming;
227 return VCM_OK;
228 }
229
230 // Register an externally defined decoder object.
RegisterExternalDecoder(VideoDecoder * externalDecoder,uint8_t payloadType)231 void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
232 uint8_t payloadType) {
233 CriticalSectionScoped cs(_receiveCritSect);
234 if (externalDecoder == NULL) {
235 // Make sure the VCM updates the decoder next time it decodes.
236 _decoder = NULL;
237 RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
238 return;
239 }
240 _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType);
241 }
242
243 // Register a frame type request callback.
RegisterFrameTypeCallback(VCMFrameTypeCallback * frameTypeCallback)244 int32_t VideoReceiver::RegisterFrameTypeCallback(
245 VCMFrameTypeCallback* frameTypeCallback) {
246 CriticalSectionScoped cs(process_crit_sect_.get());
247 _frameTypeCallback = frameTypeCallback;
248 return VCM_OK;
249 }
250
RegisterPacketRequestCallback(VCMPacketRequestCallback * callback)251 int32_t VideoReceiver::RegisterPacketRequestCallback(
252 VCMPacketRequestCallback* callback) {
253 CriticalSectionScoped cs(process_crit_sect_.get());
254 _packetRequestCallback = callback;
255 return VCM_OK;
256 }
257
RegisterRenderBufferSizeCallback(VCMRenderBufferSizeCallback * callback)258 int VideoReceiver::RegisterRenderBufferSizeCallback(
259 VCMRenderBufferSizeCallback* callback) {
260 CriticalSectionScoped cs(process_crit_sect_.get());
261 render_buffer_callback_ = callback;
262 return VCM_OK;
263 }
264
TriggerDecoderShutdown()265 void VideoReceiver::TriggerDecoderShutdown() {
266 _receiver.TriggerDecoderShutdown();
267 }
268
269 // Decode next frame, blocking.
270 // Should be called as often as possible to get the most out of the decoder.
Decode(uint16_t maxWaitTimeMs)271 int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
272 int64_t nextRenderTimeMs;
273 bool prefer_late_decoding = false;
274 {
275 CriticalSectionScoped cs(_receiveCritSect);
276 prefer_late_decoding = _codecDataBase.PrefersLateDecoding();
277 }
278
279 VCMEncodedFrame* frame = _receiver.FrameForDecoding(
280 maxWaitTimeMs, &nextRenderTimeMs, prefer_late_decoding);
281
282 if (!frame)
283 return VCM_FRAME_NOT_READY;
284
285 CriticalSectionScoped cs(_receiveCritSect);
286
287 // If this frame was too late, we should adjust the delay accordingly
288 _timing.UpdateCurrentDelay(frame->RenderTimeMs(),
289 clock_->TimeInMilliseconds());
290
291 if (pre_decode_image_callback_) {
292 EncodedImage encoded_image(frame->EncodedImage());
293 int qp = -1;
294 if (qp_parser_.GetQp(*frame, &qp)) {
295 encoded_image.qp_ = qp;
296 }
297 pre_decode_image_callback_->Encoded(encoded_image, frame->CodecSpecific(),
298 NULL);
299 }
300
301 #ifdef DEBUG_DECODER_BIT_STREAM
302 if (_bitStreamBeforeDecoder != NULL) {
303 // Write bit stream to file for debugging purposes
304 if (fwrite(frame->Buffer(), 1, frame->Length(), _bitStreamBeforeDecoder) !=
305 frame->Length()) {
306 return -1;
307 }
308 }
309 #endif
310 const int32_t ret = Decode(*frame);
311 _receiver.ReleaseFrame(frame);
312 return ret;
313 }
314
RequestSliceLossIndication(const uint64_t pictureID) const315 int32_t VideoReceiver::RequestSliceLossIndication(
316 const uint64_t pictureID) const {
317 TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID);
318 CriticalSectionScoped cs(process_crit_sect_.get());
319 if (_frameTypeCallback != NULL) {
320 const int32_t ret =
321 _frameTypeCallback->SliceLossIndicationRequest(pictureID);
322 if (ret < 0) {
323 return ret;
324 }
325 } else {
326 return VCM_MISSING_CALLBACK;
327 }
328 return VCM_OK;
329 }
330
RequestKeyFrame()331 int32_t VideoReceiver::RequestKeyFrame() {
332 TRACE_EVENT0("webrtc", "RequestKeyFrame");
333 CriticalSectionScoped process_cs(process_crit_sect_.get());
334 if (_frameTypeCallback != NULL) {
335 const int32_t ret = _frameTypeCallback->RequestKeyFrame();
336 if (ret < 0) {
337 return ret;
338 }
339 _scheduleKeyRequest = false;
340 } else {
341 return VCM_MISSING_CALLBACK;
342 }
343 return VCM_OK;
344 }
345
346 // Must be called from inside the receive side critical section.
Decode(const VCMEncodedFrame & frame)347 int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
348 TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame.TimeStamp(), "Decode",
349 "type", frame.FrameType());
350 // Change decoder if payload type has changed
351 _decoder = _codecDataBase.GetDecoder(frame, &_decodedFrameCallback);
352 if (_decoder == NULL) {
353 return VCM_NO_CODEC_REGISTERED;
354 }
355 // Decode a frame
356 int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds());
357
358 // Check for failed decoding, run frame type request callback if needed.
359 bool request_key_frame = false;
360 if (ret < 0) {
361 if (ret == VCM_ERROR_REQUEST_SLI) {
362 return RequestSliceLossIndication(
363 _decodedFrameCallback.LastReceivedPictureID() + 1);
364 } else {
365 request_key_frame = true;
366 }
367 } else if (ret == VCM_REQUEST_SLI) {
368 ret = RequestSliceLossIndication(
369 _decodedFrameCallback.LastReceivedPictureID() + 1);
370 }
371 if (!frame.Complete() || frame.MissingFrame()) {
372 request_key_frame = true;
373 ret = VCM_OK;
374 }
375 if (request_key_frame) {
376 CriticalSectionScoped cs(process_crit_sect_.get());
377 _scheduleKeyRequest = true;
378 }
379 TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp());
380 return ret;
381 }
382
383 // Reset the decoder state
ResetDecoder()384 int32_t VideoReceiver::ResetDecoder() {
385 bool reset_key_request = false;
386 {
387 CriticalSectionScoped cs(_receiveCritSect);
388 if (_decoder != NULL) {
389 _receiver.Reset();
390 _timing.Reset();
391 reset_key_request = true;
392 _decoder->Reset();
393 }
394 }
395 if (reset_key_request) {
396 CriticalSectionScoped cs(process_crit_sect_.get());
397 _scheduleKeyRequest = false;
398 }
399 return VCM_OK;
400 }
401
402 // Register possible receive codecs, can be called multiple times
RegisterReceiveCodec(const VideoCodec * receiveCodec,int32_t numberOfCores,bool requireKeyFrame)403 int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
404 int32_t numberOfCores,
405 bool requireKeyFrame) {
406 CriticalSectionScoped cs(_receiveCritSect);
407 if (receiveCodec == NULL) {
408 return VCM_PARAMETER_ERROR;
409 }
410 if (!_codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores,
411 requireKeyFrame)) {
412 return -1;
413 }
414 return 0;
415 }
416
417 // Get current received codec
ReceiveCodec(VideoCodec * currentReceiveCodec) const418 int32_t VideoReceiver::ReceiveCodec(VideoCodec* currentReceiveCodec) const {
419 CriticalSectionScoped cs(_receiveCritSect);
420 if (currentReceiveCodec == NULL) {
421 return VCM_PARAMETER_ERROR;
422 }
423 return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1;
424 }
425
426 // Get current received codec
ReceiveCodec() const427 VideoCodecType VideoReceiver::ReceiveCodec() const {
428 CriticalSectionScoped cs(_receiveCritSect);
429 return _codecDataBase.ReceiveCodec();
430 }
431
432 // Incoming packet from network parsed and ready for decode, non blocking.
IncomingPacket(const uint8_t * incomingPayload,size_t payloadLength,const WebRtcRTPHeader & rtpInfo)433 int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
434 size_t payloadLength,
435 const WebRtcRTPHeader& rtpInfo) {
436 if (rtpInfo.frameType == kVideoFrameKey) {
437 TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum",
438 rtpInfo.header.sequenceNumber);
439 }
440 if (incomingPayload == NULL) {
441 // The jitter buffer doesn't handle non-zero payload lengths for packets
442 // without payload.
443 // TODO(holmer): We should fix this in the jitter buffer.
444 payloadLength = 0;
445 }
446 const VCMPacket packet(incomingPayload, payloadLength, rtpInfo);
447 int32_t ret = _receiver.InsertPacket(packet, rtpInfo.type.Video.width,
448 rtpInfo.type.Video.height);
449 // TODO(holmer): Investigate if this somehow should use the key frame
450 // request scheduling to throttle the requests.
451 if (ret == VCM_FLUSH_INDICATOR) {
452 RequestKeyFrame();
453 ResetDecoder();
454 } else if (ret < 0) {
455 return ret;
456 }
457 return VCM_OK;
458 }
459
460 // Minimum playout delay (used for lip-sync). This is the minimum delay required
461 // to sync with audio. Not included in VideoCodingModule::Delay()
462 // Defaults to 0 ms.
SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs)463 int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) {
464 _timing.set_min_playout_delay(minPlayoutDelayMs);
465 return VCM_OK;
466 }
467
468 // The estimated delay caused by rendering, defaults to
469 // kDefaultRenderDelayMs = 10 ms
SetRenderDelay(uint32_t timeMS)470 int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) {
471 _timing.set_render_delay(timeMS);
472 return VCM_OK;
473 }
474
475 // Current video delay
Delay() const476 int32_t VideoReceiver::Delay() const {
477 return _timing.TargetVideoDelay();
478 }
479
DiscardedPackets() const480 uint32_t VideoReceiver::DiscardedPackets() const {
481 return _receiver.DiscardedPackets();
482 }
483
SetReceiverRobustnessMode(ReceiverRobustness robustnessMode,VCMDecodeErrorMode decode_error_mode)484 int VideoReceiver::SetReceiverRobustnessMode(
485 ReceiverRobustness robustnessMode,
486 VCMDecodeErrorMode decode_error_mode) {
487 CriticalSectionScoped cs(_receiveCritSect);
488 switch (robustnessMode) {
489 case VideoCodingModule::kNone:
490 _receiver.SetNackMode(kNoNack, -1, -1);
491 break;
492 case VideoCodingModule::kHardNack:
493 // Always wait for retransmissions (except when decoding with errors).
494 _receiver.SetNackMode(kNack, -1, -1);
495 break;
496 case VideoCodingModule::kSoftNack:
497 #if 1
498 assert(false); // TODO(hlundin): Not completed.
499 return VCM_NOT_IMPLEMENTED;
500 #else
501 // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add
502 // extra delay when RTT is above kLowRttNackMs.
503 _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
504 break;
505 #endif
506 case VideoCodingModule::kReferenceSelection:
507 #if 1
508 assert(false); // TODO(hlundin): Not completed.
509 return VCM_NOT_IMPLEMENTED;
510 #else
511 if (decode_error_mode == kNoErrors) {
512 return VCM_PARAMETER_ERROR;
513 }
514 _receiver.SetNackMode(kNoNack, -1, -1);
515 break;
516 #endif
517 }
518 _receiver.SetDecodeErrorMode(decode_error_mode);
519 return VCM_OK;
520 }
521
SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode)522 void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
523 CriticalSectionScoped cs(_receiveCritSect);
524 _receiver.SetDecodeErrorMode(decode_error_mode);
525 }
526
SetNackSettings(size_t max_nack_list_size,int max_packet_age_to_nack,int max_incomplete_time_ms)527 void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
528 int max_packet_age_to_nack,
529 int max_incomplete_time_ms) {
530 if (max_nack_list_size != 0) {
531 CriticalSectionScoped process_cs(process_crit_sect_.get());
532 max_nack_list_size_ = max_nack_list_size;
533 }
534 _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
535 max_incomplete_time_ms);
536 }
537
SetMinReceiverDelay(int desired_delay_ms)538 int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) {
539 return _receiver.SetMinReceiverDelay(desired_delay_ms);
540 }
541
RegisterPreDecodeImageCallback(EncodedImageCallback * observer)542 void VideoReceiver::RegisterPreDecodeImageCallback(
543 EncodedImageCallback* observer) {
544 CriticalSectionScoped cs(_receiveCritSect);
545 pre_decode_image_callback_ = observer;
546 }
547
548 } // namespace vcm
549 } // namespace webrtc
550