1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <stddef.h>
12
13 #include <cstdint>
14 #include <vector>
15
16 #include "api/rtp_headers.h"
17 #include "api/video_codecs/video_codec.h"
18 #include "api/video_codecs/video_decoder.h"
19 #include "modules/include/module_common_types.h"
20 #include "modules/utility/include/process_thread.h"
21 #include "modules/video_coding/decoder_database.h"
22 #include "modules/video_coding/encoded_frame.h"
23 #include "modules/video_coding/generic_decoder.h"
24 #include "modules/video_coding/include/video_coding.h"
25 #include "modules/video_coding/include/video_coding_defines.h"
26 #include "modules/video_coding/internal_defines.h"
27 #include "modules/video_coding/jitter_buffer.h"
28 #include "modules/video_coding/media_opt_util.h"
29 #include "modules/video_coding/packet.h"
30 #include "modules/video_coding/receiver.h"
31 #include "modules/video_coding/timing.h"
32 #include "modules/video_coding/video_coding_impl.h"
33 #include "rtc_base/checks.h"
34 #include "rtc_base/location.h"
35 #include "rtc_base/logging.h"
36 #include "rtc_base/one_time_event.h"
37 #include "rtc_base/thread_checker.h"
38 #include "rtc_base/trace_event.h"
39 #include "system_wrappers/include/clock.h"
40
41 namespace webrtc {
42 namespace vcm {
43
VideoReceiver(Clock * clock,VCMTiming * timing)44 VideoReceiver::VideoReceiver(Clock* clock, VCMTiming* timing)
45 : clock_(clock),
46 _timing(timing),
47 _receiver(_timing, clock_),
48 _decodedFrameCallback(_timing, clock_),
49 _frameTypeCallback(nullptr),
50 _packetRequestCallback(nullptr),
51 _scheduleKeyRequest(false),
52 drop_frames_until_keyframe_(false),
53 max_nack_list_size_(0),
54 _codecDataBase(),
55 _retransmissionTimer(10, clock_),
56 _keyRequestTimer(500, clock_) {
57 decoder_thread_checker_.Detach();
58 module_thread_checker_.Detach();
59 }
60
~VideoReceiver()61 VideoReceiver::~VideoReceiver() {
62 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
63 }
64
Process()65 void VideoReceiver::Process() {
66 RTC_DCHECK_RUN_ON(&module_thread_checker_);
67
68 // Key frame requests
69 if (_keyRequestTimer.TimeUntilProcess() == 0) {
70 _keyRequestTimer.Processed();
71 bool request_key_frame = _frameTypeCallback != nullptr;
72 if (request_key_frame) {
73 MutexLock lock(&process_mutex_);
74 request_key_frame = _scheduleKeyRequest;
75 }
76 if (request_key_frame)
77 RequestKeyFrame();
78 }
79
80 // Packet retransmission requests
81 // TODO(holmer): Add API for changing Process interval and make sure it's
82 // disabled when NACK is off.
83 if (_retransmissionTimer.TimeUntilProcess() == 0) {
84 _retransmissionTimer.Processed();
85 bool callback_registered = _packetRequestCallback != nullptr;
86 uint16_t length = max_nack_list_size_;
87 if (callback_registered && length > 0) {
88 // Collect sequence numbers from the default receiver.
89 bool request_key_frame = false;
90 std::vector<uint16_t> nackList = _receiver.NackList(&request_key_frame);
91 int32_t ret = VCM_OK;
92 if (request_key_frame) {
93 ret = RequestKeyFrame();
94 }
95 if (ret == VCM_OK && !nackList.empty()) {
96 MutexLock lock(&process_mutex_);
97 if (_packetRequestCallback != nullptr) {
98 _packetRequestCallback->ResendPackets(&nackList[0], nackList.size());
99 }
100 }
101 }
102 }
103 }
104
ProcessThreadAttached(ProcessThread * process_thread)105 void VideoReceiver::ProcessThreadAttached(ProcessThread* process_thread) {
106 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
107 if (process_thread) {
108 is_attached_to_process_thread_ = true;
109 RTC_DCHECK(!process_thread_ || process_thread_ == process_thread);
110 process_thread_ = process_thread;
111 } else {
112 is_attached_to_process_thread_ = false;
113 }
114 }
115
TimeUntilNextProcess()116 int64_t VideoReceiver::TimeUntilNextProcess() {
117 RTC_DCHECK_RUN_ON(&module_thread_checker_);
118 int64_t timeUntilNextProcess = _retransmissionTimer.TimeUntilProcess();
119
120 timeUntilNextProcess =
121 VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess());
122
123 return timeUntilNextProcess;
124 }
125
126 // Register a receive callback. Will be called whenever there is a new frame
127 // ready for rendering.
RegisterReceiveCallback(VCMReceiveCallback * receiveCallback)128 int32_t VideoReceiver::RegisterReceiveCallback(
129 VCMReceiveCallback* receiveCallback) {
130 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
131 // This value is set before the decoder thread starts and unset after
132 // the decoder thread has been stopped.
133 _decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
134 return VCM_OK;
135 }
136
137 // Register an externally defined decoder object.
RegisterExternalDecoder(VideoDecoder * externalDecoder,uint8_t payloadType)138 void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
139 uint8_t payloadType) {
140 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
141 if (externalDecoder == nullptr) {
142 RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
143 return;
144 }
145 _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType);
146 }
147
148 // Register a frame type request callback.
RegisterFrameTypeCallback(VCMFrameTypeCallback * frameTypeCallback)149 int32_t VideoReceiver::RegisterFrameTypeCallback(
150 VCMFrameTypeCallback* frameTypeCallback) {
151 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
152 RTC_DCHECK(!is_attached_to_process_thread_);
153 // This callback is used on the module thread, but since we don't get
154 // callbacks on the module thread while the decoder thread isn't running
155 // (and this function must not be called when the decoder is running),
156 // we don't need a lock here.
157 _frameTypeCallback = frameTypeCallback;
158 return VCM_OK;
159 }
160
RegisterPacketRequestCallback(VCMPacketRequestCallback * callback)161 int32_t VideoReceiver::RegisterPacketRequestCallback(
162 VCMPacketRequestCallback* callback) {
163 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
164 RTC_DCHECK(!is_attached_to_process_thread_);
165 // This callback is used on the module thread, but since we don't get
166 // callbacks on the module thread while the decoder thread isn't running
167 // (and this function must not be called when the decoder is running),
168 // we don't need a lock here.
169 _packetRequestCallback = callback;
170 return VCM_OK;
171 }
172
173 // Decode next frame, blocking.
174 // Should be called as often as possible to get the most out of the decoder.
Decode(uint16_t maxWaitTimeMs)175 int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
176 RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
177 VCMEncodedFrame* frame = _receiver.FrameForDecoding(
178 maxWaitTimeMs, _codecDataBase.PrefersLateDecoding());
179
180 if (!frame)
181 return VCM_FRAME_NOT_READY;
182
183 bool drop_frame = false;
184 {
185 MutexLock lock(&process_mutex_);
186 if (drop_frames_until_keyframe_) {
187 // Still getting delta frames, schedule another keyframe request as if
188 // decode failed.
189 if (frame->FrameType() != VideoFrameType::kVideoFrameKey) {
190 drop_frame = true;
191 _scheduleKeyRequest = true;
192 // TODO(tommi): Consider if we could instead post a task to the module
193 // thread and call RequestKeyFrame directly. Here we call WakeUp so that
194 // TimeUntilNextProcess() gets called straight away.
195 process_thread_->WakeUp(this);
196 } else {
197 drop_frames_until_keyframe_ = false;
198 }
199 }
200 }
201
202 if (drop_frame) {
203 _receiver.ReleaseFrame(frame);
204 return VCM_FRAME_NOT_READY;
205 }
206
207 // If this frame was too late, we should adjust the delay accordingly
208 _timing->UpdateCurrentDelay(frame->RenderTimeMs(),
209 clock_->TimeInMilliseconds());
210
211 if (first_frame_received_()) {
212 RTC_LOG(LS_INFO) << "Received first "
213 << (frame->Complete() ? "complete" : "incomplete")
214 << " decodable video frame";
215 }
216
217 const int32_t ret = Decode(*frame);
218 _receiver.ReleaseFrame(frame);
219 return ret;
220 }
221
RequestKeyFrame()222 int32_t VideoReceiver::RequestKeyFrame() {
223 RTC_DCHECK_RUN_ON(&module_thread_checker_);
224
225 TRACE_EVENT0("webrtc", "RequestKeyFrame");
226 if (_frameTypeCallback != nullptr) {
227 const int32_t ret = _frameTypeCallback->RequestKeyFrame();
228 if (ret < 0) {
229 return ret;
230 }
231 MutexLock lock(&process_mutex_);
232 _scheduleKeyRequest = false;
233 } else {
234 return VCM_MISSING_CALLBACK;
235 }
236 return VCM_OK;
237 }
238
239 // Must be called from inside the receive side critical section.
Decode(const VCMEncodedFrame & frame)240 int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
241 RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
242 TRACE_EVENT0("webrtc", "VideoReceiver::Decode");
243 // Change decoder if payload type has changed
244 VCMGenericDecoder* decoder =
245 _codecDataBase.GetDecoder(frame, &_decodedFrameCallback);
246 if (decoder == nullptr) {
247 return VCM_NO_CODEC_REGISTERED;
248 }
249 return decoder->Decode(frame, clock_->CurrentTime());
250 }
251
252 // Register possible receive codecs, can be called multiple times
RegisterReceiveCodec(const VideoCodec * receiveCodec,int32_t numberOfCores,bool requireKeyFrame)253 int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
254 int32_t numberOfCores,
255 bool requireKeyFrame) {
256 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
257 if (receiveCodec == nullptr) {
258 return VCM_PARAMETER_ERROR;
259 }
260 if (!_codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores,
261 requireKeyFrame)) {
262 return -1;
263 }
264 return 0;
265 }
266
267 // Incoming packet from network parsed and ready for decode, non blocking.
IncomingPacket(const uint8_t * incomingPayload,size_t payloadLength,const RTPHeader & rtp_header,const RTPVideoHeader & video_header)268 int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
269 size_t payloadLength,
270 const RTPHeader& rtp_header,
271 const RTPVideoHeader& video_header) {
272 RTC_DCHECK_RUN_ON(&module_thread_checker_);
273 if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
274 TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum",
275 rtp_header.sequenceNumber);
276 }
277 if (incomingPayload == nullptr) {
278 // The jitter buffer doesn't handle non-zero payload lengths for packets
279 // without payload.
280 // TODO(holmer): We should fix this in the jitter buffer.
281 payloadLength = 0;
282 }
283 // Callers don't provide any ntp time.
284 const VCMPacket packet(incomingPayload, payloadLength, rtp_header,
285 video_header, /*ntp_time_ms=*/0,
286 clock_->TimeInMilliseconds());
287 int32_t ret = _receiver.InsertPacket(packet);
288
289 // TODO(holmer): Investigate if this somehow should use the key frame
290 // request scheduling to throttle the requests.
291 if (ret == VCM_FLUSH_INDICATOR) {
292 {
293 MutexLock lock(&process_mutex_);
294 drop_frames_until_keyframe_ = true;
295 }
296 RequestKeyFrame();
297 } else if (ret < 0) {
298 return ret;
299 }
300 return VCM_OK;
301 }
302
SetNackSettings(size_t max_nack_list_size,int max_packet_age_to_nack,int max_incomplete_time_ms)303 void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
304 int max_packet_age_to_nack,
305 int max_incomplete_time_ms) {
306 RTC_DCHECK_RUN_ON(&construction_thread_checker_);
307 if (max_nack_list_size != 0) {
308 max_nack_list_size_ = max_nack_list_size;
309 }
310 _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
311 max_incomplete_time_ms);
312 }
313
314 } // namespace vcm
315 } // namespace webrtc
316