1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/video_stream_decoder_impl.h"
12
13 #include <memory>
14
15 #include "api/task_queue/queued_task.h"
16 #include "rtc_base/logging.h"
17 #include "rtc_base/numerics/mod_ops.h"
18 #include "rtc_base/time_utils.h"
19
20 namespace webrtc {
21
VideoStreamDecoderImpl(VideoStreamDecoderInterface::Callbacks * callbacks,VideoDecoderFactory * decoder_factory,TaskQueueFactory * task_queue_factory,std::map<int,std::pair<SdpVideoFormat,int>> decoder_settings)22 VideoStreamDecoderImpl::VideoStreamDecoderImpl(
23 VideoStreamDecoderInterface::Callbacks* callbacks,
24 VideoDecoderFactory* decoder_factory,
25 TaskQueueFactory* task_queue_factory,
26 std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings)
27 : timing_(Clock::GetRealTimeClock()),
28 decode_callbacks_(this),
29 next_frame_timestamps_index_(0),
30 callbacks_(callbacks),
31 keyframe_required_(true),
32 decoder_factory_(decoder_factory),
33 decoder_settings_(std::move(decoder_settings)),
34 shut_down_(false),
35 frame_buffer_(Clock::GetRealTimeClock(), &timing_, nullptr),
36 bookkeeping_queue_(task_queue_factory->CreateTaskQueue(
37 "video_stream_decoder_bookkeeping_queue",
38 TaskQueueFactory::Priority::NORMAL)),
39 decode_queue_(task_queue_factory->CreateTaskQueue(
40 "video_stream_decoder_decode_queue",
41 TaskQueueFactory::Priority::NORMAL)) {
42 frame_timestamps_.fill({-1, -1, -1});
43 bookkeeping_queue_.PostTask([this]() {
44 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
45 StartNextDecode();
46 });
47 }
48
~VideoStreamDecoderImpl()49 VideoStreamDecoderImpl::~VideoStreamDecoderImpl() {
50 MutexLock lock(&shut_down_mutex_);
51 shut_down_ = true;
52 }
53
OnFrame(std::unique_ptr<video_coding::EncodedFrame> frame)54 void VideoStreamDecoderImpl::OnFrame(
55 std::unique_ptr<video_coding::EncodedFrame> frame) {
56 if (!bookkeeping_queue_.IsCurrent()) {
57 bookkeeping_queue_.PostTask([this, frame = std::move(frame)]() mutable {
58 OnFrame(std::move(frame));
59 return true;
60 });
61
62 return;
63 }
64
65 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
66
67 uint64_t continuous_pid = frame_buffer_.InsertFrame(std::move(frame));
68 video_coding::VideoLayerFrameId continuous_id(continuous_pid, 0);
69 if (last_continuous_id_ < continuous_id) {
70 last_continuous_id_ = continuous_id;
71 callbacks_->OnContinuousUntil(last_continuous_id_);
72 }
73 }
74
SetMinPlayoutDelay(TimeDelta min_delay)75 void VideoStreamDecoderImpl::SetMinPlayoutDelay(TimeDelta min_delay) {
76 timing_.set_min_playout_delay(min_delay.ms());
77 }
78
SetMaxPlayoutDelay(TimeDelta max_delay)79 void VideoStreamDecoderImpl::SetMaxPlayoutDelay(TimeDelta max_delay) {
80 timing_.set_max_playout_delay(max_delay.ms());
81 }
82
GetDecoder(int payload_type)83 VideoDecoder* VideoStreamDecoderImpl::GetDecoder(int payload_type) {
84 if (current_payload_type_ == payload_type) {
85 RTC_DCHECK(decoder_);
86 return decoder_.get();
87 }
88
89 current_payload_type_.reset();
90 decoder_.reset();
91
92 auto decoder_settings_it = decoder_settings_.find(payload_type);
93 if (decoder_settings_it == decoder_settings_.end()) {
94 RTC_LOG(LS_WARNING) << "Payload type " << payload_type
95 << " not registered.";
96 return nullptr;
97 }
98
99 const SdpVideoFormat& video_format = decoder_settings_it->second.first;
100 std::unique_ptr<VideoDecoder> decoder =
101 decoder_factory_->CreateVideoDecoder(video_format);
102 if (!decoder) {
103 RTC_LOG(LS_WARNING) << "Failed to create decoder for payload type "
104 << payload_type << ".";
105 return nullptr;
106 }
107
108 int num_cores = decoder_settings_it->second.second;
109 int32_t init_result = decoder->InitDecode(nullptr, num_cores);
110 if (init_result != WEBRTC_VIDEO_CODEC_OK) {
111 RTC_LOG(LS_WARNING) << "Failed to initialize decoder for payload type "
112 << payload_type << ".";
113 return nullptr;
114 }
115
116 int32_t register_result =
117 decoder->RegisterDecodeCompleteCallback(&decode_callbacks_);
118 if (register_result != WEBRTC_VIDEO_CODEC_OK) {
119 RTC_LOG(LS_WARNING) << "Failed to register decode callback.";
120 return nullptr;
121 }
122
123 current_payload_type_.emplace(payload_type);
124 decoder_ = std::move(decoder);
125 return decoder_.get();
126 }
127
SaveFrameTimestamps(const video_coding::EncodedFrame & frame)128 void VideoStreamDecoderImpl::SaveFrameTimestamps(
129 const video_coding::EncodedFrame& frame) {
130 FrameTimestamps* frame_timestamps =
131 &frame_timestamps_[next_frame_timestamps_index_];
132 frame_timestamps->timestamp = frame.Timestamp();
133 frame_timestamps->decode_start_time_ms = rtc::TimeMillis();
134 frame_timestamps->render_time_us = frame.RenderTimeMs() * 1000;
135
136 next_frame_timestamps_index_ =
137 Add<kFrameTimestampsMemory>(next_frame_timestamps_index_, 1);
138 }
139
StartNextDecode()140 void VideoStreamDecoderImpl::StartNextDecode() {
141 int64_t max_wait_time = keyframe_required_ ? 200 : 3000;
142
143 frame_buffer_.NextFrame(
144 max_wait_time, keyframe_required_, &bookkeeping_queue_,
145 [this](std::unique_ptr<video_coding::EncodedFrame> frame,
146 video_coding::FrameBuffer::ReturnReason res) mutable {
147 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
148 OnNextFrameCallback(std::move(frame), res);
149 });
150 }
151
OnNextFrameCallback(std::unique_ptr<video_coding::EncodedFrame> frame,video_coding::FrameBuffer::ReturnReason result)152 void VideoStreamDecoderImpl::OnNextFrameCallback(
153 std::unique_ptr<video_coding::EncodedFrame> frame,
154 video_coding::FrameBuffer::ReturnReason result) {
155 switch (result) {
156 case video_coding::FrameBuffer::kFrameFound: {
157 RTC_DCHECK(frame);
158 SaveFrameTimestamps(*frame);
159
160 MutexLock lock(&shut_down_mutex_);
161 if (shut_down_) {
162 return;
163 }
164
165 decode_queue_.PostTask([this, frame = std::move(frame)]() mutable {
166 RTC_DCHECK_RUN_ON(&decode_queue_);
167 DecodeResult decode_result = DecodeFrame(std::move(frame));
168 bookkeeping_queue_.PostTask([this, decode_result]() {
169 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
170 switch (decode_result) {
171 case kOk: {
172 keyframe_required_ = false;
173 break;
174 }
175 case kOkRequestKeyframe: {
176 callbacks_->OnNonDecodableState();
177 keyframe_required_ = false;
178 break;
179 }
180 case kDecodeFailure: {
181 callbacks_->OnNonDecodableState();
182 keyframe_required_ = true;
183 break;
184 }
185 }
186 StartNextDecode();
187 });
188 });
189 break;
190 }
191 case video_coding::FrameBuffer::kTimeout: {
192 callbacks_->OnNonDecodableState();
193 // The |frame_buffer_| requires the frame callback function to complete
194 // before NextFrame is called again. For this reason we call
195 // StartNextDecode in a later task to allow this task to complete first.
196 bookkeeping_queue_.PostTask([this]() {
197 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
198 StartNextDecode();
199 });
200 break;
201 }
202 case video_coding::FrameBuffer::kStopped: {
203 // We are shutting down, do nothing.
204 break;
205 }
206 }
207 }
208
DecodeFrame(std::unique_ptr<video_coding::EncodedFrame> frame)209 VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeFrame(
210 std::unique_ptr<video_coding::EncodedFrame> frame) {
211 RTC_DCHECK(frame);
212
213 VideoDecoder* decoder = GetDecoder(frame->PayloadType());
214 if (!decoder) {
215 return kDecodeFailure;
216 }
217
218 int32_t decode_result = decoder->Decode(frame->EncodedImage(), //
219 /*missing_frames=*/false, //
220 frame->RenderTimeMs());
221 switch (decode_result) {
222 case WEBRTC_VIDEO_CODEC_OK: {
223 return kOk;
224 }
225 case WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME: {
226 return kOkRequestKeyframe;
227 }
228 default:
229 return kDecodeFailure;
230 }
231 }
232
233 VideoStreamDecoderImpl::FrameTimestamps*
GetFrameTimestamps(int64_t timestamp)234 VideoStreamDecoderImpl::GetFrameTimestamps(int64_t timestamp) {
235 int start_time_index = next_frame_timestamps_index_;
236 for (int i = 0; i < kFrameTimestampsMemory; ++i) {
237 start_time_index = Subtract<kFrameTimestampsMemory>(start_time_index, 1);
238
239 if (frame_timestamps_[start_time_index].timestamp == timestamp)
240 return &frame_timestamps_[start_time_index];
241 }
242
243 return nullptr;
244 }
245
OnDecodedFrameCallback(VideoFrame & decoded_image,absl::optional<int32_t> decode_time_ms,absl::optional<uint8_t> qp)246 void VideoStreamDecoderImpl::OnDecodedFrameCallback(
247 VideoFrame& decoded_image,
248 absl::optional<int32_t> decode_time_ms,
249 absl::optional<uint8_t> qp) {
250 int64_t decode_stop_time_ms = rtc::TimeMillis();
251
252 bookkeeping_queue_.PostTask([this, decode_stop_time_ms, decoded_image,
253 decode_time_ms, qp]() {
254 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
255
256 FrameTimestamps* frame_timestamps =
257 GetFrameTimestamps(decoded_image.timestamp());
258 if (!frame_timestamps) {
259 RTC_LOG(LS_ERROR) << "No frame information found for frame with timestamp"
260 << decoded_image.timestamp();
261 return;
262 }
263
264 absl::optional<int> casted_qp;
265 if (qp)
266 casted_qp.emplace(*qp);
267
268 absl::optional<int> casted_decode_time_ms(decode_time_ms.value_or(
269 decode_stop_time_ms - frame_timestamps->decode_start_time_ms));
270
271 timing_.StopDecodeTimer(*casted_decode_time_ms, decode_stop_time_ms);
272
273 VideoFrame copy = decoded_image;
274 copy.set_timestamp_us(frame_timestamps->render_time_us);
275 callbacks_->OnDecodedFrame(copy, casted_decode_time_ms, casted_qp);
276 });
277 }
278
DecodeCallbacks(VideoStreamDecoderImpl * video_stream_decoder_impl)279 VideoStreamDecoderImpl::DecodeCallbacks::DecodeCallbacks(
280 VideoStreamDecoderImpl* video_stream_decoder_impl)
281 : video_stream_decoder_impl_(video_stream_decoder_impl) {}
282
Decoded(VideoFrame & decoded_image)283 int32_t VideoStreamDecoderImpl::DecodeCallbacks::Decoded(
284 VideoFrame& decoded_image) {
285 Decoded(decoded_image, absl::nullopt, absl::nullopt);
286 return WEBRTC_VIDEO_CODEC_OK;
287 }
288
Decoded(VideoFrame & decoded_image,int64_t decode_time_ms)289 int32_t VideoStreamDecoderImpl::DecodeCallbacks::Decoded(
290 VideoFrame& decoded_image,
291 int64_t decode_time_ms) {
292 Decoded(decoded_image, decode_time_ms, absl::nullopt);
293 return WEBRTC_VIDEO_CODEC_OK;
294 }
295
Decoded(VideoFrame & decoded_image,absl::optional<int32_t> decode_time_ms,absl::optional<uint8_t> qp)296 void VideoStreamDecoderImpl::DecodeCallbacks::Decoded(
297 VideoFrame& decoded_image,
298 absl::optional<int32_t> decode_time_ms,
299 absl::optional<uint8_t> qp) {
300 video_stream_decoder_impl_->OnDecodedFrameCallback(decoded_image,
301 decode_time_ms, qp);
302 }
303 } // namespace webrtc
304