1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/video_stream_decoder_impl.h"
12
13 #include <memory>
14
15 #include "rtc_base/logging.h"
16 #include "rtc_base/numerics/mod_ops.h"
17 #include "rtc_base/time_utils.h"
18
19 namespace webrtc {
20
VideoStreamDecoderImpl(VideoStreamDecoderInterface::Callbacks * callbacks,VideoDecoderFactory * decoder_factory,TaskQueueFactory * task_queue_factory,std::map<int,std::pair<SdpVideoFormat,int>> decoder_settings,const FieldTrialsView * field_trials)21 VideoStreamDecoderImpl::VideoStreamDecoderImpl(
22 VideoStreamDecoderInterface::Callbacks* callbacks,
23 VideoDecoderFactory* decoder_factory,
24 TaskQueueFactory* task_queue_factory,
25 std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings,
26 const FieldTrialsView* field_trials)
27 : field_trials_(field_trials),
28 timing_(Clock::GetRealTimeClock(), *field_trials_),
29 decode_callbacks_(this),
30 next_frame_info_index_(0),
31 callbacks_(callbacks),
32 keyframe_required_(true),
33 decoder_factory_(decoder_factory),
34 decoder_settings_(std::move(decoder_settings)),
35 shut_down_(false),
36 frame_buffer_(Clock::GetRealTimeClock(), &timing_, *field_trials_),
37 bookkeeping_queue_(task_queue_factory->CreateTaskQueue(
38 "video_stream_decoder_bookkeeping_queue",
39 TaskQueueFactory::Priority::NORMAL)),
40 decode_queue_(task_queue_factory->CreateTaskQueue(
41 "video_stream_decoder_decode_queue",
42 TaskQueueFactory::Priority::NORMAL)) {
43 bookkeeping_queue_.PostTask([this]() {
44 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
45 StartNextDecode();
46 });
47 }
48
~VideoStreamDecoderImpl()49 VideoStreamDecoderImpl::~VideoStreamDecoderImpl() {
50 MutexLock lock(&shut_down_mutex_);
51 shut_down_ = true;
52 }
53
OnFrame(std::unique_ptr<EncodedFrame> frame)54 void VideoStreamDecoderImpl::OnFrame(std::unique_ptr<EncodedFrame> frame) {
55 if (!bookkeeping_queue_.IsCurrent()) {
56 bookkeeping_queue_.PostTask([this, frame = std::move(frame)]() mutable {
57 OnFrame(std::move(frame));
58 return true;
59 });
60
61 return;
62 }
63
64 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
65
66 int64_t continuous_frame_id = frame_buffer_.InsertFrame(std::move(frame));
67 if (last_continuous_frame_id_ < continuous_frame_id) {
68 last_continuous_frame_id_ = continuous_frame_id;
69 callbacks_->OnContinuousUntil(last_continuous_frame_id_);
70 }
71 }
72
SetMinPlayoutDelay(TimeDelta min_delay)73 void VideoStreamDecoderImpl::SetMinPlayoutDelay(TimeDelta min_delay) {
74 timing_.set_min_playout_delay(min_delay);
75 }
76
SetMaxPlayoutDelay(TimeDelta max_delay)77 void VideoStreamDecoderImpl::SetMaxPlayoutDelay(TimeDelta max_delay) {
78 timing_.set_max_playout_delay(max_delay);
79 }
80
GetDecoder(int payload_type)81 VideoDecoder* VideoStreamDecoderImpl::GetDecoder(int payload_type) {
82 if (current_payload_type_ == payload_type) {
83 RTC_DCHECK(decoder_);
84 return decoder_.get();
85 }
86
87 current_payload_type_.reset();
88 decoder_.reset();
89
90 auto decoder_settings_it = decoder_settings_.find(payload_type);
91 if (decoder_settings_it == decoder_settings_.end()) {
92 RTC_LOG(LS_WARNING) << "Payload type " << payload_type
93 << " not registered.";
94 return nullptr;
95 }
96
97 const SdpVideoFormat& video_format = decoder_settings_it->second.first;
98 std::unique_ptr<VideoDecoder> decoder =
99 decoder_factory_->CreateVideoDecoder(video_format);
100 if (!decoder) {
101 RTC_LOG(LS_WARNING) << "Failed to create decoder for payload type "
102 << payload_type << ".";
103 return nullptr;
104 }
105
106 VideoDecoder::Settings settings;
107 settings.set_number_of_cores(decoder_settings_it->second.second);
108 if (!decoder->Configure(settings)) {
109 RTC_LOG(LS_WARNING) << "Failed to initialize decoder for payload type "
110 << payload_type << ".";
111 return nullptr;
112 }
113
114 int32_t register_result =
115 decoder->RegisterDecodeCompleteCallback(&decode_callbacks_);
116 if (register_result != WEBRTC_VIDEO_CODEC_OK) {
117 RTC_LOG(LS_WARNING) << "Failed to register decode callback.";
118 return nullptr;
119 }
120
121 current_payload_type_.emplace(payload_type);
122 decoder_ = std::move(decoder);
123 return decoder_.get();
124 }
125
SaveFrameInfo(const EncodedFrame & frame)126 void VideoStreamDecoderImpl::SaveFrameInfo(const EncodedFrame& frame) {
127 FrameInfo* frame_info = &frame_info_[next_frame_info_index_];
128 frame_info->timestamp = frame.Timestamp();
129 frame_info->decode_start_time_ms = rtc::TimeMillis();
130 frame_info->render_time_us = frame.RenderTimeMs() * 1000;
131 frame_info->content_type = frame.EncodedImage().content_type_;
132
133 next_frame_info_index_ = Add<kFrameInfoMemory>(next_frame_info_index_, 1);
134 }
135
StartNextDecode()136 void VideoStreamDecoderImpl::StartNextDecode() {
137 int64_t max_wait_time = keyframe_required_ ? 200 : 3000;
138
139 frame_buffer_.NextFrame(max_wait_time, keyframe_required_,
140 bookkeeping_queue_.Get(),
141 [this](std::unique_ptr<EncodedFrame> frame) {
142 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
143 OnNextFrameCallback(std::move(frame));
144 });
145 }
146
OnNextFrameCallback(std::unique_ptr<EncodedFrame> frame)147 void VideoStreamDecoderImpl::OnNextFrameCallback(
148 std::unique_ptr<EncodedFrame> frame) {
149 if (frame) {
150 RTC_DCHECK(frame);
151 SaveFrameInfo(*frame);
152
153 MutexLock lock(&shut_down_mutex_);
154 if (shut_down_) {
155 return;
156 }
157
158 decode_queue_.PostTask([this, frame = std::move(frame)]() mutable {
159 RTC_DCHECK_RUN_ON(&decode_queue_);
160 DecodeResult decode_result = DecodeFrame(std::move(frame));
161 bookkeeping_queue_.PostTask([this, decode_result]() {
162 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
163 switch (decode_result) {
164 case kOk: {
165 keyframe_required_ = false;
166 break;
167 }
168 case kOkRequestKeyframe: {
169 callbacks_->OnNonDecodableState();
170 keyframe_required_ = false;
171 break;
172 }
173 case kDecodeFailure: {
174 callbacks_->OnNonDecodableState();
175 keyframe_required_ = true;
176 break;
177 }
178 }
179 StartNextDecode();
180 });
181 });
182 } else {
183 callbacks_->OnNonDecodableState();
184 // The `frame_buffer_` requires the frame callback function to complete
185 // before NextFrame is called again. For this reason we call
186 // StartNextDecode in a later task to allow this task to complete first.
187 bookkeeping_queue_.PostTask([this]() {
188 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
189 StartNextDecode();
190 });
191 }
192 }
193
DecodeFrame(std::unique_ptr<EncodedFrame> frame)194 VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeFrame(
195 std::unique_ptr<EncodedFrame> frame) {
196 RTC_DCHECK(frame);
197
198 VideoDecoder* decoder = GetDecoder(frame->PayloadType());
199 if (!decoder) {
200 return kDecodeFailure;
201 }
202
203 int32_t decode_result = decoder->Decode(frame->EncodedImage(), //
204 /*missing_frames=*/false, //
205 frame->RenderTimeMs());
206 switch (decode_result) {
207 case WEBRTC_VIDEO_CODEC_OK: {
208 return kOk;
209 }
210 case WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME: {
211 return kOkRequestKeyframe;
212 }
213 default:
214 return kDecodeFailure;
215 }
216 }
217
GetFrameInfo(int64_t timestamp)218 VideoStreamDecoderImpl::FrameInfo* VideoStreamDecoderImpl::GetFrameInfo(
219 int64_t timestamp) {
220 int start_time_index = next_frame_info_index_;
221 for (int i = 0; i < kFrameInfoMemory; ++i) {
222 start_time_index = Subtract<kFrameInfoMemory>(start_time_index, 1);
223
224 if (frame_info_[start_time_index].timestamp == timestamp)
225 return &frame_info_[start_time_index];
226 }
227
228 return nullptr;
229 }
230
OnDecodedFrameCallback(VideoFrame & decoded_image,absl::optional<int32_t> decode_time_ms,absl::optional<uint8_t> qp)231 void VideoStreamDecoderImpl::OnDecodedFrameCallback(
232 VideoFrame& decoded_image,
233 absl::optional<int32_t> decode_time_ms,
234 absl::optional<uint8_t> qp) {
235 int64_t decode_stop_time_ms = rtc::TimeMillis();
236
237 bookkeeping_queue_.PostTask([this, decode_stop_time_ms, decoded_image,
238 decode_time_ms, qp]() mutable {
239 RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
240
241 FrameInfo* frame_info = GetFrameInfo(decoded_image.timestamp());
242 if (!frame_info) {
243 RTC_LOG(LS_ERROR) << "No frame information found for frame with timestamp"
244 << decoded_image.timestamp();
245 return;
246 }
247
248 Callbacks::FrameInfo callback_info;
249 callback_info.content_type = frame_info->content_type;
250
251 if (qp)
252 callback_info.qp.emplace(*qp);
253
254 if (!decode_time_ms) {
255 decode_time_ms = decode_stop_time_ms - frame_info->decode_start_time_ms;
256 }
257 decoded_image.set_processing_time(
258 {Timestamp::Millis(frame_info->decode_start_time_ms),
259 Timestamp::Millis(frame_info->decode_start_time_ms +
260 *decode_time_ms)});
261 decoded_image.set_timestamp_us(frame_info->render_time_us);
262 timing_.StopDecodeTimer(TimeDelta::Millis(*decode_time_ms),
263 Timestamp::Millis(decode_stop_time_ms));
264
265 callbacks_->OnDecodedFrame(decoded_image, callback_info);
266 });
267 }
268
DecodeCallbacks(VideoStreamDecoderImpl * video_stream_decoder_impl)269 VideoStreamDecoderImpl::DecodeCallbacks::DecodeCallbacks(
270 VideoStreamDecoderImpl* video_stream_decoder_impl)
271 : video_stream_decoder_impl_(video_stream_decoder_impl) {}
272
Decoded(VideoFrame & decoded_image)273 int32_t VideoStreamDecoderImpl::DecodeCallbacks::Decoded(
274 VideoFrame& decoded_image) {
275 Decoded(decoded_image, absl::nullopt, absl::nullopt);
276 return WEBRTC_VIDEO_CODEC_OK;
277 }
278
Decoded(VideoFrame & decoded_image,int64_t decode_time_ms)279 int32_t VideoStreamDecoderImpl::DecodeCallbacks::Decoded(
280 VideoFrame& decoded_image,
281 int64_t decode_time_ms) {
282 Decoded(decoded_image, decode_time_ms, absl::nullopt);
283 return WEBRTC_VIDEO_CODEC_OK;
284 }
285
Decoded(VideoFrame & decoded_image,absl::optional<int32_t> decode_time_ms,absl::optional<uint8_t> qp)286 void VideoStreamDecoderImpl::DecodeCallbacks::Decoded(
287 VideoFrame& decoded_image,
288 absl::optional<int32_t> decode_time_ms,
289 absl::optional<uint8_t> qp) {
290 video_stream_decoder_impl_->OnDecodedFrameCallback(decoded_image,
291 decode_time_ms, qp);
292 }
293 } // namespace webrtc
294