1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/generic_decoder.h"
12
13 #include <stddef.h>
14
15 #include <algorithm>
16 #include <cmath>
17 #include <iterator>
18 #include <utility>
19
20 #include "absl/algorithm/container.h"
21 #include "absl/types/optional.h"
22 #include "api/video/video_timing.h"
23 #include "api/video_codecs/video_decoder.h"
24 #include "modules/include/module_common_types_public.h"
25 #include "modules/video_coding/include/video_error_codes.h"
26 #include "rtc_base/checks.h"
27 #include "rtc_base/logging.h"
28 #include "rtc_base/trace_event.h"
29 #include "system_wrappers/include/clock.h"
30
31 namespace webrtc {
32
33 namespace {
34
35 constexpr size_t kDecoderFrameMemoryLength = 10;
36
37 }
38
VCMDecodedFrameCallback(VCMTiming * timing,Clock * clock,const FieldTrialsView & field_trials)39 VCMDecodedFrameCallback::VCMDecodedFrameCallback(
40 VCMTiming* timing,
41 Clock* clock,
42 const FieldTrialsView& field_trials)
43 : _clock(clock), _timing(timing) {
44 ntp_offset_ =
45 _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds();
46 }
47
~VCMDecodedFrameCallback()48 VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {}
49
SetUserReceiveCallback(VCMReceiveCallback * receiveCallback)50 void VCMDecodedFrameCallback::SetUserReceiveCallback(
51 VCMReceiveCallback* receiveCallback) {
52 RTC_DCHECK(construction_thread_.IsCurrent());
53 RTC_DCHECK((!_receiveCallback && receiveCallback) ||
54 (_receiveCallback && !receiveCallback));
55 _receiveCallback = receiveCallback;
56 }
57
UserReceiveCallback()58 VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() {
59 // Called on the decode thread via VCMCodecDataBase::GetDecoder.
60 // The callback must always have been set before this happens.
61 RTC_DCHECK(_receiveCallback);
62 return _receiveCallback;
63 }
64
Decoded(VideoFrame & decodedImage)65 int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) {
66 // This function may be called on the decode TaskQueue, but may also be called
67 // on an OS provided queue such as on iOS (see e.g. b/153465112).
68 return Decoded(decodedImage, -1);
69 }
70
Decoded(VideoFrame & decodedImage,int64_t decode_time_ms)71 int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
72 int64_t decode_time_ms) {
73 Decoded(decodedImage,
74 decode_time_ms >= 0 ? absl::optional<int32_t>(decode_time_ms)
75 : absl::nullopt,
76 absl::nullopt);
77 return WEBRTC_VIDEO_CODEC_OK;
78 }
79
80 std::pair<absl::optional<FrameInfo>, size_t>
FindFrameInfo(uint32_t rtp_timestamp)81 VCMDecodedFrameCallback::FindFrameInfo(uint32_t rtp_timestamp) {
82 absl::optional<FrameInfo> frame_info;
83
84 auto it = absl::c_find_if(frame_infos_, [rtp_timestamp](const auto& entry) {
85 return entry.rtp_timestamp == rtp_timestamp ||
86 IsNewerTimestamp(entry.rtp_timestamp, rtp_timestamp);
87 });
88 size_t dropped_frames = std::distance(frame_infos_.begin(), it);
89
90 if (it != frame_infos_.end() && it->rtp_timestamp == rtp_timestamp) {
91 // Frame was found and should also be removed from the queue.
92 frame_info = std::move(*it);
93 ++it;
94 }
95
96 frame_infos_.erase(frame_infos_.begin(), it);
97 return std::make_pair(std::move(frame_info), dropped_frames);
98 }
99
Decoded(VideoFrame & decodedImage,absl::optional<int32_t> decode_time_ms,absl::optional<uint8_t> qp)100 void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
101 absl::optional<int32_t> decode_time_ms,
102 absl::optional<uint8_t> qp) {
103 RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point";
104 TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded",
105 "timestamp", decodedImage.timestamp());
106 // TODO(holmer): We should improve this so that we can handle multiple
107 // callbacks from one call to Decode().
108 absl::optional<FrameInfo> frame_info;
109 int timestamp_map_size = 0;
110 int dropped_frames = 0;
111 {
112 MutexLock lock(&lock_);
113 std::tie(frame_info, dropped_frames) =
114 FindFrameInfo(decodedImage.timestamp());
115 timestamp_map_size = frame_infos_.size();
116 }
117 if (dropped_frames > 0) {
118 _receiveCallback->OnDroppedFrames(dropped_frames);
119 }
120
121 if (!frame_info) {
122 RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping "
123 "frame with timestamp "
124 << decodedImage.timestamp();
125 return;
126 }
127
128 decodedImage.set_ntp_time_ms(frame_info->ntp_time_ms);
129 decodedImage.set_packet_infos(frame_info->packet_infos);
130 decodedImage.set_rotation(frame_info->rotation);
131 VideoFrame::RenderParameters render_parameters = _timing->RenderParameters();
132 if (render_parameters.max_composition_delay_in_frames) {
133 // Subtract frames that are in flight.
134 render_parameters.max_composition_delay_in_frames =
135 std::max(0, *render_parameters.max_composition_delay_in_frames -
136 timestamp_map_size);
137 }
138 decodedImage.set_render_parameters(render_parameters);
139
140 RTC_DCHECK(frame_info->decode_start);
141 const Timestamp now = _clock->CurrentTime();
142 const TimeDelta decode_time = decode_time_ms
143 ? TimeDelta::Millis(*decode_time_ms)
144 : now - *frame_info->decode_start;
145 _timing->StopDecodeTimer(decode_time, now);
146 decodedImage.set_processing_time(
147 {*frame_info->decode_start, *frame_info->decode_start + decode_time});
148
149 // Report timing information.
150 TimingFrameInfo timing_frame_info;
151 if (frame_info->timing.flags != VideoSendTiming::kInvalid) {
152 int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_;
153 // Convert remote timestamps to local time from ntp timestamps.
154 frame_info->timing.encode_start_ms -= ntp_offset_;
155 frame_info->timing.encode_finish_ms -= ntp_offset_;
156 frame_info->timing.packetization_finish_ms -= ntp_offset_;
157 frame_info->timing.pacer_exit_ms -= ntp_offset_;
158 frame_info->timing.network_timestamp_ms -= ntp_offset_;
159 frame_info->timing.network2_timestamp_ms -= ntp_offset_;
160
161 int64_t sender_delta_ms = 0;
162 if (decodedImage.ntp_time_ms() < 0) {
163 // Sender clock is not estimated yet. Make sure that sender times are all
164 // negative to indicate that. Yet they still should be relatively correct.
165 sender_delta_ms =
166 std::max({capture_time_ms, frame_info->timing.encode_start_ms,
167 frame_info->timing.encode_finish_ms,
168 frame_info->timing.packetization_finish_ms,
169 frame_info->timing.pacer_exit_ms,
170 frame_info->timing.network_timestamp_ms,
171 frame_info->timing.network2_timestamp_ms}) +
172 1;
173 }
174
175 timing_frame_info.capture_time_ms = capture_time_ms - sender_delta_ms;
176 timing_frame_info.encode_start_ms =
177 frame_info->timing.encode_start_ms - sender_delta_ms;
178 timing_frame_info.encode_finish_ms =
179 frame_info->timing.encode_finish_ms - sender_delta_ms;
180 timing_frame_info.packetization_finish_ms =
181 frame_info->timing.packetization_finish_ms - sender_delta_ms;
182 timing_frame_info.pacer_exit_ms =
183 frame_info->timing.pacer_exit_ms - sender_delta_ms;
184 timing_frame_info.network_timestamp_ms =
185 frame_info->timing.network_timestamp_ms - sender_delta_ms;
186 timing_frame_info.network2_timestamp_ms =
187 frame_info->timing.network2_timestamp_ms - sender_delta_ms;
188 }
189
190 timing_frame_info.flags = frame_info->timing.flags;
191 timing_frame_info.decode_start_ms = frame_info->decode_start->ms();
192 timing_frame_info.decode_finish_ms = now.ms();
193 timing_frame_info.render_time_ms =
194 frame_info->render_time ? frame_info->render_time->ms() : -1;
195 timing_frame_info.rtp_timestamp = decodedImage.timestamp();
196 timing_frame_info.receive_start_ms = frame_info->timing.receive_start_ms;
197 timing_frame_info.receive_finish_ms = frame_info->timing.receive_finish_ms;
198 _timing->SetTimingFrameInfo(timing_frame_info);
199
200 decodedImage.set_timestamp_us(
201 frame_info->render_time ? frame_info->render_time->us() : -1);
202 _receiveCallback->FrameToRender(decodedImage, qp, decode_time,
203 frame_info->content_type);
204 }
205
OnDecoderInfoChanged(const VideoDecoder::DecoderInfo & decoder_info)206 void VCMDecodedFrameCallback::OnDecoderInfoChanged(
207 const VideoDecoder::DecoderInfo& decoder_info) {
208 _receiveCallback->OnDecoderInfoChanged(decoder_info);
209 }
210
Map(FrameInfo frameInfo)211 void VCMDecodedFrameCallback::Map(FrameInfo frameInfo) {
212 int dropped_frames = 0;
213 {
214 MutexLock lock(&lock_);
215 int initial_size = frame_infos_.size();
216 if (initial_size == kDecoderFrameMemoryLength) {
217 frame_infos_.pop_front();
218 dropped_frames = 1;
219 }
220 frame_infos_.push_back(std::move(frameInfo));
221 // If no frame is dropped, the new size should be `initial_size` + 1
222 }
223 if (dropped_frames > 0) {
224 _receiveCallback->OnDroppedFrames(dropped_frames);
225 }
226 }
227
ClearTimestampMap()228 void VCMDecodedFrameCallback::ClearTimestampMap() {
229 int dropped_frames = 0;
230 {
231 MutexLock lock(&lock_);
232 dropped_frames = frame_infos_.size();
233 frame_infos_.clear();
234 }
235 if (dropped_frames > 0) {
236 _receiveCallback->OnDroppedFrames(dropped_frames);
237 }
238 }
239
VCMGenericDecoder(VideoDecoder * decoder)240 VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder)
241 : _callback(NULL),
242 decoder_(decoder),
243 _last_keyframe_content_type(VideoContentType::UNSPECIFIED) {
244 RTC_DCHECK(decoder_);
245 }
246
~VCMGenericDecoder()247 VCMGenericDecoder::~VCMGenericDecoder() {
248 decoder_->Release();
249 }
250
Configure(const VideoDecoder::Settings & settings)251 bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) {
252 TRACE_EVENT0("webrtc", "VCMGenericDecoder::Configure");
253
254 bool ok = decoder_->Configure(settings);
255 decoder_info_ = decoder_->GetDecoderInfo();
256 RTC_LOG(LS_INFO) << "Decoder implementation: " << decoder_info_.ToString();
257 if (_callback) {
258 _callback->OnDecoderInfoChanged(decoder_info_);
259 }
260 return ok;
261 }
262
Decode(const VCMEncodedFrame & frame,Timestamp now)263 int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
264 TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
265 frame.Timestamp());
266 FrameInfo frame_info;
267 frame_info.rtp_timestamp = frame.Timestamp();
268 frame_info.decode_start = now;
269 frame_info.render_time =
270 frame.RenderTimeMs() >= 0
271 ? absl::make_optional(Timestamp::Millis(frame.RenderTimeMs()))
272 : absl::nullopt;
273 frame_info.rotation = frame.rotation();
274 frame_info.timing = frame.video_timing();
275 frame_info.ntp_time_ms = frame.EncodedImage().ntp_time_ms_;
276 frame_info.packet_infos = frame.PacketInfos();
277
278 // Set correctly only for key frames. Thus, use latest key frame
279 // content type. If the corresponding key frame was lost, decode will fail
280 // and content type will be ignored.
281 if (frame.FrameType() == VideoFrameType::kVideoFrameKey) {
282 frame_info.content_type = frame.contentType();
283 _last_keyframe_content_type = frame.contentType();
284 } else {
285 frame_info.content_type = _last_keyframe_content_type;
286 }
287 _callback->Map(std::move(frame_info));
288
289 int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
290 frame.RenderTimeMs());
291 VideoDecoder::DecoderInfo decoder_info = decoder_->GetDecoderInfo();
292 if (decoder_info != decoder_info_) {
293 RTC_LOG(LS_INFO) << "Changed decoder implementation to: "
294 << decoder_info.ToString();
295 decoder_info_ = decoder_info;
296 if (decoder_info.implementation_name.empty()) {
297 decoder_info.implementation_name = "unknown";
298 }
299 _callback->OnDecoderInfoChanged(std::move(decoder_info));
300 }
301 if (ret < WEBRTC_VIDEO_CODEC_OK) {
302 RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp "
303 << frame.Timestamp() << ", error code: " << ret;
304 _callback->ClearTimestampMap();
305 } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT) {
306 // No output.
307 _callback->ClearTimestampMap();
308 }
309 return ret;
310 }
311
RegisterDecodeCompleteCallback(VCMDecodedFrameCallback * callback)312 int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback(
313 VCMDecodedFrameCallback* callback) {
314 _callback = callback;
315 int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback);
316 if (callback && !decoder_info_.implementation_name.empty()) {
317 callback->OnDecoderInfoChanged(decoder_info_);
318 }
319 return ret;
320 }
321
322 } // namespace webrtc
323