1 /*
2 * Copyright 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "sdk/android/src/jni/video_decoder_wrapper.h"
12
13 #include "api/video/video_frame.h"
14 #include "modules/video_coding/include/video_codec_interface.h"
15 #include "modules/video_coding/utility/vp8_header_parser.h"
16 #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
17 #include "rtc_base/logging.h"
18 #include "rtc_base/numerics/safe_conversions.h"
19 #include "rtc_base/time_utils.h"
20 #include "sdk/android/generated_video_jni/VideoDecoderWrapper_jni.h"
21 #include "sdk/android/generated_video_jni/VideoDecoder_jni.h"
22 #include "sdk/android/native_api/jni/java_types.h"
23 #include "sdk/android/src/jni/encoded_image.h"
24 #include "sdk/android/src/jni/video_codec_status.h"
25 #include "sdk/android/src/jni/video_frame.h"
26
27 namespace webrtc {
28 namespace jni {
29
30 namespace {
31 // RTP timestamps are 90 kHz.
32 const int64_t kNumRtpTicksPerMillisec = 90000 / rtc::kNumMillisecsPerSec;
33
34 template <typename Dst, typename Src>
cast_optional(const absl::optional<Src> & value)35 inline absl::optional<Dst> cast_optional(const absl::optional<Src>& value) {
36 return value ? absl::optional<Dst>(rtc::dchecked_cast<Dst, Src>(*value))
37 : absl::nullopt;
38 }
39 } // namespace
40
VideoDecoderWrapper(JNIEnv * jni,const JavaRef<jobject> & decoder)41 VideoDecoderWrapper::VideoDecoderWrapper(JNIEnv* jni,
42 const JavaRef<jobject>& decoder)
43 : decoder_(jni, decoder),
44 implementation_name_(JavaToStdString(
45 jni,
46 Java_VideoDecoder_getImplementationName(jni, decoder))),
47 initialized_(false),
48 qp_parsing_enabled_(true) // QP parsing starts enabled and we disable it
49 // if the decoder provides frames.
50
51 {
52 decoder_thread_checker_.Detach();
53 }
54
55 VideoDecoderWrapper::~VideoDecoderWrapper() = default;
56
InitDecode(const VideoCodec * codec_settings,int32_t number_of_cores)57 int32_t VideoDecoderWrapper::InitDecode(const VideoCodec* codec_settings,
58 int32_t number_of_cores) {
59 RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
60 JNIEnv* jni = AttachCurrentThreadIfNeeded();
61 codec_settings_ = *codec_settings;
62 number_of_cores_ = number_of_cores;
63 return InitDecodeInternal(jni);
64 }
65
InitDecodeInternal(JNIEnv * jni)66 int32_t VideoDecoderWrapper::InitDecodeInternal(JNIEnv* jni) {
67 ScopedJavaLocalRef<jobject> settings = Java_Settings_Constructor(
68 jni, number_of_cores_, codec_settings_.width, codec_settings_.height);
69
70 ScopedJavaLocalRef<jobject> callback =
71 Java_VideoDecoderWrapper_createDecoderCallback(jni,
72 jlongFromPointer(this));
73
74 int32_t status = JavaToNativeVideoCodecStatus(
75 jni, Java_VideoDecoder_initDecode(jni, decoder_, settings, callback));
76 RTC_LOG(LS_INFO) << "initDecode: " << status;
77 if (status == WEBRTC_VIDEO_CODEC_OK) {
78 initialized_ = true;
79 }
80
81 // The decoder was reinitialized so re-enable the QP parsing in case it stops
82 // providing QP values.
83 qp_parsing_enabled_ = true;
84
85 return status;
86 }
87
Decode(const EncodedImage & image_param,bool missing_frames,int64_t render_time_ms)88 int32_t VideoDecoderWrapper::Decode(
89 const EncodedImage& image_param,
90 bool missing_frames,
91 int64_t render_time_ms) {
92 RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
93 if (!initialized_) {
94 // Most likely initializing the codec failed.
95 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
96 }
97
98 // Make a mutable copy so we can modify the timestamp.
99 EncodedImage input_image(image_param);
100 // We use RTP timestamp for capture time because capture_time_ms_ is always 0.
101 input_image.capture_time_ms_ =
102 input_image.Timestamp() / kNumRtpTicksPerMillisec;
103
104 FrameExtraInfo frame_extra_info;
105 frame_extra_info.timestamp_ns =
106 input_image.capture_time_ms_ * rtc::kNumNanosecsPerMillisec;
107 frame_extra_info.timestamp_rtp = input_image.Timestamp();
108 frame_extra_info.timestamp_ntp = input_image.ntp_time_ms_;
109 frame_extra_info.qp =
110 qp_parsing_enabled_ ? ParseQP(input_image) : absl::nullopt;
111 {
112 MutexLock lock(&frame_extra_infos_lock_);
113 frame_extra_infos_.push_back(frame_extra_info);
114 }
115
116 JNIEnv* env = AttachCurrentThreadIfNeeded();
117 ScopedJavaLocalRef<jobject> jinput_image =
118 NativeToJavaEncodedImage(env, input_image);
119 ScopedJavaLocalRef<jobject> decode_info;
120 ScopedJavaLocalRef<jobject> ret =
121 Java_VideoDecoder_decode(env, decoder_, jinput_image, decode_info);
122 return HandleReturnCode(env, ret, "decode");
123 }
124
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)125 int32_t VideoDecoderWrapper::RegisterDecodeCompleteCallback(
126 DecodedImageCallback* callback) {
127 RTC_DCHECK_RUNS_SERIALIZED(&callback_race_checker_);
128 callback_ = callback;
129 return WEBRTC_VIDEO_CODEC_OK;
130 }
131
Release()132 int32_t VideoDecoderWrapper::Release() {
133 JNIEnv* jni = AttachCurrentThreadIfNeeded();
134 int32_t status = JavaToNativeVideoCodecStatus(
135 jni, Java_VideoDecoder_release(jni, decoder_));
136 RTC_LOG(LS_INFO) << "release: " << status;
137 {
138 MutexLock lock(&frame_extra_infos_lock_);
139 frame_extra_infos_.clear();
140 }
141 initialized_ = false;
142 // It is allowed to reinitialize the codec on a different thread.
143 decoder_thread_checker_.Detach();
144 return status;
145 }
146
PrefersLateDecoding() const147 bool VideoDecoderWrapper::PrefersLateDecoding() const {
148 JNIEnv* jni = AttachCurrentThreadIfNeeded();
149 return Java_VideoDecoder_getPrefersLateDecoding(jni, decoder_);
150 }
151
ImplementationName() const152 const char* VideoDecoderWrapper::ImplementationName() const {
153 return implementation_name_.c_str();
154 }
155
OnDecodedFrame(JNIEnv * env,const JavaRef<jobject> & j_frame,const JavaRef<jobject> & j_decode_time_ms,const JavaRef<jobject> & j_qp)156 void VideoDecoderWrapper::OnDecodedFrame(
157 JNIEnv* env,
158 const JavaRef<jobject>& j_frame,
159 const JavaRef<jobject>& j_decode_time_ms,
160 const JavaRef<jobject>& j_qp) {
161 RTC_DCHECK_RUNS_SERIALIZED(&callback_race_checker_);
162 const int64_t timestamp_ns = GetJavaVideoFrameTimestampNs(env, j_frame);
163
164 FrameExtraInfo frame_extra_info;
165 {
166 MutexLock lock(&frame_extra_infos_lock_);
167
168 do {
169 if (frame_extra_infos_.empty()) {
170 RTC_LOG(LS_WARNING)
171 << "Java decoder produced an unexpected frame: " << timestamp_ns;
172 return;
173 }
174
175 frame_extra_info = frame_extra_infos_.front();
176 frame_extra_infos_.pop_front();
177 // If the decoder might drop frames so iterate through the queue until we
178 // find a matching timestamp.
179 } while (frame_extra_info.timestamp_ns != timestamp_ns);
180 }
181
182 VideoFrame frame =
183 JavaToNativeFrame(env, j_frame, frame_extra_info.timestamp_rtp);
184 frame.set_ntp_time_ms(frame_extra_info.timestamp_ntp);
185
186 absl::optional<int32_t> decoding_time_ms =
187 JavaToNativeOptionalInt(env, j_decode_time_ms);
188
189 absl::optional<uint8_t> decoder_qp =
190 cast_optional<uint8_t, int32_t>(JavaToNativeOptionalInt(env, j_qp));
191 // If the decoder provides QP values itself, no need to parse the bitstream.
192 // Enable QP parsing if decoder does not provide QP values itself.
193 qp_parsing_enabled_ = !decoder_qp.has_value();
194 callback_->Decoded(frame, decoding_time_ms,
195 decoder_qp ? decoder_qp : frame_extra_info.qp);
196 }
197
198 VideoDecoderWrapper::FrameExtraInfo::FrameExtraInfo() = default;
199 VideoDecoderWrapper::FrameExtraInfo::FrameExtraInfo(const FrameExtraInfo&) =
200 default;
201 VideoDecoderWrapper::FrameExtraInfo::~FrameExtraInfo() = default;
202
HandleReturnCode(JNIEnv * jni,const JavaRef<jobject> & j_value,const char * method_name)203 int32_t VideoDecoderWrapper::HandleReturnCode(JNIEnv* jni,
204 const JavaRef<jobject>& j_value,
205 const char* method_name) {
206 int32_t value = JavaToNativeVideoCodecStatus(jni, j_value);
207 if (value >= 0) { // OK or NO_OUTPUT
208 return value;
209 }
210
211 RTC_LOG(LS_WARNING) << method_name << ": " << value;
212 if (value == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE ||
213 value == WEBRTC_VIDEO_CODEC_UNINITIALIZED) { // Critical error.
214 RTC_LOG(LS_WARNING) << "Java decoder requested software fallback.";
215 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
216 }
217
218 // Try resetting the codec.
219 if (Release() == WEBRTC_VIDEO_CODEC_OK &&
220 InitDecodeInternal(jni) == WEBRTC_VIDEO_CODEC_OK) {
221 RTC_LOG(LS_WARNING) << "Reset Java decoder.";
222 return WEBRTC_VIDEO_CODEC_ERROR;
223 }
224
225 RTC_LOG(LS_WARNING) << "Unable to reset Java decoder.";
226 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
227 }
228
ParseQP(const EncodedImage & input_image)229 absl::optional<uint8_t> VideoDecoderWrapper::ParseQP(
230 const EncodedImage& input_image) {
231 if (input_image.qp_ != -1) {
232 return input_image.qp_;
233 }
234
235 absl::optional<uint8_t> qp;
236 switch (codec_settings_.codecType) {
237 case kVideoCodecVP8: {
238 int qp_int;
239 if (vp8::GetQp(input_image.data(), input_image.size(), &qp_int)) {
240 qp = qp_int;
241 }
242 break;
243 }
244 case kVideoCodecVP9: {
245 int qp_int;
246 if (vp9::GetQp(input_image.data(), input_image.size(), &qp_int)) {
247 qp = qp_int;
248 }
249 break;
250 }
251 case kVideoCodecH264: {
252 h264_bitstream_parser_.ParseBitstream(input_image.data(),
253 input_image.size());
254 int qp_int;
255 if (h264_bitstream_parser_.GetLastSliceQp(&qp_int)) {
256 qp = qp_int;
257 }
258 break;
259 }
260 default:
261 break; // Default is to not provide QP.
262 }
263 return qp;
264 }
265
JavaToNativeVideoDecoder(JNIEnv * jni,const JavaRef<jobject> & j_decoder)266 std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
267 JNIEnv* jni,
268 const JavaRef<jobject>& j_decoder) {
269 const jlong native_decoder =
270 Java_VideoDecoder_createNativeVideoDecoder(jni, j_decoder);
271 VideoDecoder* decoder;
272 if (native_decoder == 0) {
273 decoder = new VideoDecoderWrapper(jni, j_decoder);
274 } else {
275 decoder = reinterpret_cast<VideoDecoder*>(native_decoder);
276 }
277 return std::unique_ptr<VideoDecoder>(decoder);
278 }
279
280 } // namespace jni
281 } // namespace webrtc
282