1 /*
2 * Copyright 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "sdk/android/src/jni/video_decoder_wrapper.h"
12
13 #include "api/video/render_resolution.h"
14 #include "api/video/video_frame.h"
15 #include "api/video_codecs/video_decoder.h"
16 #include "modules/video_coding/include/video_codec_interface.h"
17 #include "modules/video_coding/utility/vp8_header_parser.h"
18 #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
19 #include "rtc_base/logging.h"
20 #include "rtc_base/numerics/safe_conversions.h"
21 #include "rtc_base/time_utils.h"
22 #include "sdk/android/generated_video_jni/VideoDecoderWrapper_jni.h"
23 #include "sdk/android/generated_video_jni/VideoDecoder_jni.h"
24 #include "sdk/android/native_api/jni/java_types.h"
25 #include "sdk/android/src/jni/encoded_image.h"
26 #include "sdk/android/src/jni/video_codec_status.h"
27 #include "sdk/android/src/jni/video_frame.h"
28
29 namespace webrtc {
30 namespace jni {
31
32 namespace {
33 // RTP timestamps are 90 kHz.
34 const int64_t kNumRtpTicksPerMillisec = 90000 / rtc::kNumMillisecsPerSec;
35
36 template <typename Dst, typename Src>
cast_optional(const absl::optional<Src> & value)37 inline absl::optional<Dst> cast_optional(const absl::optional<Src>& value) {
38 return value ? absl::optional<Dst>(rtc::dchecked_cast<Dst, Src>(*value))
39 : absl::nullopt;
40 }
41 } // namespace
42
VideoDecoderWrapper(JNIEnv * jni,const JavaRef<jobject> & decoder)43 VideoDecoderWrapper::VideoDecoderWrapper(JNIEnv* jni,
44 const JavaRef<jobject>& decoder)
45 : decoder_(jni, decoder),
46 implementation_name_(JavaToStdString(
47 jni,
48 Java_VideoDecoder_getImplementationName(jni, decoder))),
49 initialized_(false),
50 qp_parsing_enabled_(true) // QP parsing starts enabled and we disable it
51 // if the decoder provides frames.
52
53 {
54 decoder_thread_checker_.Detach();
55 }
56
57 VideoDecoderWrapper::~VideoDecoderWrapper() = default;
58
Configure(const Settings & settings)59 bool VideoDecoderWrapper::Configure(const Settings& settings) {
60 RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
61 JNIEnv* jni = AttachCurrentThreadIfNeeded();
62 decoder_settings_ = settings;
63 return ConfigureInternal(jni);
64 }
65
ConfigureInternal(JNIEnv * jni)66 bool VideoDecoderWrapper::ConfigureInternal(JNIEnv* jni) {
67 RenderResolution resolution = decoder_settings_.max_render_resolution();
68 ScopedJavaLocalRef<jobject> settings =
69 Java_Settings_Constructor(jni, decoder_settings_.number_of_cores(),
70 resolution.Width(), resolution.Height());
71
72 ScopedJavaLocalRef<jobject> callback =
73 Java_VideoDecoderWrapper_createDecoderCallback(jni,
74 jlongFromPointer(this));
75
76 int32_t status = JavaToNativeVideoCodecStatus(
77 jni, Java_VideoDecoder_initDecode(jni, decoder_, settings, callback));
78 RTC_LOG(LS_INFO) << "initDecode: " << status;
79 if (status == WEBRTC_VIDEO_CODEC_OK) {
80 initialized_ = true;
81 }
82
83 // The decoder was reinitialized so re-enable the QP parsing in case it stops
84 // providing QP values.
85 qp_parsing_enabled_ = true;
86
87 return status == WEBRTC_VIDEO_CODEC_OK;
88 }
89
Decode(const EncodedImage & image_param,bool missing_frames,int64_t render_time_ms)90 int32_t VideoDecoderWrapper::Decode(
91 const EncodedImage& image_param,
92 bool missing_frames,
93 int64_t render_time_ms) {
94 RTC_DCHECK_RUN_ON(&decoder_thread_checker_);
95 if (!initialized_) {
96 // Most likely initializing the codec failed.
97 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
98 }
99
100 // Make a mutable copy so we can modify the timestamp.
101 EncodedImage input_image(image_param);
102 // We use RTP timestamp for capture time because capture_time_ms_ is always 0.
103 input_image.capture_time_ms_ =
104 input_image.Timestamp() / kNumRtpTicksPerMillisec;
105
106 FrameExtraInfo frame_extra_info;
107 frame_extra_info.timestamp_ns =
108 input_image.capture_time_ms_ * rtc::kNumNanosecsPerMillisec;
109 frame_extra_info.timestamp_rtp = input_image.Timestamp();
110 frame_extra_info.timestamp_ntp = input_image.ntp_time_ms_;
111 frame_extra_info.qp =
112 qp_parsing_enabled_ ? ParseQP(input_image) : absl::nullopt;
113 {
114 MutexLock lock(&frame_extra_infos_lock_);
115 frame_extra_infos_.push_back(frame_extra_info);
116 }
117
118 JNIEnv* env = AttachCurrentThreadIfNeeded();
119 ScopedJavaLocalRef<jobject> jinput_image =
120 NativeToJavaEncodedImage(env, input_image);
121 ScopedJavaLocalRef<jobject> decode_info;
122 ScopedJavaLocalRef<jobject> ret =
123 Java_VideoDecoder_decode(env, decoder_, jinput_image, decode_info);
124 return HandleReturnCode(env, ret, "decode");
125 }
126
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)127 int32_t VideoDecoderWrapper::RegisterDecodeCompleteCallback(
128 DecodedImageCallback* callback) {
129 RTC_DCHECK_RUNS_SERIALIZED(&callback_race_checker_);
130 callback_ = callback;
131 return WEBRTC_VIDEO_CODEC_OK;
132 }
133
Release()134 int32_t VideoDecoderWrapper::Release() {
135 JNIEnv* jni = AttachCurrentThreadIfNeeded();
136 int32_t status = JavaToNativeVideoCodecStatus(
137 jni, Java_VideoDecoder_release(jni, decoder_));
138 RTC_LOG(LS_INFO) << "release: " << status;
139 {
140 MutexLock lock(&frame_extra_infos_lock_);
141 frame_extra_infos_.clear();
142 }
143 initialized_ = false;
144 // It is allowed to reinitialize the codec on a different thread.
145 decoder_thread_checker_.Detach();
146 return status;
147 }
148
ImplementationName() const149 const char* VideoDecoderWrapper::ImplementationName() const {
150 return implementation_name_.c_str();
151 }
152
OnDecodedFrame(JNIEnv * env,const JavaRef<jobject> & j_frame,const JavaRef<jobject> & j_decode_time_ms,const JavaRef<jobject> & j_qp)153 void VideoDecoderWrapper::OnDecodedFrame(
154 JNIEnv* env,
155 const JavaRef<jobject>& j_frame,
156 const JavaRef<jobject>& j_decode_time_ms,
157 const JavaRef<jobject>& j_qp) {
158 RTC_DCHECK_RUNS_SERIALIZED(&callback_race_checker_);
159 const int64_t timestamp_ns = GetJavaVideoFrameTimestampNs(env, j_frame);
160
161 FrameExtraInfo frame_extra_info;
162 {
163 MutexLock lock(&frame_extra_infos_lock_);
164
165 do {
166 if (frame_extra_infos_.empty()) {
167 RTC_LOG(LS_WARNING)
168 << "Java decoder produced an unexpected frame: " << timestamp_ns;
169 return;
170 }
171
172 frame_extra_info = frame_extra_infos_.front();
173 frame_extra_infos_.pop_front();
174 // If the decoder might drop frames so iterate through the queue until we
175 // find a matching timestamp.
176 } while (frame_extra_info.timestamp_ns != timestamp_ns);
177 }
178
179 VideoFrame frame =
180 JavaToNativeFrame(env, j_frame, frame_extra_info.timestamp_rtp);
181 frame.set_ntp_time_ms(frame_extra_info.timestamp_ntp);
182
183 absl::optional<int32_t> decoding_time_ms =
184 JavaToNativeOptionalInt(env, j_decode_time_ms);
185
186 absl::optional<uint8_t> decoder_qp =
187 cast_optional<uint8_t, int32_t>(JavaToNativeOptionalInt(env, j_qp));
188 // If the decoder provides QP values itself, no need to parse the bitstream.
189 // Enable QP parsing if decoder does not provide QP values itself.
190 qp_parsing_enabled_ = !decoder_qp.has_value();
191 callback_->Decoded(frame, decoding_time_ms,
192 decoder_qp ? decoder_qp : frame_extra_info.qp);
193 }
194
195 VideoDecoderWrapper::FrameExtraInfo::FrameExtraInfo() = default;
196 VideoDecoderWrapper::FrameExtraInfo::FrameExtraInfo(const FrameExtraInfo&) =
197 default;
198 VideoDecoderWrapper::FrameExtraInfo::~FrameExtraInfo() = default;
199
HandleReturnCode(JNIEnv * jni,const JavaRef<jobject> & j_value,const char * method_name)200 int32_t VideoDecoderWrapper::HandleReturnCode(JNIEnv* jni,
201 const JavaRef<jobject>& j_value,
202 const char* method_name) {
203 int32_t value = JavaToNativeVideoCodecStatus(jni, j_value);
204 if (value >= 0) { // OK or NO_OUTPUT
205 return value;
206 }
207
208 RTC_LOG(LS_WARNING) << method_name << ": " << value;
209 if (value == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE ||
210 value == WEBRTC_VIDEO_CODEC_UNINITIALIZED) { // Critical error.
211 RTC_LOG(LS_WARNING) << "Java decoder requested software fallback.";
212 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
213 }
214
215 // Try resetting the codec.
216 if (Release() == WEBRTC_VIDEO_CODEC_OK && ConfigureInternal(jni)) {
217 RTC_LOG(LS_WARNING) << "Reset Java decoder.";
218 return WEBRTC_VIDEO_CODEC_ERROR;
219 }
220
221 RTC_LOG(LS_WARNING) << "Unable to reset Java decoder.";
222 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
223 }
224
ParseQP(const EncodedImage & input_image)225 absl::optional<uint8_t> VideoDecoderWrapper::ParseQP(
226 const EncodedImage& input_image) {
227 if (input_image.qp_ != -1) {
228 return input_image.qp_;
229 }
230
231 absl::optional<uint8_t> qp;
232 switch (decoder_settings_.codec_type()) {
233 case kVideoCodecVP8: {
234 int qp_int;
235 if (vp8::GetQp(input_image.data(), input_image.size(), &qp_int)) {
236 qp = qp_int;
237 }
238 break;
239 }
240 case kVideoCodecVP9: {
241 int qp_int;
242 if (vp9::GetQp(input_image.data(), input_image.size(), &qp_int)) {
243 qp = qp_int;
244 }
245 break;
246 }
247 case kVideoCodecH264: {
248 h264_bitstream_parser_.ParseBitstream(input_image);
249 qp = h264_bitstream_parser_.GetLastSliceQp();
250 break;
251 }
252 default:
253 break; // Default is to not provide QP.
254 }
255 return qp;
256 }
257
JavaToNativeVideoDecoder(JNIEnv * jni,const JavaRef<jobject> & j_decoder)258 std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
259 JNIEnv* jni,
260 const JavaRef<jobject>& j_decoder) {
261 const jlong native_decoder =
262 Java_VideoDecoder_createNativeVideoDecoder(jni, j_decoder);
263 VideoDecoder* decoder;
264 if (native_decoder == 0) {
265 decoder = new VideoDecoderWrapper(jni, j_decoder);
266 } else {
267 decoder = reinterpret_cast<VideoDecoder*>(native_decoder);
268 }
269 return std::unique_ptr<VideoDecoder>(decoder);
270 }
271
272 } // namespace jni
273 } // namespace webrtc
274