1 /*
2 * libjingle
3 * Copyright 2015 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 */
28
29 #include <algorithm>
30 #include <vector>
31
32 #include "talk/app/webrtc/java/jni/androidmediadecoder_jni.h"
33 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
34 #include "talk/app/webrtc/java/jni/classreferenceholder.h"
35 #include "talk/app/webrtc/java/jni/native_handle_impl.h"
36 #include "talk/app/webrtc/java/jni/surfacetexturehelper_jni.h"
37 #include "webrtc/base/bind.h"
38 #include "webrtc/base/checks.h"
39 #include "webrtc/base/logging.h"
40 #include "webrtc/base/scoped_ref_ptr.h"
41 #include "webrtc/base/thread.h"
42 #include "webrtc/base/timeutils.h"
43 #include "webrtc/common_video/include/i420_buffer_pool.h"
44 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
45 #include "webrtc/system_wrappers/include/logcat_trace_context.h"
46 #include "webrtc/system_wrappers/include/tick_util.h"
47 #include "third_party/libyuv/include/libyuv/convert.h"
48 #include "third_party/libyuv/include/libyuv/convert_from.h"
49 #include "third_party/libyuv/include/libyuv/video_common.h"
50
51 using rtc::Bind;
52 using rtc::Thread;
53 using rtc::ThreadManager;
54 using rtc::scoped_ptr;
55
56 using webrtc::CodecSpecificInfo;
57 using webrtc::DecodedImageCallback;
58 using webrtc::EncodedImage;
59 using webrtc::VideoFrame;
60 using webrtc::RTPFragmentationHeader;
61 using webrtc::TickTime;
62 using webrtc::VideoCodec;
63 using webrtc::VideoCodecType;
64 using webrtc::kVideoCodecH264;
65 using webrtc::kVideoCodecVP8;
66 using webrtc::kVideoCodecVP9;
67
68 namespace webrtc_jni {
69
70 class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
71 public rtc::MessageHandler {
72 public:
73 explicit MediaCodecVideoDecoder(
74 JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context);
75 virtual ~MediaCodecVideoDecoder();
76
77 int32_t InitDecode(const VideoCodec* codecSettings, int32_t numberOfCores)
78 override;
79
80 int32_t Decode(
81 const EncodedImage& inputImage, bool missingFrames,
82 const RTPFragmentationHeader* fragmentation,
83 const CodecSpecificInfo* codecSpecificInfo = NULL,
84 int64_t renderTimeMs = -1) override;
85
86 int32_t RegisterDecodeCompleteCallback(DecodedImageCallback* callback)
87 override;
88
89 int32_t Release() override;
90
91 int32_t Reset() override;
92
PrefersLateDecoding() const93 bool PrefersLateDecoding() const override { return true; }
94
95 // rtc::MessageHandler implementation.
96 void OnMessage(rtc::Message* msg) override;
97
98 const char* ImplementationName() const override;
99
100 private:
101 // CHECK-fail if not running on |codec_thread_|.
102 void CheckOnCodecThread();
103
104 int32_t InitDecodeOnCodecThread();
105 int32_t ReleaseOnCodecThread();
106 int32_t DecodeOnCodecThread(const EncodedImage& inputImage);
107 // Deliver any outputs pending in the MediaCodec to our |callback_| and return
108 // true on success.
109 bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us);
110 int32_t ProcessHWErrorOnCodecThread();
111
112 // Type of video codec.
113 VideoCodecType codecType_;
114
115 // Render EGL context - owned by factory, should not be allocated/destroyed
116 // by VideoDecoder.
117 jobject render_egl_context_;
118
119 bool key_frame_required_;
120 bool inited_;
121 bool sw_fallback_required_;
122 bool use_surface_;
123 VideoCodec codec_;
124 webrtc::I420BufferPool decoded_frame_pool_;
125 rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_;
126 DecodedImageCallback* callback_;
127 int frames_received_; // Number of frames received by decoder.
128 int frames_decoded_; // Number of frames decoded by decoder.
129 int64_t start_time_ms_; // Start time for statistics.
130 int current_frames_; // Number of frames in the current statistics interval.
131 int current_bytes_; // Encoded bytes in the current statistics interval.
132 int current_decoding_time_ms_; // Overall decoding time in the current second
133 uint32_t max_pending_frames_; // Maximum number of pending input frames
134
135 // State that is constant for the lifetime of this object once the ctor
136 // returns.
137 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec.
138 ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_;
139 ScopedGlobalRef<jobject> j_media_codec_video_decoder_;
140 jmethodID j_init_decode_method_;
141 jmethodID j_release_method_;
142 jmethodID j_dequeue_input_buffer_method_;
143 jmethodID j_queue_input_buffer_method_;
144 jmethodID j_dequeue_byte_buffer_method_;
145 jmethodID j_dequeue_texture_buffer_method_;
146 jmethodID j_return_decoded_byte_buffer_method_;
147 // MediaCodecVideoDecoder fields.
148 jfieldID j_input_buffers_field_;
149 jfieldID j_output_buffers_field_;
150 jfieldID j_color_format_field_;
151 jfieldID j_width_field_;
152 jfieldID j_height_field_;
153 jfieldID j_stride_field_;
154 jfieldID j_slice_height_field_;
155 // MediaCodecVideoDecoder.DecodedTextureBuffer fields.
156 jfieldID j_texture_id_field_;
157 jfieldID j_transform_matrix_field_;
158 jfieldID j_texture_timestamp_ms_field_;
159 jfieldID j_texture_ntp_timestamp_ms_field_;
160 jfieldID j_texture_decode_time_ms_field_;
161 jfieldID j_texture_frame_delay_ms_field_;
162 // MediaCodecVideoDecoder.DecodedOutputBuffer fields.
163 jfieldID j_info_index_field_;
164 jfieldID j_info_offset_field_;
165 jfieldID j_info_size_field_;
166 jfieldID j_info_timestamp_ms_field_;
167 jfieldID j_info_ntp_timestamp_ms_field_;
168 jfieldID j_byte_buffer_decode_time_ms_field_;
169
170 // Global references; must be deleted in Release().
171 std::vector<jobject> input_buffers_;
172 };
173
MediaCodecVideoDecoder(JNIEnv * jni,VideoCodecType codecType,jobject render_egl_context)174 MediaCodecVideoDecoder::MediaCodecVideoDecoder(
175 JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context) :
176 codecType_(codecType),
177 render_egl_context_(render_egl_context),
178 key_frame_required_(true),
179 inited_(false),
180 sw_fallback_required_(false),
181 codec_thread_(new Thread()),
182 j_media_codec_video_decoder_class_(
183 jni,
184 FindClass(jni, "org/webrtc/MediaCodecVideoDecoder")),
185 j_media_codec_video_decoder_(
186 jni,
187 jni->NewObject(*j_media_codec_video_decoder_class_,
188 GetMethodID(jni,
189 *j_media_codec_video_decoder_class_,
190 "<init>",
191 "()V"))) {
192 ScopedLocalRefFrame local_ref_frame(jni);
193 codec_thread_->SetName("MediaCodecVideoDecoder", NULL);
194 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder";
195
196 j_init_decode_method_ = GetMethodID(
197 jni, *j_media_codec_video_decoder_class_, "initDecode",
198 "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
199 "IILorg/webrtc/SurfaceTextureHelper;)Z");
200 j_release_method_ =
201 GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V");
202 j_dequeue_input_buffer_method_ = GetMethodID(
203 jni, *j_media_codec_video_decoder_class_, "dequeueInputBuffer", "()I");
204 j_queue_input_buffer_method_ = GetMethodID(
205 jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJJJ)Z");
206 j_dequeue_byte_buffer_method_ = GetMethodID(
207 jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer",
208 "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer;");
209 j_dequeue_texture_buffer_method_ = GetMethodID(
210 jni, *j_media_codec_video_decoder_class_, "dequeueTextureBuffer",
211 "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer;");
212 j_return_decoded_byte_buffer_method_ =
213 GetMethodID(jni, *j_media_codec_video_decoder_class_,
214 "returnDecodedOutputBuffer", "(I)V");
215
216 j_input_buffers_field_ = GetFieldID(
217 jni, *j_media_codec_video_decoder_class_,
218 "inputBuffers", "[Ljava/nio/ByteBuffer;");
219 j_output_buffers_field_ = GetFieldID(
220 jni, *j_media_codec_video_decoder_class_,
221 "outputBuffers", "[Ljava/nio/ByteBuffer;");
222 j_color_format_field_ = GetFieldID(
223 jni, *j_media_codec_video_decoder_class_, "colorFormat", "I");
224 j_width_field_ = GetFieldID(
225 jni, *j_media_codec_video_decoder_class_, "width", "I");
226 j_height_field_ = GetFieldID(
227 jni, *j_media_codec_video_decoder_class_, "height", "I");
228 j_stride_field_ = GetFieldID(
229 jni, *j_media_codec_video_decoder_class_, "stride", "I");
230 j_slice_height_field_ = GetFieldID(
231 jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I");
232
233 jclass j_decoded_texture_buffer_class = FindClass(jni,
234 "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer");
235 j_texture_id_field_ = GetFieldID(
236 jni, j_decoded_texture_buffer_class, "textureID", "I");
237 j_transform_matrix_field_ = GetFieldID(
238 jni, j_decoded_texture_buffer_class, "transformMatrix", "[F");
239 j_texture_timestamp_ms_field_ = GetFieldID(
240 jni, j_decoded_texture_buffer_class, "timeStampMs", "J");
241 j_texture_ntp_timestamp_ms_field_ = GetFieldID(
242 jni, j_decoded_texture_buffer_class, "ntpTimeStampMs", "J");
243 j_texture_decode_time_ms_field_ = GetFieldID(
244 jni, j_decoded_texture_buffer_class, "decodeTimeMs", "J");
245 j_texture_frame_delay_ms_field_ = GetFieldID(
246 jni, j_decoded_texture_buffer_class, "frameDelayMs", "J");
247
248 jclass j_decoded_output_buffer_class = FindClass(jni,
249 "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer");
250 j_info_index_field_ = GetFieldID(
251 jni, j_decoded_output_buffer_class, "index", "I");
252 j_info_offset_field_ = GetFieldID(
253 jni, j_decoded_output_buffer_class, "offset", "I");
254 j_info_size_field_ = GetFieldID(
255 jni, j_decoded_output_buffer_class, "size", "I");
256 j_info_timestamp_ms_field_ = GetFieldID(
257 jni, j_decoded_output_buffer_class, "timeStampMs", "J");
258 j_info_ntp_timestamp_ms_field_ = GetFieldID(
259 jni, j_decoded_output_buffer_class, "ntpTimeStampMs", "J");
260 j_byte_buffer_decode_time_ms_field_ = GetFieldID(
261 jni, j_decoded_output_buffer_class, "decodeTimeMs", "J");
262
263 CHECK_EXCEPTION(jni) << "MediaCodecVideoDecoder ctor failed";
264 use_surface_ = (render_egl_context_ != NULL);
265 ALOGD << "MediaCodecVideoDecoder ctor. Use surface: " << use_surface_;
266 memset(&codec_, 0, sizeof(codec_));
267 AllowBlockingCalls();
268 }
269
~MediaCodecVideoDecoder()270 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
271 // Call Release() to ensure no more callbacks to us after we are deleted.
272 Release();
273 }
274
InitDecode(const VideoCodec * inst,int32_t numberOfCores)275 int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
276 int32_t numberOfCores) {
277 ALOGD << "InitDecode.";
278 if (inst == NULL) {
279 ALOGE << "NULL VideoCodec instance";
280 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
281 }
282 // Factory should guard against other codecs being used with us.
283 RTC_CHECK(inst->codecType == codecType_)
284 << "Unsupported codec " << inst->codecType << " for " << codecType_;
285
286 if (sw_fallback_required_) {
287 ALOGE << "InitDecode() - fallback to SW decoder";
288 return WEBRTC_VIDEO_CODEC_OK;
289 }
290 // Save VideoCodec instance for later.
291 if (&codec_ != inst) {
292 codec_ = *inst;
293 }
294 // If maxFramerate is not set then assume 30 fps.
295 codec_.maxFramerate = (codec_.maxFramerate >= 1) ? codec_.maxFramerate : 30;
296
297 // Call Java init.
298 return codec_thread_->Invoke<int32_t>(
299 Bind(&MediaCodecVideoDecoder::InitDecodeOnCodecThread, this));
300 }
301
InitDecodeOnCodecThread()302 int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
303 CheckOnCodecThread();
304 JNIEnv* jni = AttachCurrentThreadIfNeeded();
305 ScopedLocalRefFrame local_ref_frame(jni);
306 ALOGD << "InitDecodeOnCodecThread Type: " << (int)codecType_ << ". "
307 << codec_.width << " x " << codec_.height << ". Fps: " <<
308 (int)codec_.maxFramerate;
309
310 // Release previous codec first if it was allocated before.
311 int ret_val = ReleaseOnCodecThread();
312 if (ret_val < 0) {
313 ALOGE << "Release failure: " << ret_val << " - fallback to SW codec";
314 sw_fallback_required_ = true;
315 return WEBRTC_VIDEO_CODEC_ERROR;
316 }
317
318 // Always start with a complete key frame.
319 key_frame_required_ = true;
320 frames_received_ = 0;
321 frames_decoded_ = 0;
322
323 jobject java_surface_texture_helper_ = nullptr;
324 if (use_surface_) {
325 java_surface_texture_helper_ = jni->CallStaticObjectMethod(
326 FindClass(jni, "org/webrtc/SurfaceTextureHelper"),
327 GetStaticMethodID(jni,
328 FindClass(jni, "org/webrtc/SurfaceTextureHelper"),
329 "create",
330 "(Lorg/webrtc/EglBase$Context;)"
331 "Lorg/webrtc/SurfaceTextureHelper;"),
332 render_egl_context_);
333 RTC_CHECK(java_surface_texture_helper_ != nullptr);
334 surface_texture_helper_ = new rtc::RefCountedObject<SurfaceTextureHelper>(
335 jni, java_surface_texture_helper_);
336 }
337
338 jobject j_video_codec_enum = JavaEnumFromIndex(
339 jni, "MediaCodecVideoDecoder$VideoCodecType", codecType_);
340 bool success = jni->CallBooleanMethod(
341 *j_media_codec_video_decoder_,
342 j_init_decode_method_,
343 j_video_codec_enum,
344 codec_.width,
345 codec_.height,
346 java_surface_texture_helper_);
347 if (CheckException(jni) || !success) {
348 ALOGE << "Codec initialization error - fallback to SW codec.";
349 sw_fallback_required_ = true;
350 return WEBRTC_VIDEO_CODEC_ERROR;
351 }
352 inited_ = true;
353
354 switch (codecType_) {
355 case kVideoCodecVP8:
356 max_pending_frames_ = kMaxPendingFramesVp8;
357 break;
358 case kVideoCodecVP9:
359 max_pending_frames_ = kMaxPendingFramesVp9;
360 break;
361 case kVideoCodecH264:
362 max_pending_frames_ = kMaxPendingFramesH264;
363 break;
364 default:
365 max_pending_frames_ = 0;
366 }
367 start_time_ms_ = GetCurrentTimeMs();
368 current_frames_ = 0;
369 current_bytes_ = 0;
370 current_decoding_time_ms_ = 0;
371
372 jobjectArray input_buffers = (jobjectArray)GetObjectField(
373 jni, *j_media_codec_video_decoder_, j_input_buffers_field_);
374 size_t num_input_buffers = jni->GetArrayLength(input_buffers);
375 ALOGD << "Maximum amount of pending frames: " << max_pending_frames_;
376 input_buffers_.resize(num_input_buffers);
377 for (size_t i = 0; i < num_input_buffers; ++i) {
378 input_buffers_[i] =
379 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
380 if (CheckException(jni)) {
381 ALOGE << "NewGlobalRef error - fallback to SW codec.";
382 sw_fallback_required_ = true;
383 return WEBRTC_VIDEO_CODEC_ERROR;
384 }
385 }
386
387 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
388
389 return WEBRTC_VIDEO_CODEC_OK;
390 }
391
Release()392 int32_t MediaCodecVideoDecoder::Release() {
393 ALOGD << "DecoderRelease request";
394 return codec_thread_->Invoke<int32_t>(
395 Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this));
396 }
397
ReleaseOnCodecThread()398 int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
399 if (!inited_) {
400 return WEBRTC_VIDEO_CODEC_OK;
401 }
402 CheckOnCodecThread();
403 JNIEnv* jni = AttachCurrentThreadIfNeeded();
404 ALOGD << "DecoderReleaseOnCodecThread: Frames received: " <<
405 frames_received_ << ". Frames decoded: " << frames_decoded_;
406 ScopedLocalRefFrame local_ref_frame(jni);
407 for (size_t i = 0; i < input_buffers_.size(); i++) {
408 jni->DeleteGlobalRef(input_buffers_[i]);
409 }
410 input_buffers_.clear();
411 jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_);
412 surface_texture_helper_ = nullptr;
413 inited_ = false;
414 rtc::MessageQueueManager::Clear(this);
415 if (CheckException(jni)) {
416 ALOGE << "Decoder release exception";
417 return WEBRTC_VIDEO_CODEC_ERROR;
418 }
419 ALOGD << "DecoderReleaseOnCodecThread done";
420 return WEBRTC_VIDEO_CODEC_OK;
421 }
422
CheckOnCodecThread()423 void MediaCodecVideoDecoder::CheckOnCodecThread() {
424 RTC_CHECK(codec_thread_ == ThreadManager::Instance()->CurrentThread())
425 << "Running on wrong thread!";
426 }
427
ProcessHWErrorOnCodecThread()428 int32_t MediaCodecVideoDecoder::ProcessHWErrorOnCodecThread() {
429 CheckOnCodecThread();
430 int ret_val = ReleaseOnCodecThread();
431 if (ret_val < 0) {
432 ALOGE << "ProcessHWError: Release failure";
433 }
434 if (codecType_ == kVideoCodecH264) {
435 // For now there is no SW H.264 which can be used as fallback codec.
436 // So try to restart hw codec for now.
437 ret_val = InitDecodeOnCodecThread();
438 ALOGE << "Reset H.264 codec done. Status: " << ret_val;
439 if (ret_val == WEBRTC_VIDEO_CODEC_OK) {
440 // H.264 codec was succesfully reset - return regular error code.
441 return WEBRTC_VIDEO_CODEC_ERROR;
442 } else {
443 // Fail to restart H.264 codec - return error code which should stop the
444 // call.
445 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
446 }
447 } else {
448 sw_fallback_required_ = true;
449 ALOGE << "Return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE";
450 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
451 }
452 }
453
Decode(const EncodedImage & inputImage,bool missingFrames,const RTPFragmentationHeader * fragmentation,const CodecSpecificInfo * codecSpecificInfo,int64_t renderTimeMs)454 int32_t MediaCodecVideoDecoder::Decode(
455 const EncodedImage& inputImage,
456 bool missingFrames,
457 const RTPFragmentationHeader* fragmentation,
458 const CodecSpecificInfo* codecSpecificInfo,
459 int64_t renderTimeMs) {
460 if (sw_fallback_required_) {
461 ALOGE << "Decode() - fallback to SW codec";
462 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
463 }
464 if (callback_ == NULL) {
465 ALOGE << "Decode() - callback_ is NULL";
466 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
467 }
468 if (inputImage._buffer == NULL && inputImage._length > 0) {
469 ALOGE << "Decode() - inputImage is incorrect";
470 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
471 }
472 if (!inited_) {
473 ALOGE << "Decode() - decoder is not initialized";
474 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
475 }
476
477 // Check if encoded frame dimension has changed.
478 if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) &&
479 (inputImage._encodedWidth != codec_.width ||
480 inputImage._encodedHeight != codec_.height)) {
481 codec_.width = inputImage._encodedWidth;
482 codec_.height = inputImage._encodedHeight;
483 int32_t ret = InitDecode(&codec_, 1);
484 if (ret < 0) {
485 ALOGE << "InitDecode failure: " << ret << " - fallback to SW codec";
486 sw_fallback_required_ = true;
487 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
488 }
489 }
490
491 // Always start with a complete key frame.
492 if (key_frame_required_) {
493 if (inputImage._frameType != webrtc::kVideoFrameKey) {
494 ALOGE << "Decode() - key frame is required";
495 return WEBRTC_VIDEO_CODEC_ERROR;
496 }
497 if (!inputImage._completeFrame) {
498 ALOGE << "Decode() - complete frame is required";
499 return WEBRTC_VIDEO_CODEC_ERROR;
500 }
501 key_frame_required_ = false;
502 }
503 if (inputImage._length == 0) {
504 return WEBRTC_VIDEO_CODEC_ERROR;
505 }
506
507 return codec_thread_->Invoke<int32_t>(Bind(
508 &MediaCodecVideoDecoder::DecodeOnCodecThread, this, inputImage));
509 }
510
DecodeOnCodecThread(const EncodedImage & inputImage)511 int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
512 const EncodedImage& inputImage) {
513 CheckOnCodecThread();
514 JNIEnv* jni = AttachCurrentThreadIfNeeded();
515 ScopedLocalRefFrame local_ref_frame(jni);
516
517 // Try to drain the decoder and wait until output is not too
518 // much behind the input.
519 const int64 drain_start = GetCurrentTimeMs();
520 while ((frames_received_ > frames_decoded_ + max_pending_frames_) &&
521 (GetCurrentTimeMs() - drain_start) < kMediaCodecTimeoutMs) {
522 ALOGV("Received: %d. Decoded: %d. Wait for output...",
523 frames_received_, frames_decoded_);
524 if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
525 ALOGE << "DeliverPendingOutputs error. Frames received: " <<
526 frames_received_ << ". Frames decoded: " << frames_decoded_;
527 return ProcessHWErrorOnCodecThread();
528 }
529 }
530 if (frames_received_ > frames_decoded_ + max_pending_frames_) {
531 ALOGE << "Output buffer dequeue timeout. Frames received: " <<
532 frames_received_ << ". Frames decoded: " << frames_decoded_;
533 return ProcessHWErrorOnCodecThread();
534 }
535
536 // Get input buffer.
537 int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_decoder_,
538 j_dequeue_input_buffer_method_);
539 if (CheckException(jni) || j_input_buffer_index < 0) {
540 ALOGE << "dequeueInputBuffer error";
541 return ProcessHWErrorOnCodecThread();
542 }
543
544 // Copy encoded data to Java ByteBuffer.
545 jobject j_input_buffer = input_buffers_[j_input_buffer_index];
546 uint8_t* buffer =
547 reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
548 RTC_CHECK(buffer) << "Indirect buffer??";
549 int64_t buffer_capacity = jni->GetDirectBufferCapacity(j_input_buffer);
550 if (CheckException(jni) || buffer_capacity < inputImage._length) {
551 ALOGE << "Input frame size "<< inputImage._length <<
552 " is bigger than buffer size " << buffer_capacity;
553 return ProcessHWErrorOnCodecThread();
554 }
555 jlong presentation_timestamp_us =
556 (frames_received_ * 1000000) / codec_.maxFramerate;
557 if (frames_decoded_ < kMaxDecodedLogFrames) {
558 ALOGD << "Decoder frame in # " << frames_received_ << ". Type: "
559 << inputImage._frameType << ". Buffer # " <<
560 j_input_buffer_index << ". pTS: "
561 << (int)(presentation_timestamp_us / 1000)
562 << ". TS: " << inputImage._timeStamp
563 << ". Size: " << inputImage._length;
564 }
565 memcpy(buffer, inputImage._buffer, inputImage._length);
566
567 // Save input image timestamps for later output.
568 frames_received_++;
569 current_bytes_ += inputImage._length;
570
571 // Feed input to decoder.
572 bool success = jni->CallBooleanMethod(
573 *j_media_codec_video_decoder_,
574 j_queue_input_buffer_method_,
575 j_input_buffer_index,
576 inputImage._length,
577 presentation_timestamp_us,
578 static_cast<int64_t> (inputImage._timeStamp),
579 inputImage.ntp_time_ms_);
580 if (CheckException(jni) || !success) {
581 ALOGE << "queueInputBuffer error";
582 return ProcessHWErrorOnCodecThread();
583 }
584
585 // Try to drain the decoder
586 if (!DeliverPendingOutputs(jni, 0)) {
587 ALOGE << "DeliverPendingOutputs error";
588 return ProcessHWErrorOnCodecThread();
589 }
590
591 return WEBRTC_VIDEO_CODEC_OK;
592 }
593
DeliverPendingOutputs(JNIEnv * jni,int dequeue_timeout_ms)594 bool MediaCodecVideoDecoder::DeliverPendingOutputs(
595 JNIEnv* jni, int dequeue_timeout_ms) {
596 if (frames_received_ <= frames_decoded_) {
597 // No need to query for output buffers - decoder is drained.
598 return true;
599 }
600 // Get decoder output.
601 jobject j_decoder_output_buffer =
602 jni->CallObjectMethod(*j_media_codec_video_decoder_,
603 use_surface_ ? j_dequeue_texture_buffer_method_
604 : j_dequeue_byte_buffer_method_,
605 dequeue_timeout_ms);
606
607 if (CheckException(jni)) {
608 ALOGE << "dequeueOutputBuffer() error";
609 return false;
610 }
611 if (IsNull(jni, j_decoder_output_buffer)) {
612 // No decoded frame ready.
613 return true;
614 }
615
616 // Get decoded video frame properties.
617 int color_format = GetIntField(jni, *j_media_codec_video_decoder_,
618 j_color_format_field_);
619 int width = GetIntField(jni, *j_media_codec_video_decoder_, j_width_field_);
620 int height = GetIntField(jni, *j_media_codec_video_decoder_, j_height_field_);
621 int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_);
622 int slice_height = GetIntField(jni, *j_media_codec_video_decoder_,
623 j_slice_height_field_);
624
625 rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer;
626 int64_t output_timestamps_ms = 0;
627 int64_t output_ntp_timestamps_ms = 0;
628 int decode_time_ms = 0;
629 int64_t frame_delayed_ms = 0;
630 if (use_surface_) {
631 // Extract data from Java DecodedTextureBuffer.
632 const int texture_id =
633 GetIntField(jni, j_decoder_output_buffer, j_texture_id_field_);
634 if (texture_id != 0) { // |texture_id| == 0 represents a dropped frame.
635 const jfloatArray j_transform_matrix =
636 reinterpret_cast<jfloatArray>(GetObjectField(
637 jni, j_decoder_output_buffer, j_transform_matrix_field_));
638 const int64_t timestamp_us =
639 GetLongField(jni, j_decoder_output_buffer,
640 j_texture_timestamp_ms_field_);
641 output_timestamps_ms = GetLongField(jni, j_decoder_output_buffer,
642 j_texture_timestamp_ms_field_);
643 output_ntp_timestamps_ms =
644 GetLongField(jni, j_decoder_output_buffer,
645 j_texture_ntp_timestamp_ms_field_);
646 decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
647 j_texture_decode_time_ms_field_);
648 frame_delayed_ms = GetLongField(jni, j_decoder_output_buffer,
649 j_texture_frame_delay_ms_field_);
650
651 // Create webrtc::VideoFrameBuffer with native texture handle.
652 frame_buffer = surface_texture_helper_->CreateTextureFrame(
653 width, height, NativeHandleImpl(jni, texture_id, j_transform_matrix));
654 }
655 } else {
656 // Extract data from Java ByteBuffer and create output yuv420 frame -
657 // for non surface decoding only.
658 const int output_buffer_index =
659 GetIntField(jni, j_decoder_output_buffer, j_info_index_field_);
660 const int output_buffer_offset =
661 GetIntField(jni, j_decoder_output_buffer, j_info_offset_field_);
662 const int output_buffer_size =
663 GetIntField(jni, j_decoder_output_buffer, j_info_size_field_);
664 output_timestamps_ms = GetLongField(jni, j_decoder_output_buffer,
665 j_info_timestamp_ms_field_);
666 output_ntp_timestamps_ms =
667 GetLongField(jni, j_decoder_output_buffer,
668 j_info_ntp_timestamp_ms_field_);
669
670 decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
671 j_byte_buffer_decode_time_ms_field_);
672
673 if (output_buffer_size < width * height * 3 / 2) {
674 ALOGE << "Insufficient output buffer size: " << output_buffer_size;
675 return false;
676 }
677 jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField(
678 jni, *j_media_codec_video_decoder_, j_output_buffers_field_));
679 jobject output_buffer =
680 jni->GetObjectArrayElement(output_buffers, output_buffer_index);
681 uint8_t* payload = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(
682 output_buffer));
683 if (CheckException(jni)) {
684 return false;
685 }
686 payload += output_buffer_offset;
687
688 // Create yuv420 frame.
689 frame_buffer = decoded_frame_pool_.CreateBuffer(width, height);
690 if (color_format == COLOR_FormatYUV420Planar) {
691 RTC_CHECK_EQ(0, stride % 2);
692 RTC_CHECK_EQ(0, slice_height % 2);
693 const int uv_stride = stride / 2;
694 const int u_slice_height = slice_height / 2;
695 const uint8_t* y_ptr = payload;
696 const uint8_t* u_ptr = y_ptr + stride * slice_height;
697 const uint8_t* v_ptr = u_ptr + uv_stride * u_slice_height;
698 libyuv::I420Copy(y_ptr, stride,
699 u_ptr, uv_stride,
700 v_ptr, uv_stride,
701 frame_buffer->MutableData(webrtc::kYPlane),
702 frame_buffer->stride(webrtc::kYPlane),
703 frame_buffer->MutableData(webrtc::kUPlane),
704 frame_buffer->stride(webrtc::kUPlane),
705 frame_buffer->MutableData(webrtc::kVPlane),
706 frame_buffer->stride(webrtc::kVPlane),
707 width, height);
708 } else {
709 // All other supported formats are nv12.
710 const uint8_t* y_ptr = payload;
711 const uint8_t* uv_ptr = y_ptr + stride * slice_height;
712 libyuv::NV12ToI420(
713 y_ptr, stride,
714 uv_ptr, stride,
715 frame_buffer->MutableData(webrtc::kYPlane),
716 frame_buffer->stride(webrtc::kYPlane),
717 frame_buffer->MutableData(webrtc::kUPlane),
718 frame_buffer->stride(webrtc::kUPlane),
719 frame_buffer->MutableData(webrtc::kVPlane),
720 frame_buffer->stride(webrtc::kVPlane),
721 width, height);
722 }
723 // Return output byte buffer back to codec.
724 jni->CallVoidMethod(
725 *j_media_codec_video_decoder_,
726 j_return_decoded_byte_buffer_method_,
727 output_buffer_index);
728 if (CheckException(jni)) {
729 ALOGE << "returnDecodedOutputBuffer error";
730 return false;
731 }
732 }
733 VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0);
734 decoded_frame.set_timestamp(output_timestamps_ms);
735 decoded_frame.set_ntp_time_ms(output_ntp_timestamps_ms);
736
737 if (frames_decoded_ < kMaxDecodedLogFrames) {
738 ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width <<
739 " x " << height << ". " << stride << " x " << slice_height <<
740 ". Color: " << color_format << ". TS:" << decoded_frame.timestamp() <<
741 ". DecTime: " << (int)decode_time_ms <<
742 ". DelayTime: " << (int)frame_delayed_ms;
743 }
744
745 // Calculate and print decoding statistics - every 3 seconds.
746 frames_decoded_++;
747 current_frames_++;
748 current_decoding_time_ms_ += decode_time_ms;
749 int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_;
750 if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs &&
751 current_frames_ > 0) {
752 ALOGD << "Decoded frames: " << frames_decoded_ << ". Received frames: "
753 << frames_received_ << ". Bitrate: " <<
754 (current_bytes_ * 8 / statistic_time_ms) << " kbps, fps: " <<
755 ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms)
756 << ". decTime: " << (current_decoding_time_ms_ / current_frames_) <<
757 " for last " << statistic_time_ms << " ms.";
758 start_time_ms_ = GetCurrentTimeMs();
759 current_frames_ = 0;
760 current_bytes_ = 0;
761 current_decoding_time_ms_ = 0;
762 }
763
764 // |.IsZeroSize())| returns true when a frame has been dropped.
765 if (!decoded_frame.IsZeroSize()) {
766 // Callback - output decoded frame.
767 const int32_t callback_status =
768 callback_->Decoded(decoded_frame, decode_time_ms);
769 if (callback_status > 0) {
770 ALOGE << "callback error";
771 }
772 }
773 return true;
774 }
775
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)776 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
777 DecodedImageCallback* callback) {
778 callback_ = callback;
779 return WEBRTC_VIDEO_CODEC_OK;
780 }
781
Reset()782 int32_t MediaCodecVideoDecoder::Reset() {
783 ALOGD << "DecoderReset";
784 if (!inited_) {
785 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
786 }
787 return InitDecode(&codec_, 1);
788 }
789
OnMessage(rtc::Message * msg)790 void MediaCodecVideoDecoder::OnMessage(rtc::Message* msg) {
791 JNIEnv* jni = AttachCurrentThreadIfNeeded();
792 ScopedLocalRefFrame local_ref_frame(jni);
793 if (!inited_) {
794 return;
795 }
796 // We only ever send one message to |this| directly (not through a Bind()'d
797 // functor), so expect no ID/data.
798 RTC_CHECK(!msg->message_id) << "Unexpected message!";
799 RTC_CHECK(!msg->pdata) << "Unexpected message!";
800 CheckOnCodecThread();
801
802 if (!DeliverPendingOutputs(jni, 0)) {
803 ALOGE << "OnMessage: DeliverPendingOutputs error";
804 ProcessHWErrorOnCodecThread();
805 return;
806 }
807 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
808 }
809
MediaCodecVideoDecoderFactory()810 MediaCodecVideoDecoderFactory::MediaCodecVideoDecoderFactory() :
811 render_egl_context_(NULL) {
812 ALOGD << "MediaCodecVideoDecoderFactory ctor";
813 JNIEnv* jni = AttachCurrentThreadIfNeeded();
814 ScopedLocalRefFrame local_ref_frame(jni);
815 jclass j_decoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoDecoder");
816 supported_codec_types_.clear();
817
818 bool is_vp8_hw_supported = jni->CallStaticBooleanMethod(
819 j_decoder_class,
820 GetStaticMethodID(jni, j_decoder_class, "isVp8HwSupported", "()Z"));
821 if (CheckException(jni)) {
822 is_vp8_hw_supported = false;
823 }
824 if (is_vp8_hw_supported) {
825 ALOGD << "VP8 HW Decoder supported.";
826 supported_codec_types_.push_back(kVideoCodecVP8);
827 }
828
829 bool is_vp9_hw_supported = jni->CallStaticBooleanMethod(
830 j_decoder_class,
831 GetStaticMethodID(jni, j_decoder_class, "isVp9HwSupported", "()Z"));
832 if (CheckException(jni)) {
833 is_vp9_hw_supported = false;
834 }
835 if (is_vp9_hw_supported) {
836 ALOGD << "VP9 HW Decoder supported.";
837 supported_codec_types_.push_back(kVideoCodecVP9);
838 }
839
840 bool is_h264_hw_supported = jni->CallStaticBooleanMethod(
841 j_decoder_class,
842 GetStaticMethodID(jni, j_decoder_class, "isH264HwSupported", "()Z"));
843 if (CheckException(jni)) {
844 is_h264_hw_supported = false;
845 }
846 if (is_h264_hw_supported) {
847 ALOGD << "H264 HW Decoder supported.";
848 supported_codec_types_.push_back(kVideoCodecH264);
849 }
850 }
851
~MediaCodecVideoDecoderFactory()852 MediaCodecVideoDecoderFactory::~MediaCodecVideoDecoderFactory() {
853 ALOGD << "MediaCodecVideoDecoderFactory dtor";
854 if (render_egl_context_) {
855 JNIEnv* jni = AttachCurrentThreadIfNeeded();
856 jni->DeleteGlobalRef(render_egl_context_);
857 render_egl_context_ = NULL;
858 }
859 }
860
SetEGLContext(JNIEnv * jni,jobject render_egl_context)861 void MediaCodecVideoDecoderFactory::SetEGLContext(
862 JNIEnv* jni, jobject render_egl_context) {
863 ALOGD << "MediaCodecVideoDecoderFactory::SetEGLContext";
864 if (render_egl_context_) {
865 jni->DeleteGlobalRef(render_egl_context_);
866 render_egl_context_ = NULL;
867 }
868 if (!IsNull(jni, render_egl_context)) {
869 render_egl_context_ = jni->NewGlobalRef(render_egl_context);
870 if (CheckException(jni)) {
871 ALOGE << "error calling NewGlobalRef for EGL Context.";
872 render_egl_context_ = NULL;
873 } else {
874 jclass j_egl_context_class =
875 FindClass(jni, "org/webrtc/EglBase$Context");
876 if (!jni->IsInstanceOf(render_egl_context_, j_egl_context_class)) {
877 ALOGE << "Wrong EGL Context.";
878 jni->DeleteGlobalRef(render_egl_context_);
879 render_egl_context_ = NULL;
880 }
881 }
882 }
883 if (render_egl_context_ == NULL) {
884 ALOGW << "NULL VideoDecoder EGL context - HW surface decoding is disabled.";
885 }
886 }
887
CreateVideoDecoder(VideoCodecType type)888 webrtc::VideoDecoder* MediaCodecVideoDecoderFactory::CreateVideoDecoder(
889 VideoCodecType type) {
890 if (supported_codec_types_.empty()) {
891 ALOGW << "No HW video decoder for type " << (int)type;
892 return NULL;
893 }
894 for (VideoCodecType codec_type : supported_codec_types_) {
895 if (codec_type == type) {
896 ALOGD << "Create HW video decoder for type " << (int)type;
897 return new MediaCodecVideoDecoder(
898 AttachCurrentThreadIfNeeded(), type, render_egl_context_);
899 }
900 }
901 ALOGW << "Can not find HW video decoder for type " << (int)type;
902 return NULL;
903 }
904
DestroyVideoDecoder(webrtc::VideoDecoder * decoder)905 void MediaCodecVideoDecoderFactory::DestroyVideoDecoder(
906 webrtc::VideoDecoder* decoder) {
907 ALOGD << "Destroy video decoder.";
908 delete decoder;
909 }
910
ImplementationName() const911 const char* MediaCodecVideoDecoder::ImplementationName() const {
912 return "MediaCodec";
913 }
914
915 } // namespace webrtc_jni
916
917