1 /*
2 * libjingle
3 * Copyright 2015 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 */
28
29 #include "talk/app/webrtc/java/jni/androidmediaencoder_jni.h"
30 #include "talk/app/webrtc/java/jni/classreferenceholder.h"
31 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
32 #include "talk/app/webrtc/java/jni/native_handle_impl.h"
33 #include "webrtc/base/bind.h"
34 #include "webrtc/base/checks.h"
35 #include "webrtc/base/logging.h"
36 #include "webrtc/base/thread.h"
37 #include "webrtc/base/thread_checker.h"
38 #include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h"
39 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
40 #include "webrtc/modules/video_coding/utility/quality_scaler.h"
41 #include "webrtc/modules/video_coding/utility/vp8_header_parser.h"
42 #include "webrtc/system_wrappers/include/field_trial.h"
43 #include "webrtc/system_wrappers/include/logcat_trace_context.h"
44 #include "third_party/libyuv/include/libyuv/convert.h"
45 #include "third_party/libyuv/include/libyuv/convert_from.h"
46 #include "third_party/libyuv/include/libyuv/video_common.h"
47
48 using rtc::Bind;
49 using rtc::Thread;
50 using rtc::ThreadManager;
51 using rtc::scoped_ptr;
52
53 using webrtc::CodecSpecificInfo;
54 using webrtc::EncodedImage;
55 using webrtc::VideoFrame;
56 using webrtc::RTPFragmentationHeader;
57 using webrtc::VideoCodec;
58 using webrtc::VideoCodecType;
59 using webrtc::kVideoCodecH264;
60 using webrtc::kVideoCodecVP8;
61 using webrtc::kVideoCodecVP9;
62
63 namespace webrtc_jni {
64
65 // H.264 start code length.
66 #define H264_SC_LENGTH 4
67 // Maximum allowed NALUs in one output frame.
68 #define MAX_NALUS_PERFRAME 32
69 // Maximum supported HW video encoder resolution.
70 #define MAX_VIDEO_WIDTH 1280
71 #define MAX_VIDEO_HEIGHT 1280
72 // Maximum supported HW video encoder fps.
73 #define MAX_VIDEO_FPS 30
74
75 // MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses
76 // Android's MediaCodec SDK API behind the scenes to implement (hopefully)
77 // HW-backed video encode. This C++ class is implemented as a very thin shim,
78 // delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder.
79 // MediaCodecVideoEncoder is created, operated, and destroyed on a single
80 // thread, currently the libjingle Worker thread.
81 class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
82 public rtc::MessageHandler {
83 public:
84 virtual ~MediaCodecVideoEncoder();
85 MediaCodecVideoEncoder(JNIEnv* jni,
86 VideoCodecType codecType,
87 jobject egl_context);
88
89 // webrtc::VideoEncoder implementation. Everything trampolines to
90 // |codec_thread_| for execution.
91 int32_t InitEncode(const webrtc::VideoCodec* codec_settings,
92 int32_t /* number_of_cores */,
93 size_t /* max_payload_size */) override;
94 int32_t Encode(const webrtc::VideoFrame& input_image,
95 const webrtc::CodecSpecificInfo* /* codec_specific_info */,
96 const std::vector<webrtc::FrameType>* frame_types) override;
97 int32_t RegisterEncodeCompleteCallback(
98 webrtc::EncodedImageCallback* callback) override;
99 int32_t Release() override;
100 int32_t SetChannelParameters(uint32_t /* packet_loss */,
101 int64_t /* rtt */) override;
102 int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override;
103
104 // rtc::MessageHandler implementation.
105 void OnMessage(rtc::Message* msg) override;
106
107 void OnDroppedFrame() override;
108
109 int GetTargetFramerate() override;
110
SupportsNativeHandle() const111 bool SupportsNativeHandle() const override { return true; }
112 const char* ImplementationName() const override;
113
114 private:
115 // CHECK-fail if not running on |codec_thread_|.
116 void CheckOnCodecThread();
117
118 private:
119 // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and
120 // InitEncodeOnCodecThread() in an attempt to restore the codec to an
121 // operable state. Necessary after all manner of OMX-layer errors.
122 bool ResetCodecOnCodecThread();
123
124 // Implementation of webrtc::VideoEncoder methods above, all running on the
125 // codec thread exclusively.
126 //
127 // If width==0 then this is assumed to be a re-initialization and the
128 // previously-current values are reused instead of the passed parameters
129 // (makes it easier to reason about thread-safety).
130 int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps,
131 bool use_surface);
132 // Reconfigure to match |frame| in width, height. Also reconfigures the
133 // encoder if |frame| is a texture/byte buffer and the encoder is initialized
134 // for byte buffer/texture. Returns false if reconfiguring fails.
135 bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame);
136 int32_t EncodeOnCodecThread(
137 const webrtc::VideoFrame& input_image,
138 const std::vector<webrtc::FrameType>* frame_types);
139 bool EncodeByteBufferOnCodecThread(JNIEnv* jni,
140 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index);
141 bool EncodeTextureOnCodecThread(JNIEnv* jni,
142 bool key_frame, const webrtc::VideoFrame& frame);
143
144 int32_t RegisterEncodeCompleteCallbackOnCodecThread(
145 webrtc::EncodedImageCallback* callback);
146 int32_t ReleaseOnCodecThread();
147 int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate);
148
149 // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members.
150 int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info);
151 jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info);
152 bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info);
153 jlong GetOutputBufferInfoPresentationTimestampUs(
154 JNIEnv* jni, jobject j_output_buffer_info);
155
156 // Deliver any outputs pending in the MediaCodec to our |callback_| and return
157 // true on success.
158 bool DeliverPendingOutputs(JNIEnv* jni);
159
160 // Search for H.264 start codes.
161 int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size);
162
163 // Type of video codec.
164 VideoCodecType codecType_;
165
166 // Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to
167 // |codec_thread_| synchronously.
168 webrtc::EncodedImageCallback* callback_;
169
170 // State that is constant for the lifetime of this object once the ctor
171 // returns.
172 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec.
173 rtc::ThreadChecker codec_thread_checker_;
174 ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_;
175 ScopedGlobalRef<jobject> j_media_codec_video_encoder_;
176 jmethodID j_init_encode_method_;
177 jmethodID j_get_input_buffers_method_;
178 jmethodID j_dequeue_input_buffer_method_;
179 jmethodID j_encode_buffer_method_;
180 jmethodID j_encode_texture_method_;
181 jmethodID j_release_method_;
182 jmethodID j_set_rates_method_;
183 jmethodID j_dequeue_output_buffer_method_;
184 jmethodID j_release_output_buffer_method_;
185 jfieldID j_color_format_field_;
186 jfieldID j_info_index_field_;
187 jfieldID j_info_buffer_field_;
188 jfieldID j_info_is_key_frame_field_;
189 jfieldID j_info_presentation_timestamp_us_field_;
190
191 // State that is valid only between InitEncode() and the next Release().
192 // Touched only on codec_thread_ so no explicit synchronization necessary.
193 int width_; // Frame width in pixels.
194 int height_; // Frame height in pixels.
195 bool inited_;
196 bool use_surface_;
197 uint16_t picture_id_;
198 enum libyuv::FourCC encoder_fourcc_; // Encoder color space format.
199 int last_set_bitrate_kbps_; // Last-requested bitrate in kbps.
200 int last_set_fps_; // Last-requested frame rate.
201 int64_t current_timestamp_us_; // Current frame timestamps in us.
202 int frames_received_; // Number of frames received by encoder.
203 int frames_encoded_; // Number of frames encoded by encoder.
204 int frames_dropped_; // Number of frames dropped by encoder.
205 int frames_in_queue_; // Number of frames in encoder queue.
206 int64_t start_time_ms_; // Start time for statistics.
207 int current_frames_; // Number of frames in the current statistics interval.
208 int current_bytes_; // Encoded bytes in the current statistics interval.
209 int current_encoding_time_ms_; // Overall encoding time in the current second
210 int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame.
211 int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame.
212 std::vector<int32_t> timestamps_; // Video frames timestamp queue.
213 std::vector<int64_t> render_times_ms_; // Video frames render time queue.
214 std::vector<int64_t> frame_rtc_times_ms_; // Time when video frame is sent to
215 // encoder input.
216 int32_t output_timestamp_; // Last output frame timestamp from timestamps_ Q.
217 int64_t output_render_time_ms_; // Last output frame render time from
218 // render_times_ms_ queue.
219 // Frame size in bytes fed to MediaCodec.
220 int yuv_size_;
221 // True only when between a callback_->Encoded() call return a positive value
222 // and the next Encode() call being ignored.
223 bool drop_next_input_frame_;
224 // Global references; must be deleted in Release().
225 std::vector<jobject> input_buffers_;
226 webrtc::QualityScaler quality_scaler_;
227 // Dynamic resolution change, off by default.
228 bool scale_;
229
230 // H264 bitstream parser, used to extract QP from encoded bitstreams.
231 webrtc::H264BitstreamParser h264_bitstream_parser_;
232
233 // VP9 variables to populate codec specific structure.
234 webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for
235 // non-flexible VP9 mode.
236 uint8_t tl0_pic_idx_;
237 size_t gof_idx_;
238
239 // EGL context - owned by factory, should not be allocated/destroyed
240 // by MediaCodecVideoEncoder.
241 jobject egl_context_;
242 };
243
~MediaCodecVideoEncoder()244 MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
245 // Call Release() to ensure no more callbacks to us after we are deleted.
246 Release();
247 }
248
MediaCodecVideoEncoder(JNIEnv * jni,VideoCodecType codecType,jobject egl_context)249 MediaCodecVideoEncoder::MediaCodecVideoEncoder(
250 JNIEnv* jni, VideoCodecType codecType, jobject egl_context) :
251 codecType_(codecType),
252 callback_(NULL),
253 codec_thread_(new Thread()),
254 j_media_codec_video_encoder_class_(
255 jni,
256 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
257 j_media_codec_video_encoder_(
258 jni,
259 jni->NewObject(*j_media_codec_video_encoder_class_,
260 GetMethodID(jni,
261 *j_media_codec_video_encoder_class_,
262 "<init>",
263 "()V"))),
264 inited_(false),
265 use_surface_(false),
266 picture_id_(0),
267 egl_context_(egl_context) {
268 ScopedLocalRefFrame local_ref_frame(jni);
269 // It would be nice to avoid spinning up a new thread per MediaCodec, and
270 // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug
271 // 2732 means that deadlocks abound. This class synchronously trampolines
272 // to |codec_thread_|, so if anything else can be coming to _us_ from
273 // |codec_thread_|, or from any thread holding the |_sendCritSect| described
274 // in the bug, we have a problem. For now work around that with a dedicated
275 // thread.
276 codec_thread_->SetName("MediaCodecVideoEncoder", NULL);
277 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder";
278 codec_thread_checker_.DetachFromThread();
279 jclass j_output_buffer_info_class =
280 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo");
281 j_init_encode_method_ = GetMethodID(
282 jni,
283 *j_media_codec_video_encoder_class_,
284 "initEncode",
285 "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;"
286 "IIIILorg/webrtc/EglBase14$Context;)Z");
287 j_get_input_buffers_method_ = GetMethodID(
288 jni,
289 *j_media_codec_video_encoder_class_,
290 "getInputBuffers",
291 "()[Ljava/nio/ByteBuffer;");
292 j_dequeue_input_buffer_method_ = GetMethodID(
293 jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I");
294 j_encode_buffer_method_ = GetMethodID(
295 jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z");
296 j_encode_texture_method_ = GetMethodID(
297 jni, *j_media_codec_video_encoder_class_, "encodeTexture",
298 "(ZI[FJ)Z");
299 j_release_method_ =
300 GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V");
301 j_set_rates_method_ = GetMethodID(
302 jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z");
303 j_dequeue_output_buffer_method_ = GetMethodID(
304 jni,
305 *j_media_codec_video_encoder_class_,
306 "dequeueOutputBuffer",
307 "()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;");
308 j_release_output_buffer_method_ = GetMethodID(
309 jni, *j_media_codec_video_encoder_class_, "releaseOutputBuffer", "(I)Z");
310
311 j_color_format_field_ =
312 GetFieldID(jni, *j_media_codec_video_encoder_class_, "colorFormat", "I");
313 j_info_index_field_ =
314 GetFieldID(jni, j_output_buffer_info_class, "index", "I");
315 j_info_buffer_field_ = GetFieldID(
316 jni, j_output_buffer_info_class, "buffer", "Ljava/nio/ByteBuffer;");
317 j_info_is_key_frame_field_ =
318 GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z");
319 j_info_presentation_timestamp_us_field_ = GetFieldID(
320 jni, j_output_buffer_info_class, "presentationTimestampUs", "J");
321 CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed";
322 srand(time(NULL));
323 AllowBlockingCalls();
324 }
325
InitEncode(const webrtc::VideoCodec * codec_settings,int32_t,size_t)326 int32_t MediaCodecVideoEncoder::InitEncode(
327 const webrtc::VideoCodec* codec_settings,
328 int32_t /* number_of_cores */,
329 size_t /* max_payload_size */) {
330 const int kMinWidth = 320;
331 const int kMinHeight = 180;
332 const int kLowQpThresholdDenominator = 3;
333 if (codec_settings == NULL) {
334 ALOGE << "NULL VideoCodec instance";
335 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
336 }
337 // Factory should guard against other codecs being used with us.
338 RTC_CHECK(codec_settings->codecType == codecType_)
339 << "Unsupported codec " << codec_settings->codecType << " for "
340 << codecType_;
341
342 ALOGD << "InitEncode request";
343 scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName(
344 "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled");
345 ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
346 if (scale_) {
347 if (codecType_ == kVideoCodecVP8) {
348 // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
349 // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
350 // always = 127. Note that in SW, QP is that of the user-level range [0,
351 // 63].
352 const int kMaxQp = 127;
353 // TODO(pbos): Investigate whether high-QP thresholds make sense for VP8.
354 // This effectively disables high QP as VP8 QP can't go above this
355 // threshold.
356 const int kDisabledBadQpThreshold = kMaxQp + 1;
357 quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator,
358 kDisabledBadQpThreshold, true);
359 } else if (codecType_ == kVideoCodecH264) {
360 // H264 QP is in the range [0, 51].
361 const int kMaxQp = 51;
362 const int kBadQpThreshold = 40;
363 quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold,
364 false);
365 } else {
366 // When adding codec support to additional hardware codecs, also configure
367 // their QP thresholds for scaling.
368 RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
369 }
370 quality_scaler_.SetMinResolution(kMinWidth, kMinHeight);
371 quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
372 }
373 return codec_thread_->Invoke<int32_t>(
374 Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
375 this,
376 codec_settings->width,
377 codec_settings->height,
378 codec_settings->startBitrate,
379 codec_settings->maxFramerate,
380 false /* use_surface */));
381 }
382
Encode(const webrtc::VideoFrame & frame,const webrtc::CodecSpecificInfo *,const std::vector<webrtc::FrameType> * frame_types)383 int32_t MediaCodecVideoEncoder::Encode(
384 const webrtc::VideoFrame& frame,
385 const webrtc::CodecSpecificInfo* /* codec_specific_info */,
386 const std::vector<webrtc::FrameType>* frame_types) {
387 return codec_thread_->Invoke<int32_t>(Bind(
388 &MediaCodecVideoEncoder::EncodeOnCodecThread, this, frame, frame_types));
389 }
390
RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback)391 int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback(
392 webrtc::EncodedImageCallback* callback) {
393 return codec_thread_->Invoke<int32_t>(
394 Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread,
395 this,
396 callback));
397 }
398
Release()399 int32_t MediaCodecVideoEncoder::Release() {
400 ALOGD << "EncoderRelease request";
401 return codec_thread_->Invoke<int32_t>(
402 Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this));
403 }
404
SetChannelParameters(uint32_t,int64_t)405 int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
406 int64_t /* rtt */) {
407 return WEBRTC_VIDEO_CODEC_OK;
408 }
409
SetRates(uint32_t new_bit_rate,uint32_t frame_rate)410 int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
411 uint32_t frame_rate) {
412 if (scale_)
413 quality_scaler_.ReportFramerate(frame_rate);
414
415 return codec_thread_->Invoke<int32_t>(
416 Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
417 this,
418 new_bit_rate,
419 frame_rate));
420 }
421
OnMessage(rtc::Message * msg)422 void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
423 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
424 JNIEnv* jni = AttachCurrentThreadIfNeeded();
425 ScopedLocalRefFrame local_ref_frame(jni);
426
427 // We only ever send one message to |this| directly (not through a Bind()'d
428 // functor), so expect no ID/data.
429 RTC_CHECK(!msg->message_id) << "Unexpected message!";
430 RTC_CHECK(!msg->pdata) << "Unexpected message!";
431 if (!inited_) {
432 return;
433 }
434
435 // It would be nice to recover from a failure here if one happened, but it's
436 // unclear how to signal such a failure to the app, so instead we stay silent
437 // about it and let the next app-called API method reveal the borkedness.
438 DeliverPendingOutputs(jni);
439 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
440 }
441
ResetCodecOnCodecThread()442 bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() {
443 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
444 ALOGE << "ResetOnCodecThread";
445 if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK ||
446 InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
447 WEBRTC_VIDEO_CODEC_OK) {
448 // TODO(fischman): wouldn't it be nice if there was a way to gracefully
449 // degrade to a SW encoder at this point? There isn't one AFAICT :(
450 // https://code.google.com/p/webrtc/issues/detail?id=2920
451 return false;
452 }
453 return true;
454 }
455
InitEncodeOnCodecThread(int width,int height,int kbps,int fps,bool use_surface)456 int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
457 int width, int height, int kbps, int fps, bool use_surface) {
458 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
459 RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set.";
460 JNIEnv* jni = AttachCurrentThreadIfNeeded();
461 ScopedLocalRefFrame local_ref_frame(jni);
462
463 ALOGD << "InitEncodeOnCodecThread Type: " << (int)codecType_ << ", " <<
464 width << " x " << height << ". Bitrate: " << kbps <<
465 " kbps. Fps: " << fps;
466 if (kbps == 0) {
467 kbps = last_set_bitrate_kbps_;
468 }
469 if (fps == 0) {
470 fps = last_set_fps_;
471 }
472
473 width_ = width;
474 height_ = height;
475 last_set_bitrate_kbps_ = kbps;
476 last_set_fps_ = fps;
477 yuv_size_ = width_ * height_ * 3 / 2;
478 frames_received_ = 0;
479 frames_encoded_ = 0;
480 frames_dropped_ = 0;
481 frames_in_queue_ = 0;
482 current_timestamp_us_ = 0;
483 start_time_ms_ = GetCurrentTimeMs();
484 current_frames_ = 0;
485 current_bytes_ = 0;
486 current_encoding_time_ms_ = 0;
487 last_input_timestamp_ms_ = -1;
488 last_output_timestamp_ms_ = -1;
489 output_timestamp_ = 0;
490 output_render_time_ms_ = 0;
491 timestamps_.clear();
492 render_times_ms_.clear();
493 frame_rtc_times_ms_.clear();
494 drop_next_input_frame_ = false;
495 use_surface_ = use_surface;
496 picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF;
497 gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1);
498 tl0_pic_idx_ = static_cast<uint8_t>(rand());
499 gof_idx_ = 0;
500
501 // We enforce no extra stride/padding in the format creation step.
502 jobject j_video_codec_enum = JavaEnumFromIndex(
503 jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_);
504 const bool encode_status = jni->CallBooleanMethod(
505 *j_media_codec_video_encoder_, j_init_encode_method_,
506 j_video_codec_enum, width, height, kbps, fps,
507 (use_surface ? egl_context_ : nullptr));
508 if (!encode_status) {
509 ALOGE << "Failed to configure encoder.";
510 return WEBRTC_VIDEO_CODEC_ERROR;
511 }
512 CHECK_EXCEPTION(jni);
513
514 if (!use_surface) {
515 jobjectArray input_buffers = reinterpret_cast<jobjectArray>(
516 jni->CallObjectMethod(*j_media_codec_video_encoder_,
517 j_get_input_buffers_method_));
518 CHECK_EXCEPTION(jni);
519 if (IsNull(jni, input_buffers)) {
520 return WEBRTC_VIDEO_CODEC_ERROR;
521 }
522
523 switch (GetIntField(jni, *j_media_codec_video_encoder_,
524 j_color_format_field_)) {
525 case COLOR_FormatYUV420Planar:
526 encoder_fourcc_ = libyuv::FOURCC_YU12;
527 break;
528 case COLOR_FormatYUV420SemiPlanar:
529 case COLOR_QCOM_FormatYUV420SemiPlanar:
530 case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m:
531 encoder_fourcc_ = libyuv::FOURCC_NV12;
532 break;
533 default:
534 LOG(LS_ERROR) << "Wrong color format.";
535 return WEBRTC_VIDEO_CODEC_ERROR;
536 }
537 size_t num_input_buffers = jni->GetArrayLength(input_buffers);
538 RTC_CHECK(input_buffers_.empty())
539 << "Unexpected double InitEncode without Release";
540 input_buffers_.resize(num_input_buffers);
541 for (size_t i = 0; i < num_input_buffers; ++i) {
542 input_buffers_[i] =
543 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
544 int64_t yuv_buffer_capacity =
545 jni->GetDirectBufferCapacity(input_buffers_[i]);
546 CHECK_EXCEPTION(jni);
547 RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity";
548 }
549 }
550
551 inited_ = true;
552 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
553 return WEBRTC_VIDEO_CODEC_OK;
554 }
555
EncodeOnCodecThread(const webrtc::VideoFrame & frame,const std::vector<webrtc::FrameType> * frame_types)556 int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
557 const webrtc::VideoFrame& frame,
558 const std::vector<webrtc::FrameType>* frame_types) {
559 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
560 JNIEnv* jni = AttachCurrentThreadIfNeeded();
561 ScopedLocalRefFrame local_ref_frame(jni);
562
563 if (!inited_) {
564 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
565 }
566
567 frames_received_++;
568 if (!DeliverPendingOutputs(jni)) {
569 if (!ResetCodecOnCodecThread())
570 return WEBRTC_VIDEO_CODEC_ERROR;
571 }
572
573 if (drop_next_input_frame_) {
574 ALOGW << "Encoder drop frame - failed callback.";
575 drop_next_input_frame_ = false;
576 return WEBRTC_VIDEO_CODEC_OK;
577 }
578
579 RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
580
581 VideoFrame input_frame = frame;
582 if (scale_) {
583 // Check framerate before spatial resolution change.
584 quality_scaler_.OnEncodeFrame(frame);
585 const webrtc::QualityScaler::Resolution scaled_resolution =
586 quality_scaler_.GetScaledResolution();
587 if (scaled_resolution.width != frame.width() ||
588 scaled_resolution.height != frame.height()) {
589 if (frame.native_handle() != nullptr) {
590 rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer(
591 static_cast<AndroidTextureBuffer*>(
592 frame.video_frame_buffer().get())->ScaleAndRotate(
593 scaled_resolution.width,
594 scaled_resolution.height,
595 webrtc::kVideoRotation_0));
596 input_frame.set_video_frame_buffer(scaled_buffer);
597 } else {
598 input_frame = quality_scaler_.GetScaledFrame(frame);
599 }
600 }
601 }
602
603 if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) {
604 ALOGE << "Failed to reconfigure encoder.";
605 return WEBRTC_VIDEO_CODEC_ERROR;
606 }
607
608 // Check if we accumulated too many frames in encoder input buffers
609 // or the encoder latency exceeds 70 ms and drop frame if so.
610 if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) {
611 int encoder_latency_ms = last_input_timestamp_ms_ -
612 last_output_timestamp_ms_;
613 if (frames_in_queue_ > 2 || encoder_latency_ms > 70) {
614 ALOGD << "Drop frame - encoder is behind by " << encoder_latency_ms <<
615 " ms. Q size: " << frames_in_queue_;
616 frames_dropped_++;
617 // Report dropped frame to quality_scaler_.
618 OnDroppedFrame();
619 return WEBRTC_VIDEO_CODEC_OK;
620 }
621 }
622
623 const bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta;
624 bool encode_status = true;
625 if (!input_frame.native_handle()) {
626 int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
627 j_dequeue_input_buffer_method_);
628 CHECK_EXCEPTION(jni);
629 if (j_input_buffer_index == -1) {
630 // Video codec falls behind - no input buffer available.
631 ALOGW << "Encoder drop frame - no input buffers available";
632 frames_dropped_++;
633 // Report dropped frame to quality_scaler_.
634 OnDroppedFrame();
635 return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887.
636 }
637 if (j_input_buffer_index == -2) {
638 ResetCodecOnCodecThread();
639 return WEBRTC_VIDEO_CODEC_ERROR;
640 }
641 encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
642 j_input_buffer_index);
643 } else {
644 encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame);
645 }
646
647 if (!encode_status) {
648 ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp();
649 ResetCodecOnCodecThread();
650 return WEBRTC_VIDEO_CODEC_ERROR;
651 }
652
653 last_input_timestamp_ms_ =
654 current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
655 frames_in_queue_++;
656
657 // Save input image timestamps for later output
658 timestamps_.push_back(input_frame.timestamp());
659 render_times_ms_.push_back(input_frame.render_time_ms());
660 frame_rtc_times_ms_.push_back(GetCurrentTimeMs());
661 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
662
663 if (!DeliverPendingOutputs(jni)) {
664 ALOGE << "Failed deliver pending outputs.";
665 ResetCodecOnCodecThread();
666 return WEBRTC_VIDEO_CODEC_ERROR;
667 }
668 return WEBRTC_VIDEO_CODEC_OK;
669 }
670
MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame & frame)671 bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread(
672 const webrtc::VideoFrame& frame) {
673 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
674
675 const bool is_texture_frame = frame.native_handle() != nullptr;
676 const bool reconfigure_due_to_format = is_texture_frame != use_surface_;
677 const bool reconfigure_due_to_size =
678 frame.width() != width_ || frame.height() != height_;
679
680 if (reconfigure_due_to_format) {
681 ALOGD << "Reconfigure encoder due to format change. "
682 << (use_surface_ ?
683 "Reconfiguring to encode from byte buffer." :
684 "Reconfiguring to encode from texture.");
685 }
686 if (reconfigure_due_to_size) {
687 ALOGD << "Reconfigure encoder due to frame resolution change from "
688 << width_ << " x " << height_ << " to " << frame.width() << " x "
689 << frame.height();
690 width_ = frame.width();
691 height_ = frame.height();
692 }
693
694 if (!reconfigure_due_to_format && !reconfigure_due_to_size)
695 return true;
696
697 ReleaseOnCodecThread();
698
699 return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) ==
700 WEBRTC_VIDEO_CODEC_OK;
701 }
702
EncodeByteBufferOnCodecThread(JNIEnv * jni,bool key_frame,const webrtc::VideoFrame & frame,int input_buffer_index)703 bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni,
704 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) {
705 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
706 RTC_CHECK(!use_surface_);
707
708 ALOGV("Encoder frame in # %d. TS: %lld. Q: %d",
709 frames_received_ - 1, current_timestamp_us_ / 1000, frames_in_queue_);
710
711 jobject j_input_buffer = input_buffers_[input_buffer_index];
712 uint8_t* yuv_buffer =
713 reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
714 CHECK_EXCEPTION(jni);
715 RTC_CHECK(yuv_buffer) << "Indirect buffer??";
716 RTC_CHECK(!libyuv::ConvertFromI420(
717 frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane),
718 frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane),
719 frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane),
720 yuv_buffer, width_, width_, height_, encoder_fourcc_))
721 << "ConvertFromI420 failed";
722
723 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
724 j_encode_buffer_method_,
725 key_frame,
726 input_buffer_index,
727 yuv_size_,
728 current_timestamp_us_);
729 CHECK_EXCEPTION(jni);
730 return encode_status;
731 }
732
EncodeTextureOnCodecThread(JNIEnv * jni,bool key_frame,const webrtc::VideoFrame & frame)733 bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni,
734 bool key_frame, const webrtc::VideoFrame& frame) {
735 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
736 RTC_CHECK(use_surface_);
737 NativeHandleImpl* handle =
738 static_cast<NativeHandleImpl*>(frame.native_handle());
739 jfloatArray sampling_matrix = jni->NewFloatArray(16);
740 jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix);
741
742 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
743 j_encode_texture_method_,
744 key_frame,
745 handle->oes_texture_id,
746 sampling_matrix,
747 current_timestamp_us_);
748 CHECK_EXCEPTION(jni);
749 return encode_status;
750 }
751
RegisterEncodeCompleteCallbackOnCodecThread(webrtc::EncodedImageCallback * callback)752 int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread(
753 webrtc::EncodedImageCallback* callback) {
754 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
755 JNIEnv* jni = AttachCurrentThreadIfNeeded();
756 ScopedLocalRefFrame local_ref_frame(jni);
757 callback_ = callback;
758 return WEBRTC_VIDEO_CODEC_OK;
759 }
760
ReleaseOnCodecThread()761 int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
762 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
763 if (!inited_) {
764 return WEBRTC_VIDEO_CODEC_OK;
765 }
766 JNIEnv* jni = AttachCurrentThreadIfNeeded();
767 ALOGD << "EncoderReleaseOnCodecThread: Frames received: " <<
768 frames_received_ << ". Encoded: " << frames_encoded_ <<
769 ". Dropped: " << frames_dropped_;
770 ScopedLocalRefFrame local_ref_frame(jni);
771 for (size_t i = 0; i < input_buffers_.size(); ++i)
772 jni->DeleteGlobalRef(input_buffers_[i]);
773 input_buffers_.clear();
774 jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_);
775 CHECK_EXCEPTION(jni);
776 rtc::MessageQueueManager::Clear(this);
777 inited_ = false;
778 use_surface_ = false;
779 ALOGD << "EncoderReleaseOnCodecThread done.";
780 return WEBRTC_VIDEO_CODEC_OK;
781 }
782
SetRatesOnCodecThread(uint32_t new_bit_rate,uint32_t frame_rate)783 int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
784 uint32_t frame_rate) {
785 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
786 if (last_set_bitrate_kbps_ == new_bit_rate &&
787 last_set_fps_ == frame_rate) {
788 return WEBRTC_VIDEO_CODEC_OK;
789 }
790 JNIEnv* jni = AttachCurrentThreadIfNeeded();
791 ScopedLocalRefFrame local_ref_frame(jni);
792 if (new_bit_rate > 0) {
793 last_set_bitrate_kbps_ = new_bit_rate;
794 }
795 if (frame_rate > 0) {
796 last_set_fps_ = frame_rate;
797 }
798 bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
799 j_set_rates_method_,
800 last_set_bitrate_kbps_,
801 last_set_fps_);
802 CHECK_EXCEPTION(jni);
803 if (!ret) {
804 ResetCodecOnCodecThread();
805 return WEBRTC_VIDEO_CODEC_ERROR;
806 }
807 return WEBRTC_VIDEO_CODEC_OK;
808 }
809
GetOutputBufferInfoIndex(JNIEnv * jni,jobject j_output_buffer_info)810 int MediaCodecVideoEncoder::GetOutputBufferInfoIndex(
811 JNIEnv* jni,
812 jobject j_output_buffer_info) {
813 return GetIntField(jni, j_output_buffer_info, j_info_index_field_);
814 }
815
GetOutputBufferInfoBuffer(JNIEnv * jni,jobject j_output_buffer_info)816 jobject MediaCodecVideoEncoder::GetOutputBufferInfoBuffer(
817 JNIEnv* jni,
818 jobject j_output_buffer_info) {
819 return GetObjectField(jni, j_output_buffer_info, j_info_buffer_field_);
820 }
821
GetOutputBufferInfoIsKeyFrame(JNIEnv * jni,jobject j_output_buffer_info)822 bool MediaCodecVideoEncoder::GetOutputBufferInfoIsKeyFrame(
823 JNIEnv* jni,
824 jobject j_output_buffer_info) {
825 return GetBooleanField(jni, j_output_buffer_info, j_info_is_key_frame_field_);
826 }
827
GetOutputBufferInfoPresentationTimestampUs(JNIEnv * jni,jobject j_output_buffer_info)828 jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs(
829 JNIEnv* jni,
830 jobject j_output_buffer_info) {
831 return GetLongField(
832 jni, j_output_buffer_info, j_info_presentation_timestamp_us_field_);
833 }
834
DeliverPendingOutputs(JNIEnv * jni)835 bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
836 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
837 while (true) {
838 jobject j_output_buffer_info = jni->CallObjectMethod(
839 *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_);
840 CHECK_EXCEPTION(jni);
841 if (IsNull(jni, j_output_buffer_info)) {
842 break;
843 }
844
845 int output_buffer_index =
846 GetOutputBufferInfoIndex(jni, j_output_buffer_info);
847 if (output_buffer_index == -1) {
848 ResetCodecOnCodecThread();
849 return false;
850 }
851
852 // Get key and config frame flags.
853 jobject j_output_buffer =
854 GetOutputBufferInfoBuffer(jni, j_output_buffer_info);
855 bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info);
856
857 // Get frame timestamps from a queue - for non config frames only.
858 int64_t frame_encoding_time_ms = 0;
859 last_output_timestamp_ms_ =
860 GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) /
861 1000;
862 if (frames_in_queue_ > 0) {
863 output_timestamp_ = timestamps_.front();
864 timestamps_.erase(timestamps_.begin());
865 output_render_time_ms_ = render_times_ms_.front();
866 render_times_ms_.erase(render_times_ms_.begin());
867 frame_encoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front();
868 frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
869 frames_in_queue_--;
870 }
871
872 // Extract payload.
873 size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer);
874 uint8_t* payload = reinterpret_cast<uint8_t*>(
875 jni->GetDirectBufferAddress(j_output_buffer));
876 CHECK_EXCEPTION(jni);
877
878 ALOGV("Encoder frame out # %d. Key: %d. Size: %d. TS: %lld."
879 " Latency: %lld. EncTime: %lld",
880 frames_encoded_, key_frame, payload_size,
881 last_output_timestamp_ms_,
882 last_input_timestamp_ms_ - last_output_timestamp_ms_,
883 frame_encoding_time_ms);
884
885 // Calculate and print encoding statistics - every 3 seconds.
886 frames_encoded_++;
887 current_frames_++;
888 current_bytes_ += payload_size;
889 current_encoding_time_ms_ += frame_encoding_time_ms;
890 int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_;
891 if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs &&
892 current_frames_ > 0) {
893 ALOGD << "Encoded frames: " << frames_encoded_ << ". Bitrate: " <<
894 (current_bytes_ * 8 / statistic_time_ms) <<
895 ", target: " << last_set_bitrate_kbps_ << " kbps, fps: " <<
896 ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms)
897 << ", encTime: " <<
898 (current_encoding_time_ms_ / current_frames_) << " for last " <<
899 statistic_time_ms << " ms.";
900 start_time_ms_ = GetCurrentTimeMs();
901 current_frames_ = 0;
902 current_bytes_ = 0;
903 current_encoding_time_ms_ = 0;
904 }
905
906 // Callback - return encoded frame.
907 int32_t callback_status = 0;
908 if (callback_) {
909 scoped_ptr<webrtc::EncodedImage> image(
910 new webrtc::EncodedImage(payload, payload_size, payload_size));
911 image->_encodedWidth = width_;
912 image->_encodedHeight = height_;
913 image->_timeStamp = output_timestamp_;
914 image->capture_time_ms_ = output_render_time_ms_;
915 image->_frameType =
916 (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
917 image->_completeFrame = true;
918 image->adapt_reason_.quality_resolution_downscales =
919 scale_ ? quality_scaler_.downscale_shift() : -1;
920
921 webrtc::CodecSpecificInfo info;
922 memset(&info, 0, sizeof(info));
923 info.codecType = codecType_;
924 if (codecType_ == kVideoCodecVP8) {
925 info.codecSpecific.VP8.pictureId = picture_id_;
926 info.codecSpecific.VP8.nonReference = false;
927 info.codecSpecific.VP8.simulcastIdx = 0;
928 info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx;
929 info.codecSpecific.VP8.layerSync = false;
930 info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx;
931 info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx;
932 } else if (codecType_ == kVideoCodecVP9) {
933 if (key_frame) {
934 gof_idx_ = 0;
935 }
936 info.codecSpecific.VP9.picture_id = picture_id_;
937 info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true;
938 info.codecSpecific.VP9.flexible_mode = false;
939 info.codecSpecific.VP9.ss_data_available = key_frame ? true : false;
940 info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++;
941 info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx;
942 info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx;
943 info.codecSpecific.VP9.temporal_up_switch = true;
944 info.codecSpecific.VP9.inter_layer_predicted = false;
945 info.codecSpecific.VP9.gof_idx =
946 static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof);
947 info.codecSpecific.VP9.num_spatial_layers = 1;
948 info.codecSpecific.VP9.spatial_layer_resolution_present = false;
949 if (info.codecSpecific.VP9.ss_data_available) {
950 info.codecSpecific.VP9.spatial_layer_resolution_present = true;
951 info.codecSpecific.VP9.width[0] = width_;
952 info.codecSpecific.VP9.height[0] = height_;
953 info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_);
954 }
955 }
956 picture_id_ = (picture_id_ + 1) & 0x7FFF;
957
958 // Generate a header describing a single fragment.
959 webrtc::RTPFragmentationHeader header;
960 memset(&header, 0, sizeof(header));
961 if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) {
962 header.VerifyAndAllocateFragmentationHeader(1);
963 header.fragmentationOffset[0] = 0;
964 header.fragmentationLength[0] = image->_length;
965 header.fragmentationPlType[0] = 0;
966 header.fragmentationTimeDiff[0] = 0;
967 if (codecType_ == kVideoCodecVP8 && scale_) {
968 int qp;
969 if (webrtc::vp8::GetQp(payload, payload_size, &qp))
970 quality_scaler_.ReportQP(qp);
971 }
972 } else if (codecType_ == kVideoCodecH264) {
973 if (scale_) {
974 h264_bitstream_parser_.ParseBitstream(payload, payload_size);
975 int qp;
976 if (h264_bitstream_parser_.GetLastSliceQp(&qp))
977 quality_scaler_.ReportQP(qp);
978 }
979 // For H.264 search for start codes.
980 int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {};
981 int32_t scPositionsLength = 0;
982 int32_t scPosition = 0;
983 while (scPositionsLength < MAX_NALUS_PERFRAME) {
984 int32_t naluPosition = NextNaluPosition(
985 payload + scPosition, payload_size - scPosition);
986 if (naluPosition < 0) {
987 break;
988 }
989 scPosition += naluPosition;
990 scPositions[scPositionsLength++] = scPosition;
991 scPosition += H264_SC_LENGTH;
992 }
993 if (scPositionsLength == 0) {
994 ALOGE << "Start code is not found!";
995 ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1]
996 << " " << image->_buffer[2] << " " << image->_buffer[3]
997 << " " << image->_buffer[4] << " " << image->_buffer[5];
998 ResetCodecOnCodecThread();
999 return false;
1000 }
1001 scPositions[scPositionsLength] = payload_size;
1002 header.VerifyAndAllocateFragmentationHeader(scPositionsLength);
1003 for (size_t i = 0; i < scPositionsLength; i++) {
1004 header.fragmentationOffset[i] = scPositions[i] + H264_SC_LENGTH;
1005 header.fragmentationLength[i] =
1006 scPositions[i + 1] - header.fragmentationOffset[i];
1007 header.fragmentationPlType[i] = 0;
1008 header.fragmentationTimeDiff[i] = 0;
1009 }
1010 }
1011
1012 callback_status = callback_->Encoded(*image, &info, &header);
1013 }
1014
1015 // Return output buffer back to the encoder.
1016 bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
1017 j_release_output_buffer_method_,
1018 output_buffer_index);
1019 CHECK_EXCEPTION(jni);
1020 if (!success) {
1021 ResetCodecOnCodecThread();
1022 return false;
1023 }
1024
1025 if (callback_status > 0) {
1026 drop_next_input_frame_ = true;
1027 // Theoretically could handle callback_status<0 here, but unclear what
1028 // that would mean for us.
1029 }
1030 }
1031
1032 return true;
1033 }
1034
NextNaluPosition(uint8_t * buffer,size_t buffer_size)1035 int32_t MediaCodecVideoEncoder::NextNaluPosition(
1036 uint8_t *buffer, size_t buffer_size) {
1037 if (buffer_size < H264_SC_LENGTH) {
1038 return -1;
1039 }
1040 uint8_t *head = buffer;
1041 // Set end buffer pointer to 4 bytes before actual buffer end so we can
1042 // access head[1], head[2] and head[3] in a loop without buffer overrun.
1043 uint8_t *end = buffer + buffer_size - H264_SC_LENGTH;
1044
1045 while (head < end) {
1046 if (head[0]) {
1047 head++;
1048 continue;
1049 }
1050 if (head[1]) { // got 00xx
1051 head += 2;
1052 continue;
1053 }
1054 if (head[2]) { // got 0000xx
1055 head += 3;
1056 continue;
1057 }
1058 if (head[3] != 0x01) { // got 000000xx
1059 head++; // xx != 1, continue searching.
1060 continue;
1061 }
1062 return (int32_t)(head - buffer);
1063 }
1064 return -1;
1065 }
1066
OnDroppedFrame()1067 void MediaCodecVideoEncoder::OnDroppedFrame() {
1068 if (scale_)
1069 quality_scaler_.ReportDroppedFrame();
1070 }
1071
GetTargetFramerate()1072 int MediaCodecVideoEncoder::GetTargetFramerate() {
1073 return scale_ ? quality_scaler_.GetTargetFramerate() : -1;
1074 }
1075
ImplementationName() const1076 const char* MediaCodecVideoEncoder::ImplementationName() const {
1077 return "MediaCodec";
1078 }
1079
MediaCodecVideoEncoderFactory()1080 MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory()
1081 : egl_context_(nullptr) {
1082 JNIEnv* jni = AttachCurrentThreadIfNeeded();
1083 ScopedLocalRefFrame local_ref_frame(jni);
1084 jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder");
1085 supported_codecs_.clear();
1086
1087 bool is_vp8_hw_supported = jni->CallStaticBooleanMethod(
1088 j_encoder_class,
1089 GetStaticMethodID(jni, j_encoder_class, "isVp8HwSupported", "()Z"));
1090 CHECK_EXCEPTION(jni);
1091 if (is_vp8_hw_supported) {
1092 ALOGD << "VP8 HW Encoder supported.";
1093 supported_codecs_.push_back(VideoCodec(kVideoCodecVP8, "VP8",
1094 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1095 }
1096
1097 bool is_vp9_hw_supported = jni->CallStaticBooleanMethod(
1098 j_encoder_class,
1099 GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z"));
1100 CHECK_EXCEPTION(jni);
1101 if (is_vp9_hw_supported) {
1102 ALOGD << "VP9 HW Encoder supported.";
1103 supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9",
1104 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1105 }
1106
1107 bool is_h264_hw_supported = jni->CallStaticBooleanMethod(
1108 j_encoder_class,
1109 GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z"));
1110 CHECK_EXCEPTION(jni);
1111 if (is_h264_hw_supported) {
1112 ALOGD << "H.264 HW Encoder supported.";
1113 supported_codecs_.push_back(VideoCodec(kVideoCodecH264, "H264",
1114 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1115 }
1116 }
1117
~MediaCodecVideoEncoderFactory()1118 MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() {}
1119
SetEGLContext(JNIEnv * jni,jobject render_egl_context)1120 void MediaCodecVideoEncoderFactory::SetEGLContext(
1121 JNIEnv* jni, jobject render_egl_context) {
1122 ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext";
1123 if (egl_context_) {
1124 jni->DeleteGlobalRef(egl_context_);
1125 egl_context_ = NULL;
1126 }
1127 if (!IsNull(jni, render_egl_context)) {
1128 egl_context_ = jni->NewGlobalRef(render_egl_context);
1129 if (CheckException(jni)) {
1130 ALOGE << "error calling NewGlobalRef for EGL Context.";
1131 egl_context_ = NULL;
1132 } else {
1133 jclass j_egl_context_class =
1134 FindClass(jni, "org/webrtc/EglBase14$Context");
1135 if (!jni->IsInstanceOf(egl_context_, j_egl_context_class)) {
1136 ALOGE << "Wrong EGL Context.";
1137 jni->DeleteGlobalRef(egl_context_);
1138 egl_context_ = NULL;
1139 }
1140 }
1141 }
1142 if (egl_context_ == NULL) {
1143 ALOGW << "NULL VideoDecoder EGL context - HW surface encoding is disabled.";
1144 }
1145 }
1146
CreateVideoEncoder(VideoCodecType type)1147 webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder(
1148 VideoCodecType type) {
1149 if (supported_codecs_.empty()) {
1150 ALOGW << "No HW video encoder for type " << (int)type;
1151 return NULL;
1152 }
1153 for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin();
1154 it != supported_codecs_.end(); ++it) {
1155 if (it->type == type) {
1156 ALOGD << "Create HW video encoder for type " << (int)type <<
1157 " (" << it->name << ").";
1158 return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type,
1159 egl_context_);
1160 }
1161 }
1162 ALOGW << "Can not find HW video encoder for type " << (int)type;
1163 return NULL;
1164 }
1165
1166 const std::vector<MediaCodecVideoEncoderFactory::VideoCodec>&
codecs() const1167 MediaCodecVideoEncoderFactory::codecs() const {
1168 return supported_codecs_;
1169 }
1170
DestroyVideoEncoder(webrtc::VideoEncoder * encoder)1171 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
1172 webrtc::VideoEncoder* encoder) {
1173 ALOGD << "Destroy video encoder.";
1174 delete encoder;
1175 }
1176
1177 } // namespace webrtc_jni
1178
1179