1 /*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayerDecoder"
19 #include <utils/Log.h>
20 #include <inttypes.h>
21
22 #include <algorithm>
23
24 #include "NuPlayerCCDecoder.h"
25 #include "NuPlayerDecoder.h"
26 #include "NuPlayerRenderer.h"
27 #include "NuPlayerSource.h"
28
29 #include <cutils/properties.h>
30 #include <media/ICrypto.h>
31 #include <media/stagefright/foundation/ABuffer.h>
32 #include <media/stagefright/foundation/ADebug.h>
33 #include <media/stagefright/foundation/AMessage.h>
34 #include <media/stagefright/MediaBuffer.h>
35 #include <media/stagefright/MediaCodec.h>
36 #include <media/stagefright/MediaDefs.h>
37 #include <media/stagefright/MediaErrors.h>
38
39 #include <gui/Surface.h>
40
41 #include "avc_utils.h"
42 #include "ATSParser.h"
43
44 namespace android {
45
46 static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
47
48 // The default total video frame rate of a stream when that info is not available from
49 // the source.
50 static float kDefaultVideoFrameRateTotal = 30.f;
51
getAudioDeepBufferSetting()52 static inline bool getAudioDeepBufferSetting() {
53 return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
54 }
55
Decoder(const sp<AMessage> & notify,const sp<Source> & source,pid_t pid,const sp<Renderer> & renderer,const sp<Surface> & surface,const sp<CCDecoder> & ccDecoder)56 NuPlayer::Decoder::Decoder(
57 const sp<AMessage> ¬ify,
58 const sp<Source> &source,
59 pid_t pid,
60 const sp<Renderer> &renderer,
61 const sp<Surface> &surface,
62 const sp<CCDecoder> &ccDecoder)
63 : DecoderBase(notify),
64 mSurface(surface),
65 mSource(source),
66 mRenderer(renderer),
67 mCCDecoder(ccDecoder),
68 mPid(pid),
69 mSkipRenderingUntilMediaTimeUs(-1ll),
70 mNumFramesTotal(0ll),
71 mNumInputFramesDropped(0ll),
72 mNumOutputFramesDropped(0ll),
73 mVideoWidth(0),
74 mVideoHeight(0),
75 mIsAudio(true),
76 mIsVideoAVC(false),
77 mIsSecure(false),
78 mFormatChangePending(false),
79 mTimeChangePending(false),
80 mFrameRateTotal(kDefaultVideoFrameRateTotal),
81 mPlaybackSpeed(1.0f),
82 mNumVideoTemporalLayerTotal(1), // decode all layers
83 mNumVideoTemporalLayerAllowed(1),
84 mCurrentMaxVideoTemporalLayerId(0),
85 mResumePending(false),
86 mComponentName("decoder") {
87 mCodecLooper = new ALooper;
88 mCodecLooper->setName("NPDecoder-CL");
89 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
90 mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
91 }
92
~Decoder()93 NuPlayer::Decoder::~Decoder() {
94 mCodec->release();
95 releaseAndResetMediaBuffers();
96 }
97
getStats() const98 sp<AMessage> NuPlayer::Decoder::getStats() const {
99 mStats->setInt64("frames-total", mNumFramesTotal);
100 mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
101 mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
102 return mStats;
103 }
104
setVideoSurface(const sp<Surface> & surface)105 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
106 if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
107 return BAD_VALUE;
108 }
109
110 sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
111
112 msg->setObject("surface", surface);
113 sp<AMessage> response;
114 status_t err = msg->postAndAwaitResponse(&response);
115 if (err == OK && response != NULL) {
116 CHECK(response->findInt32("err", &err));
117 }
118 return err;
119 }
120
onMessageReceived(const sp<AMessage> & msg)121 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
122 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
123
124 switch (msg->what()) {
125 case kWhatCodecNotify:
126 {
127 int32_t cbID;
128 CHECK(msg->findInt32("callbackID", &cbID));
129
130 ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
131 mIsAudio ? "audio" : "video", cbID, mPaused);
132
133 if (mPaused) {
134 break;
135 }
136
137 switch (cbID) {
138 case MediaCodec::CB_INPUT_AVAILABLE:
139 {
140 int32_t index;
141 CHECK(msg->findInt32("index", &index));
142
143 handleAnInputBuffer(index);
144 break;
145 }
146
147 case MediaCodec::CB_OUTPUT_AVAILABLE:
148 {
149 int32_t index;
150 size_t offset;
151 size_t size;
152 int64_t timeUs;
153 int32_t flags;
154
155 CHECK(msg->findInt32("index", &index));
156 CHECK(msg->findSize("offset", &offset));
157 CHECK(msg->findSize("size", &size));
158 CHECK(msg->findInt64("timeUs", &timeUs));
159 CHECK(msg->findInt32("flags", &flags));
160
161 handleAnOutputBuffer(index, offset, size, timeUs, flags);
162 break;
163 }
164
165 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
166 {
167 sp<AMessage> format;
168 CHECK(msg->findMessage("format", &format));
169
170 handleOutputFormatChange(format);
171 break;
172 }
173
174 case MediaCodec::CB_ERROR:
175 {
176 status_t err;
177 CHECK(msg->findInt32("err", &err));
178 ALOGE("Decoder (%s) reported error : 0x%x",
179 mIsAudio ? "audio" : "video", err);
180
181 handleError(err);
182 break;
183 }
184
185 default:
186 {
187 TRESPASS();
188 break;
189 }
190 }
191
192 break;
193 }
194
195 case kWhatRenderBuffer:
196 {
197 if (!isStaleReply(msg)) {
198 onRenderBuffer(msg);
199 }
200 break;
201 }
202
203 case kWhatSetVideoSurface:
204 {
205 sp<AReplyToken> replyID;
206 CHECK(msg->senderAwaitsResponse(&replyID));
207
208 sp<RefBase> obj;
209 CHECK(msg->findObject("surface", &obj));
210 sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
211 int32_t err = INVALID_OPERATION;
212 // NOTE: in practice mSurface is always non-null, but checking here for completeness
213 if (mCodec != NULL && mSurface != NULL) {
214 // TODO: once AwesomePlayer is removed, remove this automatic connecting
215 // to the surface by MediaPlayerService.
216 //
217 // at this point MediaPlayerService::client has already connected to the
218 // surface, which MediaCodec does not expect
219 err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
220 if (err == OK) {
221 err = mCodec->setSurface(surface);
222 ALOGI_IF(err, "codec setSurface returned: %d", err);
223 if (err == OK) {
224 // reconnect to the old surface as MPS::Client will expect to
225 // be able to disconnect from it.
226 (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
227 mSurface = surface;
228 }
229 }
230 if (err != OK) {
231 // reconnect to the new surface on error as MPS::Client will expect to
232 // be able to disconnect from it.
233 (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
234 }
235 }
236
237 sp<AMessage> response = new AMessage;
238 response->setInt32("err", err);
239 response->postReply(replyID);
240 break;
241 }
242
243 default:
244 DecoderBase::onMessageReceived(msg);
245 break;
246 }
247 }
248
onConfigure(const sp<AMessage> & format)249 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
250 CHECK(mCodec == NULL);
251
252 mFormatChangePending = false;
253 mTimeChangePending = false;
254
255 ++mBufferGeneration;
256
257 AString mime;
258 CHECK(format->findString("mime", &mime));
259
260 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
261 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
262
263 mComponentName = mime;
264 mComponentName.append(" decoder");
265 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
266
267 mCodec = MediaCodec::CreateByType(
268 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid);
269 int32_t secure = 0;
270 if (format->findInt32("secure", &secure) && secure != 0) {
271 if (mCodec != NULL) {
272 mCodec->getName(&mComponentName);
273 mComponentName.append(".secure");
274 mCodec->release();
275 ALOGI("[%s] creating", mComponentName.c_str());
276 mCodec = MediaCodec::CreateByComponentName(
277 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid);
278 }
279 }
280 if (mCodec == NULL) {
281 ALOGE("Failed to create %s%s decoder",
282 (secure ? "secure " : ""), mime.c_str());
283 handleError(UNKNOWN_ERROR);
284 return;
285 }
286 mIsSecure = secure;
287
288 mCodec->getName(&mComponentName);
289
290 status_t err;
291 if (mSurface != NULL) {
292 // disconnect from surface as MediaCodec will reconnect
293 err = native_window_api_disconnect(
294 mSurface.get(), NATIVE_WINDOW_API_MEDIA);
295 // We treat this as a warning, as this is a preparatory step.
296 // Codec will try to connect to the surface, which is where
297 // any error signaling will occur.
298 ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
299 }
300 err = mCodec->configure(
301 format, mSurface, NULL /* crypto */, 0 /* flags */);
302 if (err != OK) {
303 ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
304 mCodec->release();
305 mCodec.clear();
306 handleError(err);
307 return;
308 }
309 rememberCodecSpecificData(format);
310
311 // the following should work in configured state
312 CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
313 CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
314
315 mStats->setString("mime", mime.c_str());
316 mStats->setString("component-name", mComponentName.c_str());
317
318 if (!mIsAudio) {
319 int32_t width, height;
320 if (mOutputFormat->findInt32("width", &width)
321 && mOutputFormat->findInt32("height", &height)) {
322 mStats->setInt32("width", width);
323 mStats->setInt32("height", height);
324 }
325 }
326
327 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
328 mCodec->setCallback(reply);
329
330 err = mCodec->start();
331 if (err != OK) {
332 ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
333 mCodec->release();
334 mCodec.clear();
335 handleError(err);
336 return;
337 }
338
339 releaseAndResetMediaBuffers();
340
341 mPaused = false;
342 mResumePending = false;
343 }
344
onSetParameters(const sp<AMessage> & params)345 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> ¶ms) {
346 bool needAdjustLayers = false;
347 float frameRateTotal;
348 if (params->findFloat("frame-rate-total", &frameRateTotal)
349 && mFrameRateTotal != frameRateTotal) {
350 needAdjustLayers = true;
351 mFrameRateTotal = frameRateTotal;
352 }
353
354 int32_t numVideoTemporalLayerTotal;
355 if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
356 && numVideoTemporalLayerTotal >= 0
357 && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
358 && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
359 needAdjustLayers = true;
360 mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
361 }
362
363 if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
364 // TODO: For now, layer fps is calculated for some specific architectures.
365 // But it really should be extracted from the stream.
366 mVideoTemporalLayerAggregateFps[0] =
367 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
368 for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
369 mVideoTemporalLayerAggregateFps[i] =
370 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
371 + mVideoTemporalLayerAggregateFps[i - 1];
372 }
373 }
374
375 float playbackSpeed;
376 if (params->findFloat("playback-speed", &playbackSpeed)
377 && mPlaybackSpeed != playbackSpeed) {
378 needAdjustLayers = true;
379 mPlaybackSpeed = playbackSpeed;
380 }
381
382 if (needAdjustLayers) {
383 float decodeFrameRate = mFrameRateTotal;
384 // enable temporal layering optimization only if we know the layering depth
385 if (mNumVideoTemporalLayerTotal > 1) {
386 int32_t layerId;
387 for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
388 if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
389 >= kDisplayRefreshingRate * 0.9) {
390 break;
391 }
392 }
393 mNumVideoTemporalLayerAllowed = layerId + 1;
394 decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
395 }
396 ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
397 mNumVideoTemporalLayerAllowed, decodeFrameRate);
398
399 if (mCodec == NULL) {
400 ALOGW("onSetParameters called before codec is created.");
401 return;
402 }
403
404 sp<AMessage> codecParams = new AMessage();
405 codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
406 mCodec->setParameters(codecParams);
407 }
408 }
409
onSetRenderer(const sp<Renderer> & renderer)410 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
411 bool hadNoRenderer = (mRenderer == NULL);
412 mRenderer = renderer;
413 if (hadNoRenderer && mRenderer != NULL) {
414 // this means that the widevine legacy source is ready
415 onRequestInputBuffers();
416 }
417 }
418
onGetInputBuffers(Vector<sp<ABuffer>> * dstBuffers)419 void NuPlayer::Decoder::onGetInputBuffers(
420 Vector<sp<ABuffer> > *dstBuffers) {
421 CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers));
422 }
423
onResume(bool notifyComplete)424 void NuPlayer::Decoder::onResume(bool notifyComplete) {
425 mPaused = false;
426
427 if (notifyComplete) {
428 mResumePending = true;
429 }
430 mCodec->start();
431 }
432
doFlush(bool notifyComplete)433 void NuPlayer::Decoder::doFlush(bool notifyComplete) {
434 if (mCCDecoder != NULL) {
435 mCCDecoder->flush();
436 }
437
438 if (mRenderer != NULL) {
439 mRenderer->flush(mIsAudio, notifyComplete);
440 mRenderer->signalTimeDiscontinuity();
441 }
442
443 status_t err = OK;
444 if (mCodec != NULL) {
445 err = mCodec->flush();
446 mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
447 ++mBufferGeneration;
448 }
449
450 if (err != OK) {
451 ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
452 handleError(err);
453 // finish with posting kWhatFlushCompleted.
454 // we attempt to release the buffers even if flush fails.
455 }
456 releaseAndResetMediaBuffers();
457 mPaused = true;
458 }
459
460
onFlush()461 void NuPlayer::Decoder::onFlush() {
462 doFlush(true);
463
464 if (isDiscontinuityPending()) {
465 // This could happen if the client starts seeking/shutdown
466 // after we queued an EOS for discontinuities.
467 // We can consider discontinuity handled.
468 finishHandleDiscontinuity(false /* flushOnTimeChange */);
469 }
470
471 sp<AMessage> notify = mNotify->dup();
472 notify->setInt32("what", kWhatFlushCompleted);
473 notify->post();
474 }
475
onShutdown(bool notifyComplete)476 void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
477 status_t err = OK;
478
479 // if there is a pending resume request, notify complete now
480 notifyResumeCompleteIfNecessary();
481
482 if (mCodec != NULL) {
483 err = mCodec->release();
484 mCodec = NULL;
485 ++mBufferGeneration;
486
487 if (mSurface != NULL) {
488 // reconnect to surface as MediaCodec disconnected from it
489 status_t error =
490 native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
491 ALOGW_IF(error != NO_ERROR,
492 "[%s] failed to connect to native window, error=%d",
493 mComponentName.c_str(), error);
494 }
495 mComponentName = "decoder";
496 }
497
498 releaseAndResetMediaBuffers();
499
500 if (err != OK) {
501 ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
502 handleError(err);
503 // finish with posting kWhatShutdownCompleted.
504 }
505
506 if (notifyComplete) {
507 sp<AMessage> notify = mNotify->dup();
508 notify->setInt32("what", kWhatShutdownCompleted);
509 notify->post();
510 mPaused = true;
511 }
512 }
513
514 /*
515 * returns true if we should request more data
516 */
doRequestBuffers()517 bool NuPlayer::Decoder::doRequestBuffers() {
518 // mRenderer is only NULL if we have a legacy widevine source that
519 // is not yet ready. In this case we must not fetch input.
520 if (isDiscontinuityPending() || mRenderer == NULL) {
521 return false;
522 }
523 status_t err = OK;
524 while (err == OK && !mDequeuedInputBuffers.empty()) {
525 size_t bufferIx = *mDequeuedInputBuffers.begin();
526 sp<AMessage> msg = new AMessage();
527 msg->setSize("buffer-ix", bufferIx);
528 err = fetchInputData(msg);
529 if (err != OK && err != ERROR_END_OF_STREAM) {
530 // if EOS, need to queue EOS buffer
531 break;
532 }
533 mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
534
535 if (!mPendingInputMessages.empty()
536 || !onInputBufferFetched(msg)) {
537 mPendingInputMessages.push_back(msg);
538 }
539 }
540
541 return err == -EWOULDBLOCK
542 && mSource->feedMoreTSData() == OK;
543 }
544
handleError(int32_t err)545 void NuPlayer::Decoder::handleError(int32_t err)
546 {
547 // We cannot immediately release the codec due to buffers still outstanding
548 // in the renderer. We signal to the player the error so it can shutdown/release the
549 // decoder after flushing and increment the generation to discard unnecessary messages.
550
551 ++mBufferGeneration;
552
553 sp<AMessage> notify = mNotify->dup();
554 notify->setInt32("what", kWhatError);
555 notify->setInt32("err", err);
556 notify->post();
557 }
558
handleAnInputBuffer(size_t index)559 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
560 if (isDiscontinuityPending()) {
561 return false;
562 }
563
564 sp<ABuffer> buffer;
565 mCodec->getInputBuffer(index, &buffer);
566
567 if (buffer == NULL) {
568 handleError(UNKNOWN_ERROR);
569 return false;
570 }
571
572 if (index >= mInputBuffers.size()) {
573 for (size_t i = mInputBuffers.size(); i <= index; ++i) {
574 mInputBuffers.add();
575 mMediaBuffers.add();
576 mInputBufferIsDequeued.add();
577 mMediaBuffers.editItemAt(i) = NULL;
578 mInputBufferIsDequeued.editItemAt(i) = false;
579 }
580 }
581 mInputBuffers.editItemAt(index) = buffer;
582
583 //CHECK_LT(bufferIx, mInputBuffers.size());
584
585 if (mMediaBuffers[index] != NULL) {
586 mMediaBuffers[index]->release();
587 mMediaBuffers.editItemAt(index) = NULL;
588 }
589 mInputBufferIsDequeued.editItemAt(index) = true;
590
591 if (!mCSDsToSubmit.isEmpty()) {
592 sp<AMessage> msg = new AMessage();
593 msg->setSize("buffer-ix", index);
594
595 sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
596 ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
597 msg->setBuffer("buffer", buffer);
598 mCSDsToSubmit.removeAt(0);
599 if (!onInputBufferFetched(msg)) {
600 handleError(UNKNOWN_ERROR);
601 return false;
602 }
603 return true;
604 }
605
606 while (!mPendingInputMessages.empty()) {
607 sp<AMessage> msg = *mPendingInputMessages.begin();
608 if (!onInputBufferFetched(msg)) {
609 break;
610 }
611 mPendingInputMessages.erase(mPendingInputMessages.begin());
612 }
613
614 if (!mInputBufferIsDequeued.editItemAt(index)) {
615 return true;
616 }
617
618 mDequeuedInputBuffers.push_back(index);
619
620 onRequestInputBuffers();
621 return true;
622 }
623
handleAnOutputBuffer(size_t index,size_t offset,size_t size,int64_t timeUs,int32_t flags)624 bool NuPlayer::Decoder::handleAnOutputBuffer(
625 size_t index,
626 size_t offset,
627 size_t size,
628 int64_t timeUs,
629 int32_t flags) {
630 // CHECK_LT(bufferIx, mOutputBuffers.size());
631 sp<ABuffer> buffer;
632 mCodec->getOutputBuffer(index, &buffer);
633
634 if (index >= mOutputBuffers.size()) {
635 for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
636 mOutputBuffers.add();
637 }
638 }
639
640 mOutputBuffers.editItemAt(index) = buffer;
641
642 buffer->setRange(offset, size);
643 buffer->meta()->clear();
644 buffer->meta()->setInt64("timeUs", timeUs);
645
646 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
647 // we do not expect CODECCONFIG or SYNCFRAME for decoder
648
649 sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
650 reply->setSize("buffer-ix", index);
651 reply->setInt32("generation", mBufferGeneration);
652
653 if (eos) {
654 ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
655
656 buffer->meta()->setInt32("eos", true);
657 reply->setInt32("eos", true);
658 } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
659 if (timeUs < mSkipRenderingUntilMediaTimeUs) {
660 ALOGV("[%s] dropping buffer at time %lld as requested.",
661 mComponentName.c_str(), (long long)timeUs);
662
663 reply->post();
664 return true;
665 }
666
667 mSkipRenderingUntilMediaTimeUs = -1;
668 }
669
670 mNumFramesTotal += !mIsAudio;
671
672 // wait until 1st frame comes out to signal resume complete
673 notifyResumeCompleteIfNecessary();
674
675 if (mRenderer != NULL) {
676 // send the buffer to renderer.
677 mRenderer->queueBuffer(mIsAudio, buffer, reply);
678 if (eos && !isDiscontinuityPending()) {
679 mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
680 }
681 }
682
683 return true;
684 }
685
handleOutputFormatChange(const sp<AMessage> & format)686 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
687 if (!mIsAudio) {
688 int32_t width, height;
689 if (format->findInt32("width", &width)
690 && format->findInt32("height", &height)) {
691 mStats->setInt32("width", width);
692 mStats->setInt32("height", height);
693 }
694 sp<AMessage> notify = mNotify->dup();
695 notify->setInt32("what", kWhatVideoSizeChanged);
696 notify->setMessage("format", format);
697 notify->post();
698 } else if (mRenderer != NULL) {
699 uint32_t flags;
700 int64_t durationUs;
701 bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
702 if (getAudioDeepBufferSetting() // override regardless of source duration
703 || (!hasVideo
704 && mSource->getDuration(&durationUs) == OK
705 && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
706 flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
707 } else {
708 flags = AUDIO_OUTPUT_FLAG_NONE;
709 }
710
711 status_t err = mRenderer->openAudioSink(
712 format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */);
713 if (err != OK) {
714 handleError(err);
715 }
716 }
717 }
718
releaseAndResetMediaBuffers()719 void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
720 for (size_t i = 0; i < mMediaBuffers.size(); i++) {
721 if (mMediaBuffers[i] != NULL) {
722 mMediaBuffers[i]->release();
723 mMediaBuffers.editItemAt(i) = NULL;
724 }
725 }
726 mMediaBuffers.resize(mInputBuffers.size());
727 for (size_t i = 0; i < mMediaBuffers.size(); i++) {
728 mMediaBuffers.editItemAt(i) = NULL;
729 }
730 mInputBufferIsDequeued.clear();
731 mInputBufferIsDequeued.resize(mInputBuffers.size());
732 for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
733 mInputBufferIsDequeued.editItemAt(i) = false;
734 }
735
736 mPendingInputMessages.clear();
737 mDequeuedInputBuffers.clear();
738 mSkipRenderingUntilMediaTimeUs = -1;
739 }
740
requestCodecNotification()741 void NuPlayer::Decoder::requestCodecNotification() {
742 if (mCodec != NULL) {
743 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
744 reply->setInt32("generation", mBufferGeneration);
745 mCodec->requestActivityNotification(reply);
746 }
747 }
748
isStaleReply(const sp<AMessage> & msg)749 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
750 int32_t generation;
751 CHECK(msg->findInt32("generation", &generation));
752 return generation != mBufferGeneration;
753 }
754
fetchInputData(sp<AMessage> & reply)755 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
756 sp<ABuffer> accessUnit;
757 bool dropAccessUnit;
758 do {
759 status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
760
761 if (err == -EWOULDBLOCK) {
762 return err;
763 } else if (err != OK) {
764 if (err == INFO_DISCONTINUITY) {
765 int32_t type;
766 CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
767
768 bool formatChange =
769 (mIsAudio &&
770 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
771 || (!mIsAudio &&
772 (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
773
774 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
775
776 ALOGI("%s discontinuity (format=%d, time=%d)",
777 mIsAudio ? "audio" : "video", formatChange, timeChange);
778
779 bool seamlessFormatChange = false;
780 sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
781 if (formatChange) {
782 seamlessFormatChange =
783 supportsSeamlessFormatChange(newFormat);
784 // treat seamless format change separately
785 formatChange = !seamlessFormatChange;
786 }
787
788 // For format or time change, return EOS to queue EOS input,
789 // then wait for EOS on output.
790 if (formatChange /* not seamless */) {
791 mFormatChangePending = true;
792 err = ERROR_END_OF_STREAM;
793 } else if (timeChange) {
794 rememberCodecSpecificData(newFormat);
795 mTimeChangePending = true;
796 err = ERROR_END_OF_STREAM;
797 } else if (seamlessFormatChange) {
798 // reuse existing decoder and don't flush
799 rememberCodecSpecificData(newFormat);
800 continue;
801 } else {
802 // This stream is unaffected by the discontinuity
803 return -EWOULDBLOCK;
804 }
805 }
806
807 // reply should only be returned without a buffer set
808 // when there is an error (including EOS)
809 CHECK(err != OK);
810
811 reply->setInt32("err", err);
812 return ERROR_END_OF_STREAM;
813 }
814
815 dropAccessUnit = false;
816 if (!mIsAudio && !mIsSecure) {
817 int32_t layerId = 0;
818 bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
819 if (mRenderer->getVideoLateByUs() > 100000ll
820 && mIsVideoAVC
821 && !IsAVCReferenceFrame(accessUnit)) {
822 dropAccessUnit = true;
823 } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
824 // Add only one layer each time.
825 if (layerId > mCurrentMaxVideoTemporalLayerId + 1
826 || layerId >= mNumVideoTemporalLayerAllowed) {
827 dropAccessUnit = true;
828 ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
829 layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
830 mCurrentMaxVideoTemporalLayerId);
831 } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
832 mCurrentMaxVideoTemporalLayerId = layerId;
833 } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
834 mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
835 }
836 }
837 if (dropAccessUnit) {
838 if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
839 mCurrentMaxVideoTemporalLayerId = layerId - 1;
840 }
841 ++mNumInputFramesDropped;
842 }
843 }
844 } while (dropAccessUnit);
845
846 // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
847 #if 0
848 int64_t mediaTimeUs;
849 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
850 ALOGV("[%s] feeding input buffer at media time %.3f",
851 mIsAudio ? "audio" : "video",
852 mediaTimeUs / 1E6);
853 #endif
854
855 if (mCCDecoder != NULL) {
856 mCCDecoder->decode(accessUnit);
857 }
858
859 reply->setBuffer("buffer", accessUnit);
860
861 return OK;
862 }
863
onInputBufferFetched(const sp<AMessage> & msg)864 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
865 size_t bufferIx;
866 CHECK(msg->findSize("buffer-ix", &bufferIx));
867 CHECK_LT(bufferIx, mInputBuffers.size());
868 sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
869
870 sp<ABuffer> buffer;
871 bool hasBuffer = msg->findBuffer("buffer", &buffer);
872
873 // handle widevine classic source - that fills an arbitrary input buffer
874 MediaBuffer *mediaBuffer = NULL;
875 if (hasBuffer) {
876 mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
877 if (mediaBuffer != NULL) {
878 // likely filled another buffer than we requested: adjust buffer index
879 size_t ix;
880 for (ix = 0; ix < mInputBuffers.size(); ix++) {
881 const sp<ABuffer> &buf = mInputBuffers[ix];
882 if (buf->data() == mediaBuffer->data()) {
883 // all input buffers are dequeued on start, hence the check
884 if (!mInputBufferIsDequeued[ix]) {
885 ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
886 mComponentName.c_str(), ix, bufferIx);
887 mediaBuffer->release();
888 return false;
889 }
890
891 // TRICKY: need buffer for the metadata, so instead, set
892 // codecBuffer to the same (though incorrect) buffer to
893 // avoid a memcpy into the codecBuffer
894 codecBuffer = buffer;
895 codecBuffer->setRange(
896 mediaBuffer->range_offset(),
897 mediaBuffer->range_length());
898 bufferIx = ix;
899 break;
900 }
901 }
902 CHECK(ix < mInputBuffers.size());
903 }
904 }
905
906 if (buffer == NULL /* includes !hasBuffer */) {
907 int32_t streamErr = ERROR_END_OF_STREAM;
908 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
909
910 CHECK(streamErr != OK);
911
912 // attempt to queue EOS
913 status_t err = mCodec->queueInputBuffer(
914 bufferIx,
915 0,
916 0,
917 0,
918 MediaCodec::BUFFER_FLAG_EOS);
919 if (err == OK) {
920 mInputBufferIsDequeued.editItemAt(bufferIx) = false;
921 } else if (streamErr == ERROR_END_OF_STREAM) {
922 streamErr = err;
923 // err will not be ERROR_END_OF_STREAM
924 }
925
926 if (streamErr != ERROR_END_OF_STREAM) {
927 ALOGE("Stream error for %s (err=%d), EOS %s queued",
928 mComponentName.c_str(),
929 streamErr,
930 err == OK ? "successfully" : "unsuccessfully");
931 handleError(streamErr);
932 }
933 } else {
934 sp<AMessage> extra;
935 if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
936 int64_t resumeAtMediaTimeUs;
937 if (extra->findInt64(
938 "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
939 ALOGI("[%s] suppressing rendering until %lld us",
940 mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
941 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
942 }
943 }
944
945 int64_t timeUs = 0;
946 uint32_t flags = 0;
947 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
948
949 int32_t eos, csd;
950 // we do not expect SYNCFRAME for decoder
951 if (buffer->meta()->findInt32("eos", &eos) && eos) {
952 flags |= MediaCodec::BUFFER_FLAG_EOS;
953 } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
954 flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
955 }
956
957 // copy into codec buffer
958 if (buffer != codecBuffer) {
959 if (buffer->size() > codecBuffer->capacity()) {
960 handleError(ERROR_BUFFER_TOO_SMALL);
961 mDequeuedInputBuffers.push_back(bufferIx);
962 return false;
963 }
964 codecBuffer->setRange(0, buffer->size());
965 memcpy(codecBuffer->data(), buffer->data(), buffer->size());
966 }
967
968 status_t err = mCodec->queueInputBuffer(
969 bufferIx,
970 codecBuffer->offset(),
971 codecBuffer->size(),
972 timeUs,
973 flags);
974 if (err != OK) {
975 if (mediaBuffer != NULL) {
976 mediaBuffer->release();
977 }
978 ALOGE("Failed to queue input buffer for %s (err=%d)",
979 mComponentName.c_str(), err);
980 handleError(err);
981 } else {
982 mInputBufferIsDequeued.editItemAt(bufferIx) = false;
983 if (mediaBuffer != NULL) {
984 CHECK(mMediaBuffers[bufferIx] == NULL);
985 mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
986 }
987 }
988 }
989 return true;
990 }
991
onRenderBuffer(const sp<AMessage> & msg)992 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
993 status_t err;
994 int32_t render;
995 size_t bufferIx;
996 int32_t eos;
997 CHECK(msg->findSize("buffer-ix", &bufferIx));
998
999 if (!mIsAudio) {
1000 int64_t timeUs;
1001 sp<ABuffer> buffer = mOutputBuffers[bufferIx];
1002 buffer->meta()->findInt64("timeUs", &timeUs);
1003
1004 if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1005 mCCDecoder->display(timeUs);
1006 }
1007 }
1008
1009 if (msg->findInt32("render", &render) && render) {
1010 int64_t timestampNs;
1011 CHECK(msg->findInt64("timestampNs", ×tampNs));
1012 err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1013 } else {
1014 mNumOutputFramesDropped += !mIsAudio;
1015 err = mCodec->releaseOutputBuffer(bufferIx);
1016 }
1017 if (err != OK) {
1018 ALOGE("failed to release output buffer for %s (err=%d)",
1019 mComponentName.c_str(), err);
1020 handleError(err);
1021 }
1022 if (msg->findInt32("eos", &eos) && eos
1023 && isDiscontinuityPending()) {
1024 finishHandleDiscontinuity(true /* flushOnTimeChange */);
1025 }
1026 }
1027
isDiscontinuityPending() const1028 bool NuPlayer::Decoder::isDiscontinuityPending() const {
1029 return mFormatChangePending || mTimeChangePending;
1030 }
1031
finishHandleDiscontinuity(bool flushOnTimeChange)1032 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1033 ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1034 mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1035
1036 // If we have format change, pause and wait to be killed;
1037 // If we have time change only, flush and restart fetching.
1038
1039 if (mFormatChangePending) {
1040 mPaused = true;
1041 } else if (mTimeChangePending) {
1042 if (flushOnTimeChange) {
1043 doFlush(false /* notifyComplete */);
1044 signalResume(false /* notifyComplete */);
1045 }
1046 }
1047
1048 // Notify NuPlayer to either shutdown decoder, or rescan sources
1049 sp<AMessage> msg = mNotify->dup();
1050 msg->setInt32("what", kWhatInputDiscontinuity);
1051 msg->setInt32("formatChange", mFormatChangePending);
1052 msg->post();
1053
1054 mFormatChangePending = false;
1055 mTimeChangePending = false;
1056 }
1057
supportsSeamlessAudioFormatChange(const sp<AMessage> & targetFormat) const1058 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1059 const sp<AMessage> &targetFormat) const {
1060 if (targetFormat == NULL) {
1061 return true;
1062 }
1063
1064 AString mime;
1065 if (!targetFormat->findString("mime", &mime)) {
1066 return false;
1067 }
1068
1069 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1070 // field-by-field comparison
1071 const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1072 for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1073 int32_t oldVal, newVal;
1074 if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1075 !targetFormat->findInt32(keys[i], &newVal) ||
1076 oldVal != newVal) {
1077 return false;
1078 }
1079 }
1080
1081 sp<ABuffer> oldBuf, newBuf;
1082 if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1083 targetFormat->findBuffer("csd-0", &newBuf)) {
1084 if (oldBuf->size() != newBuf->size()) {
1085 return false;
1086 }
1087 return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1088 }
1089 }
1090 return false;
1091 }
1092
supportsSeamlessFormatChange(const sp<AMessage> & targetFormat) const1093 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1094 if (mInputFormat == NULL) {
1095 return false;
1096 }
1097
1098 if (targetFormat == NULL) {
1099 return true;
1100 }
1101
1102 AString oldMime, newMime;
1103 if (!mInputFormat->findString("mime", &oldMime)
1104 || !targetFormat->findString("mime", &newMime)
1105 || !(oldMime == newMime)) {
1106 return false;
1107 }
1108
1109 bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1110 bool seamless;
1111 if (audio) {
1112 seamless = supportsSeamlessAudioFormatChange(targetFormat);
1113 } else {
1114 int32_t isAdaptive;
1115 seamless = (mCodec != NULL &&
1116 mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1117 isAdaptive);
1118 }
1119
1120 ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1121 return seamless;
1122 }
1123
rememberCodecSpecificData(const sp<AMessage> & format)1124 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1125 if (format == NULL) {
1126 return;
1127 }
1128 mCSDsForCurrentFormat.clear();
1129 for (int32_t i = 0; ; ++i) {
1130 AString tag = "csd-";
1131 tag.append(i);
1132 sp<ABuffer> buffer;
1133 if (!format->findBuffer(tag.c_str(), &buffer)) {
1134 break;
1135 }
1136 mCSDsForCurrentFormat.push(buffer);
1137 }
1138 }
1139
notifyResumeCompleteIfNecessary()1140 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1141 if (mResumePending) {
1142 mResumePending = false;
1143
1144 sp<AMessage> notify = mNotify->dup();
1145 notify->setInt32("what", kWhatResumeCompleted);
1146 notify->post();
1147 }
1148 }
1149
1150 } // namespace android
1151
1152