1 /*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayerDecoder"
19 #include <utils/Log.h>
20 #include <inttypes.h>
21
22 #include <algorithm>
23
24 #include "NuPlayerCCDecoder.h"
25 #include "NuPlayerDecoder.h"
26 #include "NuPlayerDrm.h"
27 #include "NuPlayerRenderer.h"
28 #include "NuPlayerSource.h"
29
30 #include <cutils/properties.h>
31 #include <media/ICrypto.h>
32 #include <media/MediaCodecBuffer.h>
33 #include <media/stagefright/foundation/ABuffer.h>
34 #include <media/stagefright/foundation/ADebug.h>
35 #include <media/stagefright/foundation/AMessage.h>
36 #include <media/stagefright/MediaBuffer.h>
37 #include <media/stagefright/MediaCodec.h>
38 #include <media/stagefright/MediaDefs.h>
39 #include <media/stagefright/MediaErrors.h>
40 #include <media/stagefright/SurfaceUtils.h>
41 #include <gui/Surface.h>
42
43 #include "avc_utils.h"
44 #include "ATSParser.h"
45
46 namespace android {
47
48 static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
49
50 // The default total video frame rate of a stream when that info is not available from
51 // the source.
52 static float kDefaultVideoFrameRateTotal = 30.f;
53
getAudioDeepBufferSetting()54 static inline bool getAudioDeepBufferSetting() {
55 return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
56 }
57
Decoder(const sp<AMessage> & notify,const sp<Source> & source,pid_t pid,uid_t uid,const sp<Renderer> & renderer,const sp<Surface> & surface,const sp<CCDecoder> & ccDecoder)58 NuPlayer::Decoder::Decoder(
59 const sp<AMessage> ¬ify,
60 const sp<Source> &source,
61 pid_t pid,
62 uid_t uid,
63 const sp<Renderer> &renderer,
64 const sp<Surface> &surface,
65 const sp<CCDecoder> &ccDecoder)
66 : DecoderBase(notify),
67 mSurface(surface),
68 mSource(source),
69 mRenderer(renderer),
70 mCCDecoder(ccDecoder),
71 mPid(pid),
72 mUid(uid),
73 mSkipRenderingUntilMediaTimeUs(-1ll),
74 mNumFramesTotal(0ll),
75 mNumInputFramesDropped(0ll),
76 mNumOutputFramesDropped(0ll),
77 mVideoWidth(0),
78 mVideoHeight(0),
79 mIsAudio(true),
80 mIsVideoAVC(false),
81 mIsSecure(false),
82 mIsEncrypted(false),
83 mIsEncryptedObservedEarlier(false),
84 mFormatChangePending(false),
85 mTimeChangePending(false),
86 mFrameRateTotal(kDefaultVideoFrameRateTotal),
87 mPlaybackSpeed(1.0f),
88 mNumVideoTemporalLayerTotal(1), // decode all layers
89 mNumVideoTemporalLayerAllowed(1),
90 mCurrentMaxVideoTemporalLayerId(0),
91 mResumePending(false),
92 mComponentName("decoder") {
93 mCodecLooper = new ALooper;
94 mCodecLooper->setName("NPDecoder-CL");
95 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
96 mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
97 }
98
~Decoder()99 NuPlayer::Decoder::~Decoder() {
100 // Need to stop looper first since mCodec could be accessed on the mDecoderLooper.
101 stopLooper();
102 if (mCodec != NULL) {
103 mCodec->release();
104 }
105 releaseAndResetMediaBuffers();
106 }
107
getStats() const108 sp<AMessage> NuPlayer::Decoder::getStats() const {
109 mStats->setInt64("frames-total", mNumFramesTotal);
110 mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
111 mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
112 return mStats;
113 }
114
setVideoSurface(const sp<Surface> & surface)115 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
116 if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
117 return BAD_VALUE;
118 }
119
120 sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
121
122 msg->setObject("surface", surface);
123 sp<AMessage> response;
124 status_t err = msg->postAndAwaitResponse(&response);
125 if (err == OK && response != NULL) {
126 CHECK(response->findInt32("err", &err));
127 }
128 return err;
129 }
130
onMessageReceived(const sp<AMessage> & msg)131 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
132 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
133
134 switch (msg->what()) {
135 case kWhatCodecNotify:
136 {
137 int32_t cbID;
138 CHECK(msg->findInt32("callbackID", &cbID));
139
140 ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
141 mIsAudio ? "audio" : "video", cbID, mPaused);
142
143 if (mPaused) {
144 break;
145 }
146
147 switch (cbID) {
148 case MediaCodec::CB_INPUT_AVAILABLE:
149 {
150 int32_t index;
151 CHECK(msg->findInt32("index", &index));
152
153 handleAnInputBuffer(index);
154 break;
155 }
156
157 case MediaCodec::CB_OUTPUT_AVAILABLE:
158 {
159 int32_t index;
160 size_t offset;
161 size_t size;
162 int64_t timeUs;
163 int32_t flags;
164
165 CHECK(msg->findInt32("index", &index));
166 CHECK(msg->findSize("offset", &offset));
167 CHECK(msg->findSize("size", &size));
168 CHECK(msg->findInt64("timeUs", &timeUs));
169 CHECK(msg->findInt32("flags", &flags));
170
171 handleAnOutputBuffer(index, offset, size, timeUs, flags);
172 break;
173 }
174
175 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
176 {
177 sp<AMessage> format;
178 CHECK(msg->findMessage("format", &format));
179
180 handleOutputFormatChange(format);
181 break;
182 }
183
184 case MediaCodec::CB_ERROR:
185 {
186 status_t err;
187 CHECK(msg->findInt32("err", &err));
188 ALOGE("Decoder (%s) reported error : 0x%x",
189 mIsAudio ? "audio" : "video", err);
190
191 handleError(err);
192 break;
193 }
194
195 default:
196 {
197 TRESPASS();
198 break;
199 }
200 }
201
202 break;
203 }
204
205 case kWhatRenderBuffer:
206 {
207 if (!isStaleReply(msg)) {
208 onRenderBuffer(msg);
209 }
210 break;
211 }
212
213 case kWhatAudioOutputFormatChanged:
214 {
215 if (!isStaleReply(msg)) {
216 status_t err;
217 if (msg->findInt32("err", &err) && err != OK) {
218 ALOGE("Renderer reported 0x%x when changing audio output format", err);
219 handleError(err);
220 }
221 }
222 break;
223 }
224
225 case kWhatSetVideoSurface:
226 {
227 sp<AReplyToken> replyID;
228 CHECK(msg->senderAwaitsResponse(&replyID));
229
230 sp<RefBase> obj;
231 CHECK(msg->findObject("surface", &obj));
232 sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
233 int32_t err = INVALID_OPERATION;
234 // NOTE: in practice mSurface is always non-null, but checking here for completeness
235 if (mCodec != NULL && mSurface != NULL) {
236 // TODO: once AwesomePlayer is removed, remove this automatic connecting
237 // to the surface by MediaPlayerService.
238 //
239 // at this point MediaPlayerService::client has already connected to the
240 // surface, which MediaCodec does not expect
241 err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)");
242 if (err == OK) {
243 err = mCodec->setSurface(surface);
244 ALOGI_IF(err, "codec setSurface returned: %d", err);
245 if (err == OK) {
246 // reconnect to the old surface as MPS::Client will expect to
247 // be able to disconnect from it.
248 (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)");
249 mSurface = surface;
250 }
251 }
252 if (err != OK) {
253 // reconnect to the new surface on error as MPS::Client will expect to
254 // be able to disconnect from it.
255 (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)");
256 }
257 }
258
259 sp<AMessage> response = new AMessage;
260 response->setInt32("err", err);
261 response->postReply(replyID);
262 break;
263 }
264
265 case kWhatDrmReleaseCrypto:
266 {
267 ALOGV("kWhatDrmReleaseCrypto");
268 onReleaseCrypto(msg);
269 break;
270 }
271
272 default:
273 DecoderBase::onMessageReceived(msg);
274 break;
275 }
276 }
277
onConfigure(const sp<AMessage> & format)278 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
279 CHECK(mCodec == NULL);
280
281 mFormatChangePending = false;
282 mTimeChangePending = false;
283
284 ++mBufferGeneration;
285
286 AString mime;
287 CHECK(format->findString("mime", &mime));
288
289 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
290 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
291
292 mComponentName = mime;
293 mComponentName.append(" decoder");
294 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
295
296 mCodec = MediaCodec::CreateByType(
297 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
298 int32_t secure = 0;
299 if (format->findInt32("secure", &secure) && secure != 0) {
300 if (mCodec != NULL) {
301 mCodec->getName(&mComponentName);
302 mComponentName.append(".secure");
303 mCodec->release();
304 ALOGI("[%s] creating", mComponentName.c_str());
305 mCodec = MediaCodec::CreateByComponentName(
306 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
307 }
308 }
309 if (mCodec == NULL) {
310 ALOGE("Failed to create %s%s decoder",
311 (secure ? "secure " : ""), mime.c_str());
312 handleError(UNKNOWN_ERROR);
313 return;
314 }
315 mIsSecure = secure;
316
317 mCodec->getName(&mComponentName);
318
319 status_t err;
320 if (mSurface != NULL) {
321 // disconnect from surface as MediaCodec will reconnect
322 err = nativeWindowDisconnect(mSurface.get(), "onConfigure");
323 // We treat this as a warning, as this is a preparatory step.
324 // Codec will try to connect to the surface, which is where
325 // any error signaling will occur.
326 ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
327 }
328
329 // Modular DRM
330 void *pCrypto;
331 if (!format->findPointer("crypto", &pCrypto)) {
332 pCrypto = NULL;
333 }
334 sp<ICrypto> crypto = (ICrypto*)pCrypto;
335 // non-encrypted source won't have a crypto
336 mIsEncrypted = (crypto != NULL);
337 // configure is called once; still using OR in case the behavior changes.
338 mIsEncryptedObservedEarlier = mIsEncryptedObservedEarlier || mIsEncrypted;
339 ALOGV("onConfigure mCrypto: %p (%d) mIsSecure: %d",
340 crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure);
341
342 err = mCodec->configure(
343 format, mSurface, crypto, 0 /* flags */);
344
345 if (err != OK) {
346 ALOGE("Failed to configure [%s] decoder (err=%d)", mComponentName.c_str(), err);
347 mCodec->release();
348 mCodec.clear();
349 handleError(err);
350 return;
351 }
352 rememberCodecSpecificData(format);
353
354 // the following should work in configured state
355 CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
356 CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
357
358 mStats->setString("mime", mime.c_str());
359 mStats->setString("component-name", mComponentName.c_str());
360
361 if (!mIsAudio) {
362 int32_t width, height;
363 if (mOutputFormat->findInt32("width", &width)
364 && mOutputFormat->findInt32("height", &height)) {
365 mStats->setInt32("width", width);
366 mStats->setInt32("height", height);
367 }
368 }
369
370 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
371 mCodec->setCallback(reply);
372
373 err = mCodec->start();
374 if (err != OK) {
375 ALOGE("Failed to start [%s] decoder (err=%d)", mComponentName.c_str(), err);
376 mCodec->release();
377 mCodec.clear();
378 handleError(err);
379 return;
380 }
381
382 releaseAndResetMediaBuffers();
383
384 mPaused = false;
385 mResumePending = false;
386 }
387
onSetParameters(const sp<AMessage> & params)388 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> ¶ms) {
389 bool needAdjustLayers = false;
390 float frameRateTotal;
391 if (params->findFloat("frame-rate-total", &frameRateTotal)
392 && mFrameRateTotal != frameRateTotal) {
393 needAdjustLayers = true;
394 mFrameRateTotal = frameRateTotal;
395 }
396
397 int32_t numVideoTemporalLayerTotal;
398 if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
399 && numVideoTemporalLayerTotal >= 0
400 && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
401 && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
402 needAdjustLayers = true;
403 mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
404 }
405
406 if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
407 // TODO: For now, layer fps is calculated for some specific architectures.
408 // But it really should be extracted from the stream.
409 mVideoTemporalLayerAggregateFps[0] =
410 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
411 for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
412 mVideoTemporalLayerAggregateFps[i] =
413 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
414 + mVideoTemporalLayerAggregateFps[i - 1];
415 }
416 }
417
418 float playbackSpeed;
419 if (params->findFloat("playback-speed", &playbackSpeed)
420 && mPlaybackSpeed != playbackSpeed) {
421 needAdjustLayers = true;
422 mPlaybackSpeed = playbackSpeed;
423 }
424
425 if (needAdjustLayers) {
426 float decodeFrameRate = mFrameRateTotal;
427 // enable temporal layering optimization only if we know the layering depth
428 if (mNumVideoTemporalLayerTotal > 1) {
429 int32_t layerId;
430 for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
431 if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
432 >= kDisplayRefreshingRate * 0.9) {
433 break;
434 }
435 }
436 mNumVideoTemporalLayerAllowed = layerId + 1;
437 decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
438 }
439 ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
440 mNumVideoTemporalLayerAllowed, decodeFrameRate);
441
442 if (mCodec == NULL) {
443 ALOGW("onSetParameters called before codec is created.");
444 return;
445 }
446
447 sp<AMessage> codecParams = new AMessage();
448 codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
449 mCodec->setParameters(codecParams);
450 }
451 }
452
onSetRenderer(const sp<Renderer> & renderer)453 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
454 mRenderer = renderer;
455 }
456
onResume(bool notifyComplete)457 void NuPlayer::Decoder::onResume(bool notifyComplete) {
458 mPaused = false;
459
460 if (notifyComplete) {
461 mResumePending = true;
462 }
463
464 if (mCodec == NULL) {
465 ALOGE("[%s] onResume without a valid codec", mComponentName.c_str());
466 handleError(NO_INIT);
467 return;
468 }
469 mCodec->start();
470 }
471
doFlush(bool notifyComplete)472 void NuPlayer::Decoder::doFlush(bool notifyComplete) {
473 if (mCCDecoder != NULL) {
474 mCCDecoder->flush();
475 }
476
477 if (mRenderer != NULL) {
478 mRenderer->flush(mIsAudio, notifyComplete);
479 mRenderer->signalTimeDiscontinuity();
480 }
481
482 status_t err = OK;
483 if (mCodec != NULL) {
484 err = mCodec->flush();
485 mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
486 ++mBufferGeneration;
487 }
488
489 if (err != OK) {
490 ALOGE("failed to flush [%s] (err=%d)", mComponentName.c_str(), err);
491 handleError(err);
492 // finish with posting kWhatFlushCompleted.
493 // we attempt to release the buffers even if flush fails.
494 }
495 releaseAndResetMediaBuffers();
496 mPaused = true;
497 }
498
499
onFlush()500 void NuPlayer::Decoder::onFlush() {
501 doFlush(true);
502
503 if (isDiscontinuityPending()) {
504 // This could happen if the client starts seeking/shutdown
505 // after we queued an EOS for discontinuities.
506 // We can consider discontinuity handled.
507 finishHandleDiscontinuity(false /* flushOnTimeChange */);
508 }
509
510 sp<AMessage> notify = mNotify->dup();
511 notify->setInt32("what", kWhatFlushCompleted);
512 notify->post();
513 }
514
onShutdown(bool notifyComplete)515 void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
516 status_t err = OK;
517
518 // if there is a pending resume request, notify complete now
519 notifyResumeCompleteIfNecessary();
520
521 if (mCodec != NULL) {
522 err = mCodec->release();
523 mCodec = NULL;
524 ++mBufferGeneration;
525
526 if (mSurface != NULL) {
527 // reconnect to surface as MediaCodec disconnected from it
528 status_t error = nativeWindowConnect(mSurface.get(), "onShutdown");
529 ALOGW_IF(error != NO_ERROR,
530 "[%s] failed to connect to native window, error=%d",
531 mComponentName.c_str(), error);
532 }
533 mComponentName = "decoder";
534 }
535
536 releaseAndResetMediaBuffers();
537
538 if (err != OK) {
539 ALOGE("failed to release [%s] (err=%d)", mComponentName.c_str(), err);
540 handleError(err);
541 // finish with posting kWhatShutdownCompleted.
542 }
543
544 if (notifyComplete) {
545 sp<AMessage> notify = mNotify->dup();
546 notify->setInt32("what", kWhatShutdownCompleted);
547 notify->post();
548 mPaused = true;
549 }
550 }
551
552 /*
553 * returns true if we should request more data
554 */
doRequestBuffers()555 bool NuPlayer::Decoder::doRequestBuffers() {
556 if (isDiscontinuityPending()) {
557 return false;
558 }
559 status_t err = OK;
560 while (err == OK && !mDequeuedInputBuffers.empty()) {
561 size_t bufferIx = *mDequeuedInputBuffers.begin();
562 sp<AMessage> msg = new AMessage();
563 msg->setSize("buffer-ix", bufferIx);
564 err = fetchInputData(msg);
565 if (err != OK && err != ERROR_END_OF_STREAM) {
566 // if EOS, need to queue EOS buffer
567 break;
568 }
569 mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
570
571 if (!mPendingInputMessages.empty()
572 || !onInputBufferFetched(msg)) {
573 mPendingInputMessages.push_back(msg);
574 }
575 }
576
577 return err == -EWOULDBLOCK
578 && mSource->feedMoreTSData() == OK;
579 }
580
handleError(int32_t err)581 void NuPlayer::Decoder::handleError(int32_t err)
582 {
583 // We cannot immediately release the codec due to buffers still outstanding
584 // in the renderer. We signal to the player the error so it can shutdown/release the
585 // decoder after flushing and increment the generation to discard unnecessary messages.
586
587 ++mBufferGeneration;
588
589 sp<AMessage> notify = mNotify->dup();
590 notify->setInt32("what", kWhatError);
591 notify->setInt32("err", err);
592 notify->post();
593 }
594
releaseCrypto()595 status_t NuPlayer::Decoder::releaseCrypto()
596 {
597 ALOGV("releaseCrypto");
598
599 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
600
601 sp<AMessage> response;
602 status_t status = msg->postAndAwaitResponse(&response);
603 if (status == OK && response != NULL) {
604 CHECK(response->findInt32("status", &status));
605 ALOGV("releaseCrypto ret: %d ", status);
606 } else {
607 ALOGE("releaseCrypto err: %d", status);
608 }
609
610 return status;
611 }
612
onReleaseCrypto(const sp<AMessage> & msg)613 void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg)
614 {
615 status_t status = INVALID_OPERATION;
616 if (mCodec != NULL) {
617 status = mCodec->releaseCrypto();
618 } else {
619 // returning OK if the codec has been already released
620 status = OK;
621 ALOGE("onReleaseCrypto No mCodec. err: %d", status);
622 }
623
624 sp<AMessage> response = new AMessage;
625 response->setInt32("status", status);
626 // Clearing the state as it's tied to crypto. mIsEncryptedObservedEarlier is sticky though
627 // and lasts for the lifetime of this codec. See its use in fetchInputData.
628 mIsEncrypted = false;
629
630 sp<AReplyToken> replyID;
631 CHECK(msg->senderAwaitsResponse(&replyID));
632 response->postReply(replyID);
633 }
634
handleAnInputBuffer(size_t index)635 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
636 if (isDiscontinuityPending()) {
637 return false;
638 }
639
640 if (mCodec == NULL) {
641 ALOGE("[%s] handleAnInputBuffer without a valid codec", mComponentName.c_str());
642 handleError(NO_INIT);
643 return false;
644 }
645
646 sp<MediaCodecBuffer> buffer;
647 mCodec->getInputBuffer(index, &buffer);
648
649 if (buffer == NULL) {
650 ALOGE("[%s] handleAnInputBuffer, failed to get input buffer", mComponentName.c_str());
651 handleError(UNKNOWN_ERROR);
652 return false;
653 }
654
655 if (index >= mInputBuffers.size()) {
656 for (size_t i = mInputBuffers.size(); i <= index; ++i) {
657 mInputBuffers.add();
658 mMediaBuffers.add();
659 mInputBufferIsDequeued.add();
660 mMediaBuffers.editItemAt(i) = NULL;
661 mInputBufferIsDequeued.editItemAt(i) = false;
662 }
663 }
664 mInputBuffers.editItemAt(index) = buffer;
665
666 //CHECK_LT(bufferIx, mInputBuffers.size());
667
668 if (mMediaBuffers[index] != NULL) {
669 mMediaBuffers[index]->release();
670 mMediaBuffers.editItemAt(index) = NULL;
671 }
672 mInputBufferIsDequeued.editItemAt(index) = true;
673
674 if (!mCSDsToSubmit.isEmpty()) {
675 sp<AMessage> msg = new AMessage();
676 msg->setSize("buffer-ix", index);
677
678 sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
679 ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
680 msg->setBuffer("buffer", buffer);
681 mCSDsToSubmit.removeAt(0);
682 if (!onInputBufferFetched(msg)) {
683 handleError(UNKNOWN_ERROR);
684 return false;
685 }
686 return true;
687 }
688
689 while (!mPendingInputMessages.empty()) {
690 sp<AMessage> msg = *mPendingInputMessages.begin();
691 if (!onInputBufferFetched(msg)) {
692 break;
693 }
694 mPendingInputMessages.erase(mPendingInputMessages.begin());
695 }
696
697 if (!mInputBufferIsDequeued.editItemAt(index)) {
698 return true;
699 }
700
701 mDequeuedInputBuffers.push_back(index);
702
703 onRequestInputBuffers();
704 return true;
705 }
706
handleAnOutputBuffer(size_t index,size_t offset,size_t size,int64_t timeUs,int32_t flags)707 bool NuPlayer::Decoder::handleAnOutputBuffer(
708 size_t index,
709 size_t offset,
710 size_t size,
711 int64_t timeUs,
712 int32_t flags) {
713 if (mCodec == NULL) {
714 ALOGE("[%s] handleAnOutputBuffer without a valid codec", mComponentName.c_str());
715 handleError(NO_INIT);
716 return false;
717 }
718
719 // CHECK_LT(bufferIx, mOutputBuffers.size());
720 sp<MediaCodecBuffer> buffer;
721 mCodec->getOutputBuffer(index, &buffer);
722
723 if (buffer == NULL) {
724 ALOGE("[%s] handleAnOutputBuffer, failed to get output buffer", mComponentName.c_str());
725 handleError(UNKNOWN_ERROR);
726 return false;
727 }
728
729 if (index >= mOutputBuffers.size()) {
730 for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
731 mOutputBuffers.add();
732 }
733 }
734
735 mOutputBuffers.editItemAt(index) = buffer;
736
737 buffer->setRange(offset, size);
738 buffer->meta()->clear();
739 buffer->meta()->setInt64("timeUs", timeUs);
740
741 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
742 // we do not expect CODECCONFIG or SYNCFRAME for decoder
743
744 sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
745 reply->setSize("buffer-ix", index);
746 reply->setInt32("generation", mBufferGeneration);
747
748 if (eos) {
749 ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
750
751 buffer->meta()->setInt32("eos", true);
752 reply->setInt32("eos", true);
753 } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
754 if (timeUs < mSkipRenderingUntilMediaTimeUs) {
755 ALOGV("[%s] dropping buffer at time %lld as requested.",
756 mComponentName.c_str(), (long long)timeUs);
757
758 reply->post();
759 return true;
760 }
761
762 mSkipRenderingUntilMediaTimeUs = -1;
763 }
764
765 mNumFramesTotal += !mIsAudio;
766
767 // wait until 1st frame comes out to signal resume complete
768 notifyResumeCompleteIfNecessary();
769
770 if (mRenderer != NULL) {
771 // send the buffer to renderer.
772 mRenderer->queueBuffer(mIsAudio, buffer, reply);
773 if (eos && !isDiscontinuityPending()) {
774 mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
775 }
776 }
777
778 return true;
779 }
780
handleOutputFormatChange(const sp<AMessage> & format)781 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
782 if (!mIsAudio) {
783 int32_t width, height;
784 if (format->findInt32("width", &width)
785 && format->findInt32("height", &height)) {
786 mStats->setInt32("width", width);
787 mStats->setInt32("height", height);
788 }
789 sp<AMessage> notify = mNotify->dup();
790 notify->setInt32("what", kWhatVideoSizeChanged);
791 notify->setMessage("format", format);
792 notify->post();
793 } else if (mRenderer != NULL) {
794 uint32_t flags;
795 int64_t durationUs;
796 bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
797 if (getAudioDeepBufferSetting() // override regardless of source duration
798 || (mSource->getDuration(&durationUs) == OK
799 && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
800 flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
801 } else {
802 flags = AUDIO_OUTPUT_FLAG_NONE;
803 }
804
805 sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this);
806 reply->setInt32("generation", mBufferGeneration);
807 mRenderer->changeAudioFormat(
808 format, false /* offloadOnly */, hasVideo,
809 flags, mSource->isStreaming(), reply);
810 }
811 }
812
releaseAndResetMediaBuffers()813 void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
814 for (size_t i = 0; i < mMediaBuffers.size(); i++) {
815 if (mMediaBuffers[i] != NULL) {
816 mMediaBuffers[i]->release();
817 mMediaBuffers.editItemAt(i) = NULL;
818 }
819 }
820 mMediaBuffers.resize(mInputBuffers.size());
821 for (size_t i = 0; i < mMediaBuffers.size(); i++) {
822 mMediaBuffers.editItemAt(i) = NULL;
823 }
824 mInputBufferIsDequeued.clear();
825 mInputBufferIsDequeued.resize(mInputBuffers.size());
826 for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
827 mInputBufferIsDequeued.editItemAt(i) = false;
828 }
829
830 mPendingInputMessages.clear();
831 mDequeuedInputBuffers.clear();
832 mSkipRenderingUntilMediaTimeUs = -1;
833 }
834
requestCodecNotification()835 void NuPlayer::Decoder::requestCodecNotification() {
836 if (mCodec != NULL) {
837 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
838 reply->setInt32("generation", mBufferGeneration);
839 mCodec->requestActivityNotification(reply);
840 }
841 }
842
isStaleReply(const sp<AMessage> & msg)843 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
844 int32_t generation;
845 CHECK(msg->findInt32("generation", &generation));
846 return generation != mBufferGeneration;
847 }
848
fetchInputData(sp<AMessage> & reply)849 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
850 sp<ABuffer> accessUnit;
851 bool dropAccessUnit = true;
852 do {
853 status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
854
855 if (err == -EWOULDBLOCK) {
856 return err;
857 } else if (err != OK) {
858 if (err == INFO_DISCONTINUITY) {
859 int32_t type;
860 CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
861
862 bool formatChange =
863 (mIsAudio &&
864 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
865 || (!mIsAudio &&
866 (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
867
868 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
869
870 ALOGI("%s discontinuity (format=%d, time=%d)",
871 mIsAudio ? "audio" : "video", formatChange, timeChange);
872
873 bool seamlessFormatChange = false;
874 sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
875 if (formatChange) {
876 seamlessFormatChange =
877 supportsSeamlessFormatChange(newFormat);
878 // treat seamless format change separately
879 formatChange = !seamlessFormatChange;
880 }
881
882 // For format or time change, return EOS to queue EOS input,
883 // then wait for EOS on output.
884 if (formatChange /* not seamless */) {
885 mFormatChangePending = true;
886 err = ERROR_END_OF_STREAM;
887 } else if (timeChange) {
888 rememberCodecSpecificData(newFormat);
889 mTimeChangePending = true;
890 err = ERROR_END_OF_STREAM;
891 } else if (seamlessFormatChange) {
892 // reuse existing decoder and don't flush
893 rememberCodecSpecificData(newFormat);
894 continue;
895 } else {
896 // This stream is unaffected by the discontinuity
897 return -EWOULDBLOCK;
898 }
899 }
900
901 // reply should only be returned without a buffer set
902 // when there is an error (including EOS)
903 CHECK(err != OK);
904
905 reply->setInt32("err", err);
906 return ERROR_END_OF_STREAM;
907 }
908
909 dropAccessUnit = false;
910 if (!mIsAudio && !mIsEncrypted) {
911 // Extra safeguard if higher-level behavior changes. Otherwise, not required now.
912 // Preventing the buffer from being processed (and sent to codec) if this is a later
913 // round of playback but this time without prepareDrm. Or if there is a race between
914 // stop (which is not blocking) and releaseDrm allowing buffers being processed after
915 // Crypto has been released (GenericSource currently prevents this race though).
916 // Particularly doing this check before IsAVCReferenceFrame call to prevent parsing
917 // of encrypted data.
918 if (mIsEncryptedObservedEarlier) {
919 ALOGE("fetchInputData: mismatched mIsEncrypted/mIsEncryptedObservedEarlier (0/1)");
920
921 return INVALID_OPERATION;
922 }
923
924 int32_t layerId = 0;
925 bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
926 if (mRenderer->getVideoLateByUs() > 100000ll
927 && mIsVideoAVC
928 && !IsAVCReferenceFrame(accessUnit)) {
929 dropAccessUnit = true;
930 } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
931 // Add only one layer each time.
932 if (layerId > mCurrentMaxVideoTemporalLayerId + 1
933 || layerId >= mNumVideoTemporalLayerAllowed) {
934 dropAccessUnit = true;
935 ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
936 layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
937 mCurrentMaxVideoTemporalLayerId);
938 } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
939 mCurrentMaxVideoTemporalLayerId = layerId;
940 } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
941 mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
942 }
943 }
944 if (dropAccessUnit) {
945 if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
946 mCurrentMaxVideoTemporalLayerId = layerId - 1;
947 }
948 ++mNumInputFramesDropped;
949 }
950 }
951 } while (dropAccessUnit);
952
953 // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
954 #if 0
955 int64_t mediaTimeUs;
956 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
957 ALOGV("[%s] feeding input buffer at media time %.3f",
958 mIsAudio ? "audio" : "video",
959 mediaTimeUs / 1E6);
960 #endif
961
962 if (mCCDecoder != NULL) {
963 mCCDecoder->decode(accessUnit);
964 }
965
966 reply->setBuffer("buffer", accessUnit);
967
968 return OK;
969 }
970
onInputBufferFetched(const sp<AMessage> & msg)971 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
972 if (mCodec == NULL) {
973 ALOGE("[%s] onInputBufferFetched without a valid codec", mComponentName.c_str());
974 handleError(NO_INIT);
975 return false;
976 }
977
978 size_t bufferIx;
979 CHECK(msg->findSize("buffer-ix", &bufferIx));
980 CHECK_LT(bufferIx, mInputBuffers.size());
981 sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx];
982
983 sp<ABuffer> buffer;
984 bool hasBuffer = msg->findBuffer("buffer", &buffer);
985 bool needsCopy = true;
986
987 if (buffer == NULL /* includes !hasBuffer */) {
988 int32_t streamErr = ERROR_END_OF_STREAM;
989 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
990
991 CHECK(streamErr != OK);
992
993 // attempt to queue EOS
994 status_t err = mCodec->queueInputBuffer(
995 bufferIx,
996 0,
997 0,
998 0,
999 MediaCodec::BUFFER_FLAG_EOS);
1000 if (err == OK) {
1001 mInputBufferIsDequeued.editItemAt(bufferIx) = false;
1002 } else if (streamErr == ERROR_END_OF_STREAM) {
1003 streamErr = err;
1004 // err will not be ERROR_END_OF_STREAM
1005 }
1006
1007 if (streamErr != ERROR_END_OF_STREAM) {
1008 ALOGE("Stream error for [%s] (err=%d), EOS %s queued",
1009 mComponentName.c_str(),
1010 streamErr,
1011 err == OK ? "successfully" : "unsuccessfully");
1012 handleError(streamErr);
1013 }
1014 } else {
1015 sp<AMessage> extra;
1016 if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
1017 int64_t resumeAtMediaTimeUs;
1018 if (extra->findInt64(
1019 "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
1020 ALOGI("[%s] suppressing rendering until %lld us",
1021 mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
1022 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
1023 }
1024 }
1025
1026 int64_t timeUs = 0;
1027 uint32_t flags = 0;
1028 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1029
1030 int32_t eos, csd;
1031 // we do not expect SYNCFRAME for decoder
1032 if (buffer->meta()->findInt32("eos", &eos) && eos) {
1033 flags |= MediaCodec::BUFFER_FLAG_EOS;
1034 } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
1035 flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
1036 }
1037
1038 // Modular DRM
1039 MediaBuffer *mediaBuf = NULL;
1040 NuPlayerDrm::CryptoInfo *cryptInfo = NULL;
1041
1042 // copy into codec buffer
1043 if (needsCopy) {
1044 if (buffer->size() > codecBuffer->capacity()) {
1045 handleError(ERROR_BUFFER_TOO_SMALL);
1046 mDequeuedInputBuffers.push_back(bufferIx);
1047 return false;
1048 }
1049
1050 if (buffer->data() != NULL) {
1051 codecBuffer->setRange(0, buffer->size());
1052 memcpy(codecBuffer->data(), buffer->data(), buffer->size());
1053 } else { // No buffer->data()
1054 //Modular DRM
1055 mediaBuf = (MediaBuffer*)buffer->getMediaBufferBase();
1056 if (mediaBuf != NULL) {
1057 codecBuffer->setRange(0, mediaBuf->size());
1058 memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size());
1059
1060 sp<MetaData> meta_data = mediaBuf->meta_data();
1061 cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data);
1062
1063 // since getMediaBuffer() has incremented the refCount
1064 mediaBuf->release();
1065 } else { // No mediaBuf
1066 ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p",
1067 buffer.get());
1068 handleError(UNKNOWN_ERROR);
1069 return false;
1070 }
1071 } // buffer->data()
1072 } // needsCopy
1073
1074 status_t err;
1075 AString errorDetailMsg;
1076 if (cryptInfo != NULL) {
1077 err = mCodec->queueSecureInputBuffer(
1078 bufferIx,
1079 codecBuffer->offset(),
1080 cryptInfo->subSamples,
1081 cryptInfo->numSubSamples,
1082 cryptInfo->key,
1083 cryptInfo->iv,
1084 cryptInfo->mode,
1085 cryptInfo->pattern,
1086 timeUs,
1087 flags,
1088 &errorDetailMsg);
1089 // synchronous call so done with cryptInfo here
1090 free(cryptInfo);
1091 } else {
1092 err = mCodec->queueInputBuffer(
1093 bufferIx,
1094 codecBuffer->offset(),
1095 codecBuffer->size(),
1096 timeUs,
1097 flags,
1098 &errorDetailMsg);
1099 } // no cryptInfo
1100
1101 if (err != OK) {
1102 ALOGE("onInputBufferFetched: queue%sInputBuffer failed for [%s] (err=%d, %s)",
1103 (cryptInfo != NULL ? "Secure" : ""),
1104 mComponentName.c_str(), err, errorDetailMsg.c_str());
1105 handleError(err);
1106 } else {
1107 mInputBufferIsDequeued.editItemAt(bufferIx) = false;
1108 }
1109
1110 } // buffer != NULL
1111 return true;
1112 }
1113
onRenderBuffer(const sp<AMessage> & msg)1114 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
1115 status_t err;
1116 int32_t render;
1117 size_t bufferIx;
1118 int32_t eos;
1119 CHECK(msg->findSize("buffer-ix", &bufferIx));
1120
1121 if (!mIsAudio) {
1122 int64_t timeUs;
1123 sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
1124 buffer->meta()->findInt64("timeUs", &timeUs);
1125
1126 if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
1127 mCCDecoder->display(timeUs);
1128 }
1129 }
1130
1131 if (mCodec == NULL) {
1132 err = NO_INIT;
1133 } else if (msg->findInt32("render", &render) && render) {
1134 int64_t timestampNs;
1135 CHECK(msg->findInt64("timestampNs", ×tampNs));
1136 err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
1137 } else {
1138 mNumOutputFramesDropped += !mIsAudio;
1139 err = mCodec->releaseOutputBuffer(bufferIx);
1140 }
1141 if (err != OK) {
1142 ALOGE("failed to release output buffer for [%s] (err=%d)",
1143 mComponentName.c_str(), err);
1144 handleError(err);
1145 }
1146 if (msg->findInt32("eos", &eos) && eos
1147 && isDiscontinuityPending()) {
1148 finishHandleDiscontinuity(true /* flushOnTimeChange */);
1149 }
1150 }
1151
isDiscontinuityPending() const1152 bool NuPlayer::Decoder::isDiscontinuityPending() const {
1153 return mFormatChangePending || mTimeChangePending;
1154 }
1155
finishHandleDiscontinuity(bool flushOnTimeChange)1156 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
1157 ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
1158 mFormatChangePending, mTimeChangePending, flushOnTimeChange);
1159
1160 // If we have format change, pause and wait to be killed;
1161 // If we have time change only, flush and restart fetching.
1162
1163 if (mFormatChangePending) {
1164 mPaused = true;
1165 } else if (mTimeChangePending) {
1166 if (flushOnTimeChange) {
1167 doFlush(false /* notifyComplete */);
1168 signalResume(false /* notifyComplete */);
1169 }
1170 }
1171
1172 // Notify NuPlayer to either shutdown decoder, or rescan sources
1173 sp<AMessage> msg = mNotify->dup();
1174 msg->setInt32("what", kWhatInputDiscontinuity);
1175 msg->setInt32("formatChange", mFormatChangePending);
1176 msg->post();
1177
1178 mFormatChangePending = false;
1179 mTimeChangePending = false;
1180 }
1181
supportsSeamlessAudioFormatChange(const sp<AMessage> & targetFormat) const1182 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
1183 const sp<AMessage> &targetFormat) const {
1184 if (targetFormat == NULL) {
1185 return true;
1186 }
1187
1188 AString mime;
1189 if (!targetFormat->findString("mime", &mime)) {
1190 return false;
1191 }
1192
1193 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
1194 // field-by-field comparison
1195 const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
1196 for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
1197 int32_t oldVal, newVal;
1198 if (!mInputFormat->findInt32(keys[i], &oldVal) ||
1199 !targetFormat->findInt32(keys[i], &newVal) ||
1200 oldVal != newVal) {
1201 return false;
1202 }
1203 }
1204
1205 sp<ABuffer> oldBuf, newBuf;
1206 if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
1207 targetFormat->findBuffer("csd-0", &newBuf)) {
1208 if (oldBuf->size() != newBuf->size()) {
1209 return false;
1210 }
1211 return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
1212 }
1213 }
1214 return false;
1215 }
1216
supportsSeamlessFormatChange(const sp<AMessage> & targetFormat) const1217 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
1218 if (mInputFormat == NULL) {
1219 return false;
1220 }
1221
1222 if (targetFormat == NULL) {
1223 return true;
1224 }
1225
1226 AString oldMime, newMime;
1227 if (!mInputFormat->findString("mime", &oldMime)
1228 || !targetFormat->findString("mime", &newMime)
1229 || !(oldMime == newMime)) {
1230 return false;
1231 }
1232
1233 bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1234 bool seamless;
1235 if (audio) {
1236 seamless = supportsSeamlessAudioFormatChange(targetFormat);
1237 } else {
1238 int32_t isAdaptive;
1239 seamless = (mCodec != NULL &&
1240 mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1241 isAdaptive);
1242 }
1243
1244 ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1245 return seamless;
1246 }
1247
rememberCodecSpecificData(const sp<AMessage> & format)1248 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1249 if (format == NULL) {
1250 return;
1251 }
1252 mCSDsForCurrentFormat.clear();
1253 for (int32_t i = 0; ; ++i) {
1254 AString tag = "csd-";
1255 tag.append(i);
1256 sp<ABuffer> buffer;
1257 if (!format->findBuffer(tag.c_str(), &buffer)) {
1258 break;
1259 }
1260 mCSDsForCurrentFormat.push(buffer);
1261 }
1262 }
1263
notifyResumeCompleteIfNecessary()1264 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1265 if (mResumePending) {
1266 mResumePending = false;
1267
1268 sp<AMessage> notify = mNotify->dup();
1269 notify->setInt32("what", kWhatResumeCompleted);
1270 notify->post();
1271 }
1272 }
1273
1274 } // namespace android
1275
1276