1 /*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayerDecoder"
19 #include <utils/Log.h>
20 #include <inttypes.h>
21
22 #include "NuPlayerCCDecoder.h"
23 #include "NuPlayerDecoder.h"
24 #include "NuPlayerRenderer.h"
25 #include "NuPlayerSource.h"
26
27 #include <cutils/properties.h>
28 #include <media/ICrypto.h>
29 #include <media/stagefright/foundation/ABuffer.h>
30 #include <media/stagefright/foundation/ADebug.h>
31 #include <media/stagefright/foundation/AMessage.h>
32 #include <media/stagefright/MediaBuffer.h>
33 #include <media/stagefright/MediaCodec.h>
34 #include <media/stagefright/MediaDefs.h>
35 #include <media/stagefright/MediaErrors.h>
36
37 #include <gui/Surface.h>
38
39 #include "avc_utils.h"
40 #include "ATSParser.h"
41
42 namespace android {
43
getAudioDeepBufferSetting()44 static inline bool getAudioDeepBufferSetting() {
45 return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
46 }
47
Decoder(const sp<AMessage> & notify,const sp<Source> & source,pid_t pid,const sp<Renderer> & renderer,const sp<Surface> & surface,const sp<CCDecoder> & ccDecoder)48 NuPlayer::Decoder::Decoder(
49 const sp<AMessage> ¬ify,
50 const sp<Source> &source,
51 pid_t pid,
52 const sp<Renderer> &renderer,
53 const sp<Surface> &surface,
54 const sp<CCDecoder> &ccDecoder)
55 : DecoderBase(notify),
56 mSurface(surface),
57 mSource(source),
58 mRenderer(renderer),
59 mCCDecoder(ccDecoder),
60 mPid(pid),
61 mSkipRenderingUntilMediaTimeUs(-1ll),
62 mNumFramesTotal(0ll),
63 mNumInputFramesDropped(0ll),
64 mNumOutputFramesDropped(0ll),
65 mVideoWidth(0),
66 mVideoHeight(0),
67 mIsAudio(true),
68 mIsVideoAVC(false),
69 mIsSecure(false),
70 mFormatChangePending(false),
71 mTimeChangePending(false),
72 mPaused(true),
73 mResumePending(false),
74 mComponentName("decoder") {
75 mCodecLooper = new ALooper;
76 mCodecLooper->setName("NPDecoder-CL");
77 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
78 }
79
~Decoder()80 NuPlayer::Decoder::~Decoder() {
81 mCodec->release();
82 releaseAndResetMediaBuffers();
83 }
84
getStats() const85 sp<AMessage> NuPlayer::Decoder::getStats() const {
86 mStats->setInt64("frames-total", mNumFramesTotal);
87 mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
88 mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
89 return mStats;
90 }
91
setVideoSurface(const sp<Surface> & surface)92 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
93 if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
94 return BAD_VALUE;
95 }
96
97 sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
98
99 msg->setObject("surface", surface);
100 sp<AMessage> response;
101 status_t err = msg->postAndAwaitResponse(&response);
102 if (err == OK && response != NULL) {
103 CHECK(response->findInt32("err", &err));
104 }
105 return err;
106 }
107
onMessageReceived(const sp<AMessage> & msg)108 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
109 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
110
111 switch (msg->what()) {
112 case kWhatCodecNotify:
113 {
114 int32_t cbID;
115 CHECK(msg->findInt32("callbackID", &cbID));
116
117 ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
118 mIsAudio ? "audio" : "video", cbID, mPaused);
119
120 if (mPaused) {
121 break;
122 }
123
124 switch (cbID) {
125 case MediaCodec::CB_INPUT_AVAILABLE:
126 {
127 int32_t index;
128 CHECK(msg->findInt32("index", &index));
129
130 handleAnInputBuffer(index);
131 break;
132 }
133
134 case MediaCodec::CB_OUTPUT_AVAILABLE:
135 {
136 int32_t index;
137 size_t offset;
138 size_t size;
139 int64_t timeUs;
140 int32_t flags;
141
142 CHECK(msg->findInt32("index", &index));
143 CHECK(msg->findSize("offset", &offset));
144 CHECK(msg->findSize("size", &size));
145 CHECK(msg->findInt64("timeUs", &timeUs));
146 CHECK(msg->findInt32("flags", &flags));
147
148 handleAnOutputBuffer(index, offset, size, timeUs, flags);
149 break;
150 }
151
152 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
153 {
154 sp<AMessage> format;
155 CHECK(msg->findMessage("format", &format));
156
157 handleOutputFormatChange(format);
158 break;
159 }
160
161 case MediaCodec::CB_ERROR:
162 {
163 status_t err;
164 CHECK(msg->findInt32("err", &err));
165 ALOGE("Decoder (%s) reported error : 0x%x",
166 mIsAudio ? "audio" : "video", err);
167
168 handleError(err);
169 break;
170 }
171
172 default:
173 {
174 TRESPASS();
175 break;
176 }
177 }
178
179 break;
180 }
181
182 case kWhatRenderBuffer:
183 {
184 if (!isStaleReply(msg)) {
185 onRenderBuffer(msg);
186 }
187 break;
188 }
189
190 case kWhatSetVideoSurface:
191 {
192 sp<AReplyToken> replyID;
193 CHECK(msg->senderAwaitsResponse(&replyID));
194
195 sp<RefBase> obj;
196 CHECK(msg->findObject("surface", &obj));
197 sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
198 int32_t err = INVALID_OPERATION;
199 // NOTE: in practice mSurface is always non-null, but checking here for completeness
200 if (mCodec != NULL && mSurface != NULL) {
201 // TODO: once AwesomePlayer is removed, remove this automatic connecting
202 // to the surface by MediaPlayerService.
203 //
204 // at this point MediaPlayerService::client has already connected to the
205 // surface, which MediaCodec does not expect
206 err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
207 if (err == OK) {
208 err = mCodec->setSurface(surface);
209 ALOGI_IF(err, "codec setSurface returned: %d", err);
210 if (err == OK) {
211 // reconnect to the old surface as MPS::Client will expect to
212 // be able to disconnect from it.
213 (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
214 mSurface = surface;
215 }
216 }
217 if (err != OK) {
218 // reconnect to the new surface on error as MPS::Client will expect to
219 // be able to disconnect from it.
220 (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
221 }
222 }
223
224 sp<AMessage> response = new AMessage;
225 response->setInt32("err", err);
226 response->postReply(replyID);
227 break;
228 }
229
230 default:
231 DecoderBase::onMessageReceived(msg);
232 break;
233 }
234 }
235
onConfigure(const sp<AMessage> & format)236 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
237 CHECK(mCodec == NULL);
238
239 mFormatChangePending = false;
240 mTimeChangePending = false;
241
242 ++mBufferGeneration;
243
244 AString mime;
245 CHECK(format->findString("mime", &mime));
246
247 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
248 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
249
250 mComponentName = mime;
251 mComponentName.append(" decoder");
252 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
253
254 mCodec = MediaCodec::CreateByType(
255 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid);
256 int32_t secure = 0;
257 if (format->findInt32("secure", &secure) && secure != 0) {
258 if (mCodec != NULL) {
259 mCodec->getName(&mComponentName);
260 mComponentName.append(".secure");
261 mCodec->release();
262 ALOGI("[%s] creating", mComponentName.c_str());
263 mCodec = MediaCodec::CreateByComponentName(
264 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid);
265 }
266 }
267 if (mCodec == NULL) {
268 ALOGE("Failed to create %s%s decoder",
269 (secure ? "secure " : ""), mime.c_str());
270 handleError(UNKNOWN_ERROR);
271 return;
272 }
273 mIsSecure = secure;
274
275 mCodec->getName(&mComponentName);
276
277 status_t err;
278 if (mSurface != NULL) {
279 // disconnect from surface as MediaCodec will reconnect
280 err = native_window_api_disconnect(
281 mSurface.get(), NATIVE_WINDOW_API_MEDIA);
282 // We treat this as a warning, as this is a preparatory step.
283 // Codec will try to connect to the surface, which is where
284 // any error signaling will occur.
285 ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
286 }
287 err = mCodec->configure(
288 format, mSurface, NULL /* crypto */, 0 /* flags */);
289 if (err != OK) {
290 ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
291 mCodec->release();
292 mCodec.clear();
293 handleError(err);
294 return;
295 }
296 rememberCodecSpecificData(format);
297
298 // the following should work in configured state
299 CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
300 CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
301
302 mStats->setString("mime", mime.c_str());
303 mStats->setString("component-name", mComponentName.c_str());
304
305 if (!mIsAudio) {
306 int32_t width, height;
307 if (mOutputFormat->findInt32("width", &width)
308 && mOutputFormat->findInt32("height", &height)) {
309 mStats->setInt32("width", width);
310 mStats->setInt32("height", height);
311 }
312 }
313
314 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
315 mCodec->setCallback(reply);
316
317 err = mCodec->start();
318 if (err != OK) {
319 ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
320 mCodec->release();
321 mCodec.clear();
322 handleError(err);
323 return;
324 }
325
326 releaseAndResetMediaBuffers();
327
328 mPaused = false;
329 mResumePending = false;
330 }
331
onSetParameters(const sp<AMessage> & params)332 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> ¶ms) {
333 if (mCodec == NULL) {
334 ALOGW("onSetParameters called before codec is created.");
335 return;
336 }
337 mCodec->setParameters(params);
338 }
339
onSetRenderer(const sp<Renderer> & renderer)340 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
341 bool hadNoRenderer = (mRenderer == NULL);
342 mRenderer = renderer;
343 if (hadNoRenderer && mRenderer != NULL) {
344 // this means that the widevine legacy source is ready
345 onRequestInputBuffers();
346 }
347 }
348
onGetInputBuffers(Vector<sp<ABuffer>> * dstBuffers)349 void NuPlayer::Decoder::onGetInputBuffers(
350 Vector<sp<ABuffer> > *dstBuffers) {
351 CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers));
352 }
353
onResume(bool notifyComplete)354 void NuPlayer::Decoder::onResume(bool notifyComplete) {
355 mPaused = false;
356
357 if (notifyComplete) {
358 mResumePending = true;
359 }
360 mCodec->start();
361 }
362
doFlush(bool notifyComplete)363 void NuPlayer::Decoder::doFlush(bool notifyComplete) {
364 if (mCCDecoder != NULL) {
365 mCCDecoder->flush();
366 }
367
368 if (mRenderer != NULL) {
369 mRenderer->flush(mIsAudio, notifyComplete);
370 mRenderer->signalTimeDiscontinuity();
371 }
372
373 status_t err = OK;
374 if (mCodec != NULL) {
375 err = mCodec->flush();
376 mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
377 ++mBufferGeneration;
378 }
379
380 if (err != OK) {
381 ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
382 handleError(err);
383 // finish with posting kWhatFlushCompleted.
384 // we attempt to release the buffers even if flush fails.
385 }
386 releaseAndResetMediaBuffers();
387 mPaused = true;
388 }
389
390
onFlush()391 void NuPlayer::Decoder::onFlush() {
392 doFlush(true);
393
394 if (isDiscontinuityPending()) {
395 // This could happen if the client starts seeking/shutdown
396 // after we queued an EOS for discontinuities.
397 // We can consider discontinuity handled.
398 finishHandleDiscontinuity(false /* flushOnTimeChange */);
399 }
400
401 sp<AMessage> notify = mNotify->dup();
402 notify->setInt32("what", kWhatFlushCompleted);
403 notify->post();
404 }
405
onShutdown(bool notifyComplete)406 void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
407 status_t err = OK;
408
409 // if there is a pending resume request, notify complete now
410 notifyResumeCompleteIfNecessary();
411
412 if (mCodec != NULL) {
413 err = mCodec->release();
414 mCodec = NULL;
415 ++mBufferGeneration;
416
417 if (mSurface != NULL) {
418 // reconnect to surface as MediaCodec disconnected from it
419 status_t error =
420 native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
421 ALOGW_IF(error != NO_ERROR,
422 "[%s] failed to connect to native window, error=%d",
423 mComponentName.c_str(), error);
424 }
425 mComponentName = "decoder";
426 }
427
428 releaseAndResetMediaBuffers();
429
430 if (err != OK) {
431 ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
432 handleError(err);
433 // finish with posting kWhatShutdownCompleted.
434 }
435
436 if (notifyComplete) {
437 sp<AMessage> notify = mNotify->dup();
438 notify->setInt32("what", kWhatShutdownCompleted);
439 notify->post();
440 mPaused = true;
441 }
442 }
443
444 /*
445 * returns true if we should request more data
446 */
doRequestBuffers()447 bool NuPlayer::Decoder::doRequestBuffers() {
448 // mRenderer is only NULL if we have a legacy widevine source that
449 // is not yet ready. In this case we must not fetch input.
450 if (isDiscontinuityPending() || mRenderer == NULL) {
451 return false;
452 }
453 status_t err = OK;
454 while (err == OK && !mDequeuedInputBuffers.empty()) {
455 size_t bufferIx = *mDequeuedInputBuffers.begin();
456 sp<AMessage> msg = new AMessage();
457 msg->setSize("buffer-ix", bufferIx);
458 err = fetchInputData(msg);
459 if (err != OK && err != ERROR_END_OF_STREAM) {
460 // if EOS, need to queue EOS buffer
461 break;
462 }
463 mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
464
465 if (!mPendingInputMessages.empty()
466 || !onInputBufferFetched(msg)) {
467 mPendingInputMessages.push_back(msg);
468 }
469 }
470
471 return err == -EWOULDBLOCK
472 && mSource->feedMoreTSData() == OK;
473 }
474
handleError(int32_t err)475 void NuPlayer::Decoder::handleError(int32_t err)
476 {
477 // We cannot immediately release the codec due to buffers still outstanding
478 // in the renderer. We signal to the player the error so it can shutdown/release the
479 // decoder after flushing and increment the generation to discard unnecessary messages.
480
481 ++mBufferGeneration;
482
483 sp<AMessage> notify = mNotify->dup();
484 notify->setInt32("what", kWhatError);
485 notify->setInt32("err", err);
486 notify->post();
487 }
488
handleAnInputBuffer(size_t index)489 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
490 if (isDiscontinuityPending()) {
491 return false;
492 }
493
494 sp<ABuffer> buffer;
495 mCodec->getInputBuffer(index, &buffer);
496
497 if (buffer == NULL) {
498 handleError(UNKNOWN_ERROR);
499 return false;
500 }
501
502 if (index >= mInputBuffers.size()) {
503 for (size_t i = mInputBuffers.size(); i <= index; ++i) {
504 mInputBuffers.add();
505 mMediaBuffers.add();
506 mInputBufferIsDequeued.add();
507 mMediaBuffers.editItemAt(i) = NULL;
508 mInputBufferIsDequeued.editItemAt(i) = false;
509 }
510 }
511 mInputBuffers.editItemAt(index) = buffer;
512
513 //CHECK_LT(bufferIx, mInputBuffers.size());
514
515 if (mMediaBuffers[index] != NULL) {
516 mMediaBuffers[index]->release();
517 mMediaBuffers.editItemAt(index) = NULL;
518 }
519 mInputBufferIsDequeued.editItemAt(index) = true;
520
521 if (!mCSDsToSubmit.isEmpty()) {
522 sp<AMessage> msg = new AMessage();
523 msg->setSize("buffer-ix", index);
524
525 sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
526 ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
527 msg->setBuffer("buffer", buffer);
528 mCSDsToSubmit.removeAt(0);
529 CHECK(onInputBufferFetched(msg));
530 return true;
531 }
532
533 while (!mPendingInputMessages.empty()) {
534 sp<AMessage> msg = *mPendingInputMessages.begin();
535 if (!onInputBufferFetched(msg)) {
536 break;
537 }
538 mPendingInputMessages.erase(mPendingInputMessages.begin());
539 }
540
541 if (!mInputBufferIsDequeued.editItemAt(index)) {
542 return true;
543 }
544
545 mDequeuedInputBuffers.push_back(index);
546
547 onRequestInputBuffers();
548 return true;
549 }
550
handleAnOutputBuffer(size_t index,size_t offset,size_t size,int64_t timeUs,int32_t flags)551 bool NuPlayer::Decoder::handleAnOutputBuffer(
552 size_t index,
553 size_t offset,
554 size_t size,
555 int64_t timeUs,
556 int32_t flags) {
557 // CHECK_LT(bufferIx, mOutputBuffers.size());
558 sp<ABuffer> buffer;
559 mCodec->getOutputBuffer(index, &buffer);
560
561 if (index >= mOutputBuffers.size()) {
562 for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
563 mOutputBuffers.add();
564 }
565 }
566
567 mOutputBuffers.editItemAt(index) = buffer;
568
569 buffer->setRange(offset, size);
570 buffer->meta()->clear();
571 buffer->meta()->setInt64("timeUs", timeUs);
572
573 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
574 // we do not expect CODECCONFIG or SYNCFRAME for decoder
575
576 sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
577 reply->setSize("buffer-ix", index);
578 reply->setInt32("generation", mBufferGeneration);
579
580 if (eos) {
581 ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
582
583 buffer->meta()->setInt32("eos", true);
584 reply->setInt32("eos", true);
585 } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
586 if (timeUs < mSkipRenderingUntilMediaTimeUs) {
587 ALOGV("[%s] dropping buffer at time %lld as requested.",
588 mComponentName.c_str(), (long long)timeUs);
589
590 reply->post();
591 return true;
592 }
593
594 mSkipRenderingUntilMediaTimeUs = -1;
595 }
596
597 mNumFramesTotal += !mIsAudio;
598
599 // wait until 1st frame comes out to signal resume complete
600 notifyResumeCompleteIfNecessary();
601
602 if (mRenderer != NULL) {
603 // send the buffer to renderer.
604 mRenderer->queueBuffer(mIsAudio, buffer, reply);
605 if (eos && !isDiscontinuityPending()) {
606 mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
607 }
608 }
609
610 return true;
611 }
612
handleOutputFormatChange(const sp<AMessage> & format)613 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
614 if (!mIsAudio) {
615 int32_t width, height;
616 if (format->findInt32("width", &width)
617 && format->findInt32("height", &height)) {
618 mStats->setInt32("width", width);
619 mStats->setInt32("height", height);
620 }
621 sp<AMessage> notify = mNotify->dup();
622 notify->setInt32("what", kWhatVideoSizeChanged);
623 notify->setMessage("format", format);
624 notify->post();
625 } else if (mRenderer != NULL) {
626 uint32_t flags;
627 int64_t durationUs;
628 bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
629 if (getAudioDeepBufferSetting() // override regardless of source duration
630 || (!hasVideo
631 && mSource->getDuration(&durationUs) == OK
632 && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
633 flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
634 } else {
635 flags = AUDIO_OUTPUT_FLAG_NONE;
636 }
637
638 status_t err = mRenderer->openAudioSink(
639 format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */);
640 if (err != OK) {
641 handleError(err);
642 }
643 }
644 }
645
releaseAndResetMediaBuffers()646 void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
647 for (size_t i = 0; i < mMediaBuffers.size(); i++) {
648 if (mMediaBuffers[i] != NULL) {
649 mMediaBuffers[i]->release();
650 mMediaBuffers.editItemAt(i) = NULL;
651 }
652 }
653 mMediaBuffers.resize(mInputBuffers.size());
654 for (size_t i = 0; i < mMediaBuffers.size(); i++) {
655 mMediaBuffers.editItemAt(i) = NULL;
656 }
657 mInputBufferIsDequeued.clear();
658 mInputBufferIsDequeued.resize(mInputBuffers.size());
659 for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
660 mInputBufferIsDequeued.editItemAt(i) = false;
661 }
662
663 mPendingInputMessages.clear();
664 mDequeuedInputBuffers.clear();
665 mSkipRenderingUntilMediaTimeUs = -1;
666 }
667
requestCodecNotification()668 void NuPlayer::Decoder::requestCodecNotification() {
669 if (mCodec != NULL) {
670 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
671 reply->setInt32("generation", mBufferGeneration);
672 mCodec->requestActivityNotification(reply);
673 }
674 }
675
isStaleReply(const sp<AMessage> & msg)676 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
677 int32_t generation;
678 CHECK(msg->findInt32("generation", &generation));
679 return generation != mBufferGeneration;
680 }
681
fetchInputData(sp<AMessage> & reply)682 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
683 sp<ABuffer> accessUnit;
684 bool dropAccessUnit;
685 do {
686 status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
687
688 if (err == -EWOULDBLOCK) {
689 return err;
690 } else if (err != OK) {
691 if (err == INFO_DISCONTINUITY) {
692 int32_t type;
693 CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
694
695 bool formatChange =
696 (mIsAudio &&
697 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
698 || (!mIsAudio &&
699 (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
700
701 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
702
703 ALOGI("%s discontinuity (format=%d, time=%d)",
704 mIsAudio ? "audio" : "video", formatChange, timeChange);
705
706 bool seamlessFormatChange = false;
707 sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
708 if (formatChange) {
709 seamlessFormatChange =
710 supportsSeamlessFormatChange(newFormat);
711 // treat seamless format change separately
712 formatChange = !seamlessFormatChange;
713 }
714
715 // For format or time change, return EOS to queue EOS input,
716 // then wait for EOS on output.
717 if (formatChange /* not seamless */) {
718 mFormatChangePending = true;
719 err = ERROR_END_OF_STREAM;
720 } else if (timeChange) {
721 rememberCodecSpecificData(newFormat);
722 mTimeChangePending = true;
723 err = ERROR_END_OF_STREAM;
724 } else if (seamlessFormatChange) {
725 // reuse existing decoder and don't flush
726 rememberCodecSpecificData(newFormat);
727 continue;
728 } else {
729 // This stream is unaffected by the discontinuity
730 return -EWOULDBLOCK;
731 }
732 }
733
734 // reply should only be returned without a buffer set
735 // when there is an error (including EOS)
736 CHECK(err != OK);
737
738 reply->setInt32("err", err);
739 return ERROR_END_OF_STREAM;
740 }
741
742 dropAccessUnit = false;
743 if (!mIsAudio
744 && !mIsSecure
745 && mRenderer->getVideoLateByUs() > 100000ll
746 && mIsVideoAVC
747 && !IsAVCReferenceFrame(accessUnit)) {
748 dropAccessUnit = true;
749 ++mNumInputFramesDropped;
750 }
751 } while (dropAccessUnit);
752
753 // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
754 #if 0
755 int64_t mediaTimeUs;
756 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
757 ALOGV("[%s] feeding input buffer at media time %.3f",
758 mIsAudio ? "audio" : "video",
759 mediaTimeUs / 1E6);
760 #endif
761
762 if (mCCDecoder != NULL) {
763 mCCDecoder->decode(accessUnit);
764 }
765
766 reply->setBuffer("buffer", accessUnit);
767
768 return OK;
769 }
770
onInputBufferFetched(const sp<AMessage> & msg)771 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
772 size_t bufferIx;
773 CHECK(msg->findSize("buffer-ix", &bufferIx));
774 CHECK_LT(bufferIx, mInputBuffers.size());
775 sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
776
777 sp<ABuffer> buffer;
778 bool hasBuffer = msg->findBuffer("buffer", &buffer);
779
780 // handle widevine classic source - that fills an arbitrary input buffer
781 MediaBuffer *mediaBuffer = NULL;
782 if (hasBuffer) {
783 mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
784 if (mediaBuffer != NULL) {
785 // likely filled another buffer than we requested: adjust buffer index
786 size_t ix;
787 for (ix = 0; ix < mInputBuffers.size(); ix++) {
788 const sp<ABuffer> &buf = mInputBuffers[ix];
789 if (buf->data() == mediaBuffer->data()) {
790 // all input buffers are dequeued on start, hence the check
791 if (!mInputBufferIsDequeued[ix]) {
792 ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
793 mComponentName.c_str(), ix, bufferIx);
794 mediaBuffer->release();
795 return false;
796 }
797
798 // TRICKY: need buffer for the metadata, so instead, set
799 // codecBuffer to the same (though incorrect) buffer to
800 // avoid a memcpy into the codecBuffer
801 codecBuffer = buffer;
802 codecBuffer->setRange(
803 mediaBuffer->range_offset(),
804 mediaBuffer->range_length());
805 bufferIx = ix;
806 break;
807 }
808 }
809 CHECK(ix < mInputBuffers.size());
810 }
811 }
812
813 if (buffer == NULL /* includes !hasBuffer */) {
814 int32_t streamErr = ERROR_END_OF_STREAM;
815 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
816
817 CHECK(streamErr != OK);
818
819 // attempt to queue EOS
820 status_t err = mCodec->queueInputBuffer(
821 bufferIx,
822 0,
823 0,
824 0,
825 MediaCodec::BUFFER_FLAG_EOS);
826 if (err == OK) {
827 mInputBufferIsDequeued.editItemAt(bufferIx) = false;
828 } else if (streamErr == ERROR_END_OF_STREAM) {
829 streamErr = err;
830 // err will not be ERROR_END_OF_STREAM
831 }
832
833 if (streamErr != ERROR_END_OF_STREAM) {
834 ALOGE("Stream error for %s (err=%d), EOS %s queued",
835 mComponentName.c_str(),
836 streamErr,
837 err == OK ? "successfully" : "unsuccessfully");
838 handleError(streamErr);
839 }
840 } else {
841 sp<AMessage> extra;
842 if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
843 int64_t resumeAtMediaTimeUs;
844 if (extra->findInt64(
845 "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
846 ALOGI("[%s] suppressing rendering until %lld us",
847 mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
848 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
849 }
850 }
851
852 int64_t timeUs = 0;
853 uint32_t flags = 0;
854 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
855
856 int32_t eos, csd;
857 // we do not expect SYNCFRAME for decoder
858 if (buffer->meta()->findInt32("eos", &eos) && eos) {
859 flags |= MediaCodec::BUFFER_FLAG_EOS;
860 } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
861 flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
862 }
863
864 // copy into codec buffer
865 if (buffer != codecBuffer) {
866 CHECK_LE(buffer->size(), codecBuffer->capacity());
867 codecBuffer->setRange(0, buffer->size());
868 memcpy(codecBuffer->data(), buffer->data(), buffer->size());
869 }
870
871 status_t err = mCodec->queueInputBuffer(
872 bufferIx,
873 codecBuffer->offset(),
874 codecBuffer->size(),
875 timeUs,
876 flags);
877 if (err != OK) {
878 if (mediaBuffer != NULL) {
879 mediaBuffer->release();
880 }
881 ALOGE("Failed to queue input buffer for %s (err=%d)",
882 mComponentName.c_str(), err);
883 handleError(err);
884 } else {
885 mInputBufferIsDequeued.editItemAt(bufferIx) = false;
886 if (mediaBuffer != NULL) {
887 CHECK(mMediaBuffers[bufferIx] == NULL);
888 mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
889 }
890 }
891 }
892 return true;
893 }
894
onRenderBuffer(const sp<AMessage> & msg)895 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
896 status_t err;
897 int32_t render;
898 size_t bufferIx;
899 int32_t eos;
900 CHECK(msg->findSize("buffer-ix", &bufferIx));
901
902 if (!mIsAudio) {
903 int64_t timeUs;
904 sp<ABuffer> buffer = mOutputBuffers[bufferIx];
905 buffer->meta()->findInt64("timeUs", &timeUs);
906
907 if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
908 mCCDecoder->display(timeUs);
909 }
910 }
911
912 if (msg->findInt32("render", &render) && render) {
913 int64_t timestampNs;
914 CHECK(msg->findInt64("timestampNs", ×tampNs));
915 err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
916 } else {
917 mNumOutputFramesDropped += !mIsAudio;
918 err = mCodec->releaseOutputBuffer(bufferIx);
919 }
920 if (err != OK) {
921 ALOGE("failed to release output buffer for %s (err=%d)",
922 mComponentName.c_str(), err);
923 handleError(err);
924 }
925 if (msg->findInt32("eos", &eos) && eos
926 && isDiscontinuityPending()) {
927 finishHandleDiscontinuity(true /* flushOnTimeChange */);
928 }
929 }
930
isDiscontinuityPending() const931 bool NuPlayer::Decoder::isDiscontinuityPending() const {
932 return mFormatChangePending || mTimeChangePending;
933 }
934
finishHandleDiscontinuity(bool flushOnTimeChange)935 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
936 ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
937 mFormatChangePending, mTimeChangePending, flushOnTimeChange);
938
939 // If we have format change, pause and wait to be killed;
940 // If we have time change only, flush and restart fetching.
941
942 if (mFormatChangePending) {
943 mPaused = true;
944 } else if (mTimeChangePending) {
945 if (flushOnTimeChange) {
946 doFlush(false /* notifyComplete */);
947 signalResume(false /* notifyComplete */);
948 }
949 }
950
951 // Notify NuPlayer to either shutdown decoder, or rescan sources
952 sp<AMessage> msg = mNotify->dup();
953 msg->setInt32("what", kWhatInputDiscontinuity);
954 msg->setInt32("formatChange", mFormatChangePending);
955 msg->post();
956
957 mFormatChangePending = false;
958 mTimeChangePending = false;
959 }
960
supportsSeamlessAudioFormatChange(const sp<AMessage> & targetFormat) const961 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
962 const sp<AMessage> &targetFormat) const {
963 if (targetFormat == NULL) {
964 return true;
965 }
966
967 AString mime;
968 if (!targetFormat->findString("mime", &mime)) {
969 return false;
970 }
971
972 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
973 // field-by-field comparison
974 const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
975 for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
976 int32_t oldVal, newVal;
977 if (!mInputFormat->findInt32(keys[i], &oldVal) ||
978 !targetFormat->findInt32(keys[i], &newVal) ||
979 oldVal != newVal) {
980 return false;
981 }
982 }
983
984 sp<ABuffer> oldBuf, newBuf;
985 if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
986 targetFormat->findBuffer("csd-0", &newBuf)) {
987 if (oldBuf->size() != newBuf->size()) {
988 return false;
989 }
990 return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
991 }
992 }
993 return false;
994 }
995
supportsSeamlessFormatChange(const sp<AMessage> & targetFormat) const996 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
997 if (mInputFormat == NULL) {
998 return false;
999 }
1000
1001 if (targetFormat == NULL) {
1002 return true;
1003 }
1004
1005 AString oldMime, newMime;
1006 if (!mInputFormat->findString("mime", &oldMime)
1007 || !targetFormat->findString("mime", &newMime)
1008 || !(oldMime == newMime)) {
1009 return false;
1010 }
1011
1012 bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
1013 bool seamless;
1014 if (audio) {
1015 seamless = supportsSeamlessAudioFormatChange(targetFormat);
1016 } else {
1017 int32_t isAdaptive;
1018 seamless = (mCodec != NULL &&
1019 mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
1020 isAdaptive);
1021 }
1022
1023 ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
1024 return seamless;
1025 }
1026
rememberCodecSpecificData(const sp<AMessage> & format)1027 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
1028 if (format == NULL) {
1029 return;
1030 }
1031 mCSDsForCurrentFormat.clear();
1032 for (int32_t i = 0; ; ++i) {
1033 AString tag = "csd-";
1034 tag.append(i);
1035 sp<ABuffer> buffer;
1036 if (!format->findBuffer(tag.c_str(), &buffer)) {
1037 break;
1038 }
1039 mCSDsForCurrentFormat.push(buffer);
1040 }
1041 }
1042
notifyResumeCompleteIfNecessary()1043 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
1044 if (mResumePending) {
1045 mResumePending = false;
1046
1047 sp<AMessage> notify = mNotify->dup();
1048 notify->setInt32("what", kWhatResumeCompleted);
1049 notify->post();
1050 }
1051 }
1052
1053 } // namespace android
1054
1055