• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayerRenderer"
19 #include <utils/Log.h>
20 
21 #include "NuPlayerRenderer.h"
22 
23 #include <media/stagefright/foundation/ABuffer.h>
24 #include <media/stagefright/foundation/ADebug.h>
25 #include <media/stagefright/foundation/AMessage.h>
26 
27 namespace android {
28 
29 // static
30 const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
31 
Renderer(const sp<MediaPlayerBase::AudioSink> & sink,const sp<AMessage> & notify)32 NuPlayer::Renderer::Renderer(
33         const sp<MediaPlayerBase::AudioSink> &sink,
34         const sp<AMessage> &notify)
35     : mAudioSink(sink),
36       mNotify(notify),
37       mNumFramesWritten(0),
38       mDrainAudioQueuePending(false),
39       mDrainVideoQueuePending(false),
40       mAudioQueueGeneration(0),
41       mVideoQueueGeneration(0),
42       mAnchorTimeMediaUs(-1),
43       mAnchorTimeRealUs(-1),
44       mFlushingAudio(false),
45       mFlushingVideo(false),
46       mHasAudio(false),
47       mHasVideo(false),
48       mSyncQueues(false),
49       mPaused(false),
50       mVideoRenderingStarted(false),
51       mLastPositionUpdateUs(-1ll),
52       mVideoLateByUs(0ll) {
53 }
54 
~Renderer()55 NuPlayer::Renderer::~Renderer() {
56 }
57 
queueBuffer(bool audio,const sp<ABuffer> & buffer,const sp<AMessage> & notifyConsumed)58 void NuPlayer::Renderer::queueBuffer(
59         bool audio,
60         const sp<ABuffer> &buffer,
61         const sp<AMessage> &notifyConsumed) {
62     sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
63     msg->setInt32("audio", static_cast<int32_t>(audio));
64     msg->setBuffer("buffer", buffer);
65     msg->setMessage("notifyConsumed", notifyConsumed);
66     msg->post();
67 }
68 
queueEOS(bool audio,status_t finalResult)69 void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
70     CHECK_NE(finalResult, (status_t)OK);
71 
72     sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
73     msg->setInt32("audio", static_cast<int32_t>(audio));
74     msg->setInt32("finalResult", finalResult);
75     msg->post();
76 }
77 
flush(bool audio)78 void NuPlayer::Renderer::flush(bool audio) {
79     {
80         Mutex::Autolock autoLock(mFlushLock);
81         if (audio) {
82             CHECK(!mFlushingAudio);
83             mFlushingAudio = true;
84         } else {
85             CHECK(!mFlushingVideo);
86             mFlushingVideo = true;
87         }
88     }
89 
90     sp<AMessage> msg = new AMessage(kWhatFlush, id());
91     msg->setInt32("audio", static_cast<int32_t>(audio));
92     msg->post();
93 }
94 
signalTimeDiscontinuity()95 void NuPlayer::Renderer::signalTimeDiscontinuity() {
96     CHECK(mAudioQueue.empty());
97     CHECK(mVideoQueue.empty());
98     mAnchorTimeMediaUs = -1;
99     mAnchorTimeRealUs = -1;
100     mSyncQueues = mHasAudio && mHasVideo;
101 }
102 
pause()103 void NuPlayer::Renderer::pause() {
104     (new AMessage(kWhatPause, id()))->post();
105 }
106 
resume()107 void NuPlayer::Renderer::resume() {
108     (new AMessage(kWhatResume, id()))->post();
109 }
110 
onMessageReceived(const sp<AMessage> & msg)111 void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
112     switch (msg->what()) {
113         case kWhatDrainAudioQueue:
114         {
115             int32_t generation;
116             CHECK(msg->findInt32("generation", &generation));
117             if (generation != mAudioQueueGeneration) {
118                 break;
119             }
120 
121             mDrainAudioQueuePending = false;
122 
123             if (onDrainAudioQueue()) {
124                 uint32_t numFramesPlayed;
125                 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
126                          (status_t)OK);
127 
128                 uint32_t numFramesPendingPlayout =
129                     mNumFramesWritten - numFramesPlayed;
130 
131                 // This is how long the audio sink will have data to
132                 // play back.
133                 int64_t delayUs =
134                     mAudioSink->msecsPerFrame()
135                         * numFramesPendingPlayout * 1000ll;
136 
137                 // Let's give it more data after about half that time
138                 // has elapsed.
139                 postDrainAudioQueue(delayUs / 2);
140             }
141             break;
142         }
143 
144         case kWhatDrainVideoQueue:
145         {
146             int32_t generation;
147             CHECK(msg->findInt32("generation", &generation));
148             if (generation != mVideoQueueGeneration) {
149                 break;
150             }
151 
152             mDrainVideoQueuePending = false;
153 
154             onDrainVideoQueue();
155 
156             postDrainVideoQueue();
157             break;
158         }
159 
160         case kWhatQueueBuffer:
161         {
162             onQueueBuffer(msg);
163             break;
164         }
165 
166         case kWhatQueueEOS:
167         {
168             onQueueEOS(msg);
169             break;
170         }
171 
172         case kWhatFlush:
173         {
174             onFlush(msg);
175             break;
176         }
177 
178         case kWhatAudioSinkChanged:
179         {
180             onAudioSinkChanged();
181             break;
182         }
183 
184         case kWhatPause:
185         {
186             onPause();
187             break;
188         }
189 
190         case kWhatResume:
191         {
192             onResume();
193             break;
194         }
195 
196         default:
197             TRESPASS();
198             break;
199     }
200 }
201 
postDrainAudioQueue(int64_t delayUs)202 void NuPlayer::Renderer::postDrainAudioQueue(int64_t delayUs) {
203     if (mDrainAudioQueuePending || mSyncQueues || mPaused) {
204         return;
205     }
206 
207     if (mAudioQueue.empty()) {
208         return;
209     }
210 
211     mDrainAudioQueuePending = true;
212     sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
213     msg->setInt32("generation", mAudioQueueGeneration);
214     msg->post(delayUs);
215 }
216 
signalAudioSinkChanged()217 void NuPlayer::Renderer::signalAudioSinkChanged() {
218     (new AMessage(kWhatAudioSinkChanged, id()))->post();
219 }
220 
onDrainAudioQueue()221 bool NuPlayer::Renderer::onDrainAudioQueue() {
222     uint32_t numFramesPlayed;
223     if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
224         return false;
225     }
226 
227     ssize_t numFramesAvailableToWrite =
228         mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
229 
230 #if 0
231     if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
232         ALOGI("audio sink underrun");
233     } else {
234         ALOGV("audio queue has %d frames left to play",
235              mAudioSink->frameCount() - numFramesAvailableToWrite);
236     }
237 #endif
238 
239     size_t numBytesAvailableToWrite =
240         numFramesAvailableToWrite * mAudioSink->frameSize();
241 
242     while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) {
243         QueueEntry *entry = &*mAudioQueue.begin();
244 
245         if (entry->mBuffer == NULL) {
246             // EOS
247 
248             notifyEOS(true /* audio */, entry->mFinalResult);
249 
250             mAudioQueue.erase(mAudioQueue.begin());
251             entry = NULL;
252             return false;
253         }
254 
255         if (entry->mOffset == 0) {
256             int64_t mediaTimeUs;
257             CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
258 
259             ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
260 
261             mAnchorTimeMediaUs = mediaTimeUs;
262 
263             uint32_t numFramesPlayed;
264             CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
265 
266             uint32_t numFramesPendingPlayout =
267                 mNumFramesWritten - numFramesPlayed;
268 
269             int64_t realTimeOffsetUs =
270                 (mAudioSink->latency() / 2  /* XXX */
271                     + numFramesPendingPlayout
272                         * mAudioSink->msecsPerFrame()) * 1000ll;
273 
274             // ALOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
275 
276             mAnchorTimeRealUs =
277                 ALooper::GetNowUs() + realTimeOffsetUs;
278         }
279 
280         size_t copy = entry->mBuffer->size() - entry->mOffset;
281         if (copy > numBytesAvailableToWrite) {
282             copy = numBytesAvailableToWrite;
283         }
284 
285         CHECK_EQ(mAudioSink->write(
286                     entry->mBuffer->data() + entry->mOffset, copy),
287                  (ssize_t)copy);
288 
289         entry->mOffset += copy;
290         if (entry->mOffset == entry->mBuffer->size()) {
291             entry->mNotifyConsumed->post();
292             mAudioQueue.erase(mAudioQueue.begin());
293 
294             entry = NULL;
295         }
296 
297         numBytesAvailableToWrite -= copy;
298         size_t copiedFrames = copy / mAudioSink->frameSize();
299         mNumFramesWritten += copiedFrames;
300     }
301 
302     notifyPosition();
303 
304     return !mAudioQueue.empty();
305 }
306 
postDrainVideoQueue()307 void NuPlayer::Renderer::postDrainVideoQueue() {
308     if (mDrainVideoQueuePending || mSyncQueues || mPaused) {
309         return;
310     }
311 
312     if (mVideoQueue.empty()) {
313         return;
314     }
315 
316     QueueEntry &entry = *mVideoQueue.begin();
317 
318     sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
319     msg->setInt32("generation", mVideoQueueGeneration);
320 
321     int64_t delayUs;
322 
323     if (entry.mBuffer == NULL) {
324         // EOS doesn't carry a timestamp.
325         delayUs = 0;
326     } else {
327         int64_t mediaTimeUs;
328         CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
329 
330         if (mAnchorTimeMediaUs < 0) {
331             delayUs = 0;
332 
333             if (!mHasAudio) {
334                 mAnchorTimeMediaUs = mediaTimeUs;
335                 mAnchorTimeRealUs = ALooper::GetNowUs();
336             }
337         } else {
338             int64_t realTimeUs =
339                 (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
340 
341             delayUs = realTimeUs - ALooper::GetNowUs();
342         }
343     }
344 
345     msg->post(delayUs);
346 
347     mDrainVideoQueuePending = true;
348 }
349 
onDrainVideoQueue()350 void NuPlayer::Renderer::onDrainVideoQueue() {
351     if (mVideoQueue.empty()) {
352         return;
353     }
354 
355     QueueEntry *entry = &*mVideoQueue.begin();
356 
357     if (entry->mBuffer == NULL) {
358         // EOS
359 
360         notifyEOS(false /* audio */, entry->mFinalResult);
361 
362         mVideoQueue.erase(mVideoQueue.begin());
363         entry = NULL;
364 
365         mVideoLateByUs = 0ll;
366 
367         notifyPosition();
368         return;
369     }
370 
371     int64_t mediaTimeUs;
372     CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
373 
374     int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
375     mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
376 
377     bool tooLate = (mVideoLateByUs > 40000);
378 
379     if (tooLate) {
380         ALOGV("video late by %lld us (%.2f secs)",
381              mVideoLateByUs, mVideoLateByUs / 1E6);
382     } else {
383         ALOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
384     }
385 
386     entry->mNotifyConsumed->setInt32("render", !tooLate);
387     entry->mNotifyConsumed->post();
388     mVideoQueue.erase(mVideoQueue.begin());
389     entry = NULL;
390 
391     if (!mVideoRenderingStarted) {
392         mVideoRenderingStarted = true;
393         notifyVideoRenderingStart();
394     }
395 
396     notifyPosition();
397 }
398 
notifyVideoRenderingStart()399 void NuPlayer::Renderer::notifyVideoRenderingStart() {
400     sp<AMessage> notify = mNotify->dup();
401     notify->setInt32("what", kWhatVideoRenderingStart);
402     notify->post();
403 }
404 
notifyEOS(bool audio,status_t finalResult)405 void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult) {
406     sp<AMessage> notify = mNotify->dup();
407     notify->setInt32("what", kWhatEOS);
408     notify->setInt32("audio", static_cast<int32_t>(audio));
409     notify->setInt32("finalResult", finalResult);
410     notify->post();
411 }
412 
onQueueBuffer(const sp<AMessage> & msg)413 void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
414     int32_t audio;
415     CHECK(msg->findInt32("audio", &audio));
416 
417     if (audio) {
418         mHasAudio = true;
419     } else {
420         mHasVideo = true;
421     }
422 
423     if (dropBufferWhileFlushing(audio, msg)) {
424         return;
425     }
426 
427     sp<ABuffer> buffer;
428     CHECK(msg->findBuffer("buffer", &buffer));
429 
430     sp<AMessage> notifyConsumed;
431     CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
432 
433     QueueEntry entry;
434     entry.mBuffer = buffer;
435     entry.mNotifyConsumed = notifyConsumed;
436     entry.mOffset = 0;
437     entry.mFinalResult = OK;
438 
439     if (audio) {
440         mAudioQueue.push_back(entry);
441         postDrainAudioQueue();
442     } else {
443         mVideoQueue.push_back(entry);
444         postDrainVideoQueue();
445     }
446 
447     if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
448         return;
449     }
450 
451     sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
452     sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
453 
454     if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
455         // EOS signalled on either queue.
456         syncQueuesDone();
457         return;
458     }
459 
460     int64_t firstAudioTimeUs;
461     int64_t firstVideoTimeUs;
462     CHECK(firstAudioBuffer->meta()
463             ->findInt64("timeUs", &firstAudioTimeUs));
464     CHECK(firstVideoBuffer->meta()
465             ->findInt64("timeUs", &firstVideoTimeUs));
466 
467     int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
468 
469     ALOGV("queueDiff = %.2f secs", diff / 1E6);
470 
471     if (diff > 100000ll) {
472         // Audio data starts More than 0.1 secs before video.
473         // Drop some audio.
474 
475         (*mAudioQueue.begin()).mNotifyConsumed->post();
476         mAudioQueue.erase(mAudioQueue.begin());
477         return;
478     }
479 
480     syncQueuesDone();
481 }
482 
syncQueuesDone()483 void NuPlayer::Renderer::syncQueuesDone() {
484     if (!mSyncQueues) {
485         return;
486     }
487 
488     mSyncQueues = false;
489 
490     if (!mAudioQueue.empty()) {
491         postDrainAudioQueue();
492     }
493 
494     if (!mVideoQueue.empty()) {
495         postDrainVideoQueue();
496     }
497 }
498 
onQueueEOS(const sp<AMessage> & msg)499 void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
500     int32_t audio;
501     CHECK(msg->findInt32("audio", &audio));
502 
503     if (dropBufferWhileFlushing(audio, msg)) {
504         return;
505     }
506 
507     int32_t finalResult;
508     CHECK(msg->findInt32("finalResult", &finalResult));
509 
510     QueueEntry entry;
511     entry.mOffset = 0;
512     entry.mFinalResult = finalResult;
513 
514     if (audio) {
515         mAudioQueue.push_back(entry);
516         postDrainAudioQueue();
517     } else {
518         mVideoQueue.push_back(entry);
519         postDrainVideoQueue();
520     }
521 }
522 
onFlush(const sp<AMessage> & msg)523 void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
524     int32_t audio;
525     CHECK(msg->findInt32("audio", &audio));
526 
527     // If we're currently syncing the queues, i.e. dropping audio while
528     // aligning the first audio/video buffer times and only one of the
529     // two queues has data, we may starve that queue by not requesting
530     // more buffers from the decoder. If the other source then encounters
531     // a discontinuity that leads to flushing, we'll never find the
532     // corresponding discontinuity on the other queue.
533     // Therefore we'll stop syncing the queues if at least one of them
534     // is flushed.
535     syncQueuesDone();
536 
537     if (audio) {
538         flushQueue(&mAudioQueue);
539 
540         Mutex::Autolock autoLock(mFlushLock);
541         mFlushingAudio = false;
542 
543         mDrainAudioQueuePending = false;
544         ++mAudioQueueGeneration;
545     } else {
546         flushQueue(&mVideoQueue);
547 
548         Mutex::Autolock autoLock(mFlushLock);
549         mFlushingVideo = false;
550 
551         mDrainVideoQueuePending = false;
552         ++mVideoQueueGeneration;
553     }
554 
555     notifyFlushComplete(audio);
556 }
557 
flushQueue(List<QueueEntry> * queue)558 void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
559     while (!queue->empty()) {
560         QueueEntry *entry = &*queue->begin();
561 
562         if (entry->mBuffer != NULL) {
563             entry->mNotifyConsumed->post();
564         }
565 
566         queue->erase(queue->begin());
567         entry = NULL;
568     }
569 }
570 
notifyFlushComplete(bool audio)571 void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
572     sp<AMessage> notify = mNotify->dup();
573     notify->setInt32("what", kWhatFlushComplete);
574     notify->setInt32("audio", static_cast<int32_t>(audio));
575     notify->post();
576 }
577 
dropBufferWhileFlushing(bool audio,const sp<AMessage> & msg)578 bool NuPlayer::Renderer::dropBufferWhileFlushing(
579         bool audio, const sp<AMessage> &msg) {
580     bool flushing = false;
581 
582     {
583         Mutex::Autolock autoLock(mFlushLock);
584         if (audio) {
585             flushing = mFlushingAudio;
586         } else {
587             flushing = mFlushingVideo;
588         }
589     }
590 
591     if (!flushing) {
592         return false;
593     }
594 
595     sp<AMessage> notifyConsumed;
596     if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
597         notifyConsumed->post();
598     }
599 
600     return true;
601 }
602 
onAudioSinkChanged()603 void NuPlayer::Renderer::onAudioSinkChanged() {
604     CHECK(!mDrainAudioQueuePending);
605     mNumFramesWritten = 0;
606     uint32_t written;
607     if (mAudioSink->getFramesWritten(&written) == OK) {
608         mNumFramesWritten = written;
609     }
610 }
611 
notifyPosition()612 void NuPlayer::Renderer::notifyPosition() {
613     if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) {
614         return;
615     }
616 
617     int64_t nowUs = ALooper::GetNowUs();
618 
619     if (mLastPositionUpdateUs >= 0
620             && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) {
621         return;
622     }
623     mLastPositionUpdateUs = nowUs;
624 
625     int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
626 
627     sp<AMessage> notify = mNotify->dup();
628     notify->setInt32("what", kWhatPosition);
629     notify->setInt64("positionUs", positionUs);
630     notify->setInt64("videoLateByUs", mVideoLateByUs);
631     notify->post();
632 }
633 
onPause()634 void NuPlayer::Renderer::onPause() {
635     CHECK(!mPaused);
636 
637     mDrainAudioQueuePending = false;
638     ++mAudioQueueGeneration;
639 
640     mDrainVideoQueuePending = false;
641     ++mVideoQueueGeneration;
642 
643     if (mHasAudio) {
644         mAudioSink->pause();
645     }
646 
647     ALOGV("now paused audio queue has %d entries, video has %d entries",
648           mAudioQueue.size(), mVideoQueue.size());
649 
650     mPaused = true;
651 }
652 
onResume()653 void NuPlayer::Renderer::onResume() {
654     if (!mPaused) {
655         return;
656     }
657 
658     if (mHasAudio) {
659         mAudioSink->start();
660     }
661 
662     mPaused = false;
663 
664     if (!mAudioQueue.empty()) {
665         postDrainAudioQueue();
666     }
667 
668     if (!mVideoQueue.empty()) {
669         postDrainVideoQueue();
670     }
671 }
672 
673 }  // namespace android
674 
675