1 /*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "PlaybackSession"
19 #include <utils/Log.h>
20
21 #include "PlaybackSession.h"
22
23 #include "Converter.h"
24 #include "MediaPuller.h"
25 #include "RepeaterSource.h"
26 #include "include/avc_utils.h"
27 #include "WifiDisplaySource.h"
28
29 #include <binder/IServiceManager.h>
30 #include <cutils/properties.h>
31 #include <media/IHDCP.h>
32 #include <media/IMediaHTTPService.h>
33 #include <media/stagefright/foundation/ABitReader.h>
34 #include <media/stagefright/foundation/ABuffer.h>
35 #include <media/stagefright/foundation/ADebug.h>
36 #include <media/stagefright/foundation/AMessage.h>
37 #include <media/stagefright/foundation/hexdump.h>
38 #include <media/stagefright/AudioSource.h>
39 #include <media/stagefright/MediaDefs.h>
40 #include <media/stagefright/MediaErrors.h>
41 #include <media/stagefright/MediaSource.h>
42 #include <media/stagefright/MetaData.h>
43 #include <media/stagefright/NuMediaExtractor.h>
44 #include <media/stagefright/SurfaceMediaSource.h>
45 #include <media/stagefright/Utils.h>
46
47 #include <OMX_IVCommon.h>
48
49 namespace android {
50
51 struct WifiDisplaySource::PlaybackSession::Track : public AHandler {
52 enum {
53 kWhatStopped,
54 };
55
56 Track(const sp<AMessage> ¬ify,
57 const sp<ALooper> &pullLooper,
58 const sp<ALooper> &codecLooper,
59 const sp<MediaPuller> &mediaPuller,
60 const sp<Converter> &converter);
61
62 Track(const sp<AMessage> ¬ify, const sp<AMessage> &format);
63
64 void setRepeaterSource(const sp<RepeaterSource> &source);
65
66 sp<AMessage> getFormat();
67 bool isAudio() const;
68
69 const sp<Converter> &converter() const;
70 const sp<RepeaterSource> &repeaterSource() const;
71
72 ssize_t mediaSenderTrackIndex() const;
73 void setMediaSenderTrackIndex(size_t index);
74
75 status_t start();
76 void stopAsync();
77
78 void pause();
79 void resume();
80
81 void queueAccessUnit(const sp<ABuffer> &accessUnit);
82 sp<ABuffer> dequeueAccessUnit();
83
84 bool hasOutputBuffer(int64_t *timeUs) const;
85 void queueOutputBuffer(const sp<ABuffer> &accessUnit);
86 sp<ABuffer> dequeueOutputBuffer();
87
88 #if SUSPEND_VIDEO_IF_IDLE
89 bool isSuspended() const;
90 #endif
91
countQueuedOutputBuffersandroid::WifiDisplaySource::PlaybackSession::Track92 size_t countQueuedOutputBuffers() const {
93 return mQueuedOutputBuffers.size();
94 }
95
96 void requestIDRFrame();
97
98 protected:
99 virtual void onMessageReceived(const sp<AMessage> &msg);
100 virtual ~Track();
101
102 private:
103 enum {
104 kWhatMediaPullerStopped,
105 };
106
107 sp<AMessage> mNotify;
108 sp<ALooper> mPullLooper;
109 sp<ALooper> mCodecLooper;
110 sp<MediaPuller> mMediaPuller;
111 sp<Converter> mConverter;
112 sp<AMessage> mFormat;
113 bool mStarted;
114 ssize_t mMediaSenderTrackIndex;
115 bool mIsAudio;
116 List<sp<ABuffer> > mQueuedAccessUnits;
117 sp<RepeaterSource> mRepeaterSource;
118 List<sp<ABuffer> > mQueuedOutputBuffers;
119 int64_t mLastOutputBufferQueuedTimeUs;
120
121 static bool IsAudioFormat(const sp<AMessage> &format);
122
123 DISALLOW_EVIL_CONSTRUCTORS(Track);
124 };
125
Track(const sp<AMessage> & notify,const sp<ALooper> & pullLooper,const sp<ALooper> & codecLooper,const sp<MediaPuller> & mediaPuller,const sp<Converter> & converter)126 WifiDisplaySource::PlaybackSession::Track::Track(
127 const sp<AMessage> ¬ify,
128 const sp<ALooper> &pullLooper,
129 const sp<ALooper> &codecLooper,
130 const sp<MediaPuller> &mediaPuller,
131 const sp<Converter> &converter)
132 : mNotify(notify),
133 mPullLooper(pullLooper),
134 mCodecLooper(codecLooper),
135 mMediaPuller(mediaPuller),
136 mConverter(converter),
137 mStarted(false),
138 mIsAudio(IsAudioFormat(mConverter->getOutputFormat())),
139 mLastOutputBufferQueuedTimeUs(-1ll) {
140 }
141
Track(const sp<AMessage> & notify,const sp<AMessage> & format)142 WifiDisplaySource::PlaybackSession::Track::Track(
143 const sp<AMessage> ¬ify, const sp<AMessage> &format)
144 : mNotify(notify),
145 mFormat(format),
146 mStarted(false),
147 mIsAudio(IsAudioFormat(format)),
148 mLastOutputBufferQueuedTimeUs(-1ll) {
149 }
150
~Track()151 WifiDisplaySource::PlaybackSession::Track::~Track() {
152 CHECK(!mStarted);
153 }
154
155 // static
IsAudioFormat(const sp<AMessage> & format)156 bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat(
157 const sp<AMessage> &format) {
158 AString mime;
159 CHECK(format->findString("mime", &mime));
160
161 return !strncasecmp(mime.c_str(), "audio/", 6);
162 }
163
getFormat()164 sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() {
165 return mFormat != NULL ? mFormat : mConverter->getOutputFormat();
166 }
167
isAudio() const168 bool WifiDisplaySource::PlaybackSession::Track::isAudio() const {
169 return mIsAudio;
170 }
171
converter() const172 const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const {
173 return mConverter;
174 }
175
176 const sp<RepeaterSource> &
repeaterSource() const177 WifiDisplaySource::PlaybackSession::Track::repeaterSource() const {
178 return mRepeaterSource;
179 }
180
mediaSenderTrackIndex() const181 ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const {
182 CHECK_GE(mMediaSenderTrackIndex, 0);
183 return mMediaSenderTrackIndex;
184 }
185
setMediaSenderTrackIndex(size_t index)186 void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex(
187 size_t index) {
188 mMediaSenderTrackIndex = index;
189 }
190
start()191 status_t WifiDisplaySource::PlaybackSession::Track::start() {
192 ALOGV("Track::start isAudio=%d", mIsAudio);
193
194 CHECK(!mStarted);
195
196 status_t err = OK;
197
198 if (mMediaPuller != NULL) {
199 err = mMediaPuller->start();
200 }
201
202 if (err == OK) {
203 mStarted = true;
204 }
205
206 return err;
207 }
208
stopAsync()209 void WifiDisplaySource::PlaybackSession::Track::stopAsync() {
210 ALOGV("Track::stopAsync isAudio=%d", mIsAudio);
211
212 if (mConverter != NULL) {
213 mConverter->shutdownAsync();
214 }
215
216 sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, this);
217
218 if (mStarted && mMediaPuller != NULL) {
219 if (mRepeaterSource != NULL) {
220 // Let's unblock MediaPuller's MediaSource::read().
221 mRepeaterSource->wakeUp();
222 }
223
224 mMediaPuller->stopAsync(msg);
225 } else {
226 mStarted = false;
227 msg->post();
228 }
229 }
230
pause()231 void WifiDisplaySource::PlaybackSession::Track::pause() {
232 mMediaPuller->pause();
233 }
234
resume()235 void WifiDisplaySource::PlaybackSession::Track::resume() {
236 mMediaPuller->resume();
237 }
238
onMessageReceived(const sp<AMessage> & msg)239 void WifiDisplaySource::PlaybackSession::Track::onMessageReceived(
240 const sp<AMessage> &msg) {
241 switch (msg->what()) {
242 case kWhatMediaPullerStopped:
243 {
244 mConverter.clear();
245
246 mStarted = false;
247
248 sp<AMessage> notify = mNotify->dup();
249 notify->setInt32("what", kWhatStopped);
250 notify->post();
251
252 ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video");
253 break;
254 }
255
256 default:
257 TRESPASS();
258 }
259 }
260
queueAccessUnit(const sp<ABuffer> & accessUnit)261 void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit(
262 const sp<ABuffer> &accessUnit) {
263 mQueuedAccessUnits.push_back(accessUnit);
264 }
265
dequeueAccessUnit()266 sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() {
267 if (mQueuedAccessUnits.empty()) {
268 return NULL;
269 }
270
271 sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin();
272 CHECK(accessUnit != NULL);
273
274 mQueuedAccessUnits.erase(mQueuedAccessUnits.begin());
275
276 return accessUnit;
277 }
278
setRepeaterSource(const sp<RepeaterSource> & source)279 void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource(
280 const sp<RepeaterSource> &source) {
281 mRepeaterSource = source;
282 }
283
requestIDRFrame()284 void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() {
285 if (mIsAudio) {
286 return;
287 }
288
289 if (mRepeaterSource != NULL) {
290 mRepeaterSource->wakeUp();
291 }
292
293 mConverter->requestIDRFrame();
294 }
295
hasOutputBuffer(int64_t * timeUs) const296 bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer(
297 int64_t *timeUs) const {
298 *timeUs = 0ll;
299
300 if (mQueuedOutputBuffers.empty()) {
301 return false;
302 }
303
304 const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin();
305
306 CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs));
307
308 return true;
309 }
310
queueOutputBuffer(const sp<ABuffer> & accessUnit)311 void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer(
312 const sp<ABuffer> &accessUnit) {
313 mQueuedOutputBuffers.push_back(accessUnit);
314 mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs();
315 }
316
dequeueOutputBuffer()317 sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() {
318 CHECK(!mQueuedOutputBuffers.empty());
319
320 sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin();
321 mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin());
322
323 return outputBuffer;
324 }
325
326 #if SUSPEND_VIDEO_IF_IDLE
isSuspended() const327 bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const {
328 if (!mQueuedOutputBuffers.empty()) {
329 return false;
330 }
331
332 if (mLastOutputBufferQueuedTimeUs < 0ll) {
333 // We've never seen an output buffer queued, but tracks start
334 // out live, not suspended.
335 return false;
336 }
337
338 // If we've not seen new output data for 60ms or more, we consider
339 // this track suspended for the time being.
340 return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll;
341 }
342 #endif
343
344 ////////////////////////////////////////////////////////////////////////////////
345
PlaybackSession(const String16 & opPackageName,const sp<ANetworkSession> & netSession,const sp<AMessage> & notify,const in_addr & interfaceAddr,const sp<IHDCP> & hdcp,const char * path)346 WifiDisplaySource::PlaybackSession::PlaybackSession(
347 const String16 &opPackageName,
348 const sp<ANetworkSession> &netSession,
349 const sp<AMessage> ¬ify,
350 const in_addr &interfaceAddr,
351 const sp<IHDCP> &hdcp,
352 const char *path)
353 : mOpPackageName(opPackageName),
354 mNetSession(netSession),
355 mNotify(notify),
356 mInterfaceAddr(interfaceAddr),
357 mHDCP(hdcp),
358 mLocalRTPPort(-1),
359 mWeAreDead(false),
360 mPaused(false),
361 mLastLifesignUs(),
362 mVideoTrackIndex(-1),
363 mPrevTimeUs(-1ll),
364 mPullExtractorPending(false),
365 mPullExtractorGeneration(0),
366 mFirstSampleTimeRealUs(-1ll),
367 mFirstSampleTimeUs(-1ll) {
368 if (path != NULL) {
369 mMediaPath.setTo(path);
370 }
371 }
372
init(const char * clientIP,int32_t clientRtp,RTPSender::TransportMode rtpMode,int32_t clientRtcp,RTPSender::TransportMode rtcpMode,bool enableAudio,bool usePCMAudio,bool enableVideo,VideoFormats::ResolutionType videoResolutionType,size_t videoResolutionIndex,VideoFormats::ProfileType videoProfileType,VideoFormats::LevelType videoLevelType)373 status_t WifiDisplaySource::PlaybackSession::init(
374 const char *clientIP,
375 int32_t clientRtp,
376 RTPSender::TransportMode rtpMode,
377 int32_t clientRtcp,
378 RTPSender::TransportMode rtcpMode,
379 bool enableAudio,
380 bool usePCMAudio,
381 bool enableVideo,
382 VideoFormats::ResolutionType videoResolutionType,
383 size_t videoResolutionIndex,
384 VideoFormats::ProfileType videoProfileType,
385 VideoFormats::LevelType videoLevelType) {
386 sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, this);
387 mMediaSender = new MediaSender(mNetSession, notify);
388 looper()->registerHandler(mMediaSender);
389
390 mMediaSender->setHDCP(mHDCP);
391
392 status_t err = setupPacketizer(
393 enableAudio,
394 usePCMAudio,
395 enableVideo,
396 videoResolutionType,
397 videoResolutionIndex,
398 videoProfileType,
399 videoLevelType);
400
401 if (err == OK) {
402 err = mMediaSender->initAsync(
403 -1 /* trackIndex */,
404 clientIP,
405 clientRtp,
406 rtpMode,
407 clientRtcp,
408 rtcpMode,
409 &mLocalRTPPort);
410 }
411
412 if (err != OK) {
413 mLocalRTPPort = -1;
414
415 looper()->unregisterHandler(mMediaSender->id());
416 mMediaSender.clear();
417
418 return err;
419 }
420
421 updateLiveness();
422
423 return OK;
424 }
425
~PlaybackSession()426 WifiDisplaySource::PlaybackSession::~PlaybackSession() {
427 }
428
getRTPPort() const429 int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const {
430 return mLocalRTPPort;
431 }
432
getLastLifesignUs() const433 int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const {
434 return mLastLifesignUs;
435 }
436
updateLiveness()437 void WifiDisplaySource::PlaybackSession::updateLiveness() {
438 mLastLifesignUs = ALooper::GetNowUs();
439 }
440
play()441 status_t WifiDisplaySource::PlaybackSession::play() {
442 updateLiveness();
443
444 (new AMessage(kWhatResume, this))->post();
445
446 return OK;
447 }
448
onMediaSenderInitialized()449 status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() {
450 for (size_t i = 0; i < mTracks.size(); ++i) {
451 CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start());
452 }
453
454 sp<AMessage> notify = mNotify->dup();
455 notify->setInt32("what", kWhatSessionEstablished);
456 notify->post();
457
458 return OK;
459 }
460
pause()461 status_t WifiDisplaySource::PlaybackSession::pause() {
462 updateLiveness();
463
464 (new AMessage(kWhatPause, this))->post();
465
466 return OK;
467 }
468
destroyAsync()469 void WifiDisplaySource::PlaybackSession::destroyAsync() {
470 ALOGI("destroyAsync");
471
472 for (size_t i = 0; i < mTracks.size(); ++i) {
473 mTracks.valueAt(i)->stopAsync();
474 }
475 }
476
onMessageReceived(const sp<AMessage> & msg)477 void WifiDisplaySource::PlaybackSession::onMessageReceived(
478 const sp<AMessage> &msg) {
479 switch (msg->what()) {
480 case kWhatConverterNotify:
481 {
482 if (mWeAreDead) {
483 ALOGV("dropping msg '%s' because we're dead",
484 msg->debugString().c_str());
485
486 break;
487 }
488
489 int32_t what;
490 CHECK(msg->findInt32("what", &what));
491
492 size_t trackIndex;
493 CHECK(msg->findSize("trackIndex", &trackIndex));
494
495 if (what == Converter::kWhatAccessUnit) {
496 sp<ABuffer> accessUnit;
497 CHECK(msg->findBuffer("accessUnit", &accessUnit));
498
499 const sp<Track> &track = mTracks.valueFor(trackIndex);
500
501 status_t err = mMediaSender->queueAccessUnit(
502 track->mediaSenderTrackIndex(),
503 accessUnit);
504
505 if (err != OK) {
506 notifySessionDead();
507 }
508 break;
509 } else if (what == Converter::kWhatEOS) {
510 CHECK_EQ(what, Converter::kWhatEOS);
511
512 ALOGI("output EOS on track %zu", trackIndex);
513
514 ssize_t index = mTracks.indexOfKey(trackIndex);
515 CHECK_GE(index, 0);
516
517 const sp<Converter> &converter =
518 mTracks.valueAt(index)->converter();
519 looper()->unregisterHandler(converter->id());
520
521 mTracks.removeItemsAt(index);
522
523 if (mTracks.isEmpty()) {
524 ALOGI("Reached EOS");
525 }
526 } else if (what != Converter::kWhatShutdownCompleted) {
527 CHECK_EQ(what, Converter::kWhatError);
528
529 status_t err;
530 CHECK(msg->findInt32("err", &err));
531
532 ALOGE("converter signaled error %d", err);
533
534 notifySessionDead();
535 }
536 break;
537 }
538
539 case kWhatMediaSenderNotify:
540 {
541 int32_t what;
542 CHECK(msg->findInt32("what", &what));
543
544 if (what == MediaSender::kWhatInitDone) {
545 status_t err;
546 CHECK(msg->findInt32("err", &err));
547
548 if (err == OK) {
549 onMediaSenderInitialized();
550 } else {
551 notifySessionDead();
552 }
553 } else if (what == MediaSender::kWhatError) {
554 notifySessionDead();
555 } else if (what == MediaSender::kWhatNetworkStall) {
556 size_t numBytesQueued;
557 CHECK(msg->findSize("numBytesQueued", &numBytesQueued));
558
559 if (mVideoTrackIndex >= 0) {
560 const sp<Track> &videoTrack =
561 mTracks.valueFor(mVideoTrackIndex);
562
563 sp<Converter> converter = videoTrack->converter();
564 if (converter != NULL) {
565 converter->dropAFrame();
566 }
567 }
568 } else if (what == MediaSender::kWhatInformSender) {
569 onSinkFeedback(msg);
570 } else {
571 TRESPASS();
572 }
573 break;
574 }
575
576 case kWhatTrackNotify:
577 {
578 int32_t what;
579 CHECK(msg->findInt32("what", &what));
580
581 size_t trackIndex;
582 CHECK(msg->findSize("trackIndex", &trackIndex));
583
584 if (what == Track::kWhatStopped) {
585 ALOGI("Track %zu stopped", trackIndex);
586
587 sp<Track> track = mTracks.valueFor(trackIndex);
588 looper()->unregisterHandler(track->id());
589 mTracks.removeItem(trackIndex);
590 track.clear();
591
592 if (!mTracks.isEmpty()) {
593 ALOGI("not all tracks are stopped yet");
594 break;
595 }
596
597 looper()->unregisterHandler(mMediaSender->id());
598 mMediaSender.clear();
599
600 sp<AMessage> notify = mNotify->dup();
601 notify->setInt32("what", kWhatSessionDestroyed);
602 notify->post();
603 }
604 break;
605 }
606
607 case kWhatPause:
608 {
609 if (mExtractor != NULL) {
610 ++mPullExtractorGeneration;
611 mFirstSampleTimeRealUs = -1ll;
612 mFirstSampleTimeUs = -1ll;
613 }
614
615 if (mPaused) {
616 break;
617 }
618
619 for (size_t i = 0; i < mTracks.size(); ++i) {
620 mTracks.editValueAt(i)->pause();
621 }
622
623 mPaused = true;
624 break;
625 }
626
627 case kWhatResume:
628 {
629 if (mExtractor != NULL) {
630 schedulePullExtractor();
631 }
632
633 if (!mPaused) {
634 break;
635 }
636
637 for (size_t i = 0; i < mTracks.size(); ++i) {
638 mTracks.editValueAt(i)->resume();
639 }
640
641 mPaused = false;
642 break;
643 }
644
645 case kWhatPullExtractorSample:
646 {
647 int32_t generation;
648 CHECK(msg->findInt32("generation", &generation));
649
650 if (generation != mPullExtractorGeneration) {
651 break;
652 }
653
654 mPullExtractorPending = false;
655
656 onPullExtractor();
657 break;
658 }
659
660 default:
661 TRESPASS();
662 }
663 }
664
onSinkFeedback(const sp<AMessage> & msg)665 void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) {
666 int64_t avgLatencyUs;
667 CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
668
669 int64_t maxLatencyUs;
670 CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
671
672 ALOGI("sink reports avg. latency of %lld ms (max %lld ms)",
673 avgLatencyUs / 1000ll,
674 maxLatencyUs / 1000ll);
675
676 if (mVideoTrackIndex >= 0) {
677 const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex);
678 sp<Converter> converter = videoTrack->converter();
679
680 if (converter != NULL) {
681 int32_t videoBitrate =
682 Converter::GetInt32Property("media.wfd.video-bitrate", -1);
683
684 char val[PROPERTY_VALUE_MAX];
685 if (videoBitrate < 0
686 && property_get("media.wfd.video-bitrate", val, NULL)
687 && !strcasecmp("adaptive", val)) {
688 videoBitrate = converter->getVideoBitrate();
689
690 if (avgLatencyUs > 300000ll) {
691 videoBitrate *= 0.6;
692 } else if (avgLatencyUs < 100000ll) {
693 videoBitrate *= 1.1;
694 }
695 }
696
697 if (videoBitrate > 0) {
698 if (videoBitrate < 500000) {
699 videoBitrate = 500000;
700 } else if (videoBitrate > 10000000) {
701 videoBitrate = 10000000;
702 }
703
704 if (videoBitrate != converter->getVideoBitrate()) {
705 ALOGI("setting video bitrate to %d bps", videoBitrate);
706
707 converter->setVideoBitrate(videoBitrate);
708 }
709 }
710 }
711
712 sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource();
713 if (repeaterSource != NULL) {
714 double rateHz =
715 Converter::GetInt32Property(
716 "media.wfd.video-framerate", -1);
717
718 char val[PROPERTY_VALUE_MAX];
719 if (rateHz < 0.0
720 && property_get("media.wfd.video-framerate", val, NULL)
721 && !strcasecmp("adaptive", val)) {
722 rateHz = repeaterSource->getFrameRate();
723
724 if (avgLatencyUs > 300000ll) {
725 rateHz *= 0.9;
726 } else if (avgLatencyUs < 200000ll) {
727 rateHz *= 1.1;
728 }
729 }
730
731 if (rateHz > 0) {
732 if (rateHz < 5.0) {
733 rateHz = 5.0;
734 } else if (rateHz > 30.0) {
735 rateHz = 30.0;
736 }
737
738 if (rateHz != repeaterSource->getFrameRate()) {
739 ALOGI("setting frame rate to %.2f Hz", rateHz);
740
741 repeaterSource->setFrameRate(rateHz);
742 }
743 }
744 }
745 }
746 }
747
setupMediaPacketizer(bool enableAudio,bool enableVideo)748 status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer(
749 bool enableAudio, bool enableVideo) {
750 mExtractor = new NuMediaExtractor;
751
752 status_t err = mExtractor->setDataSource(
753 NULL /* httpService */, mMediaPath.c_str());
754
755 if (err != OK) {
756 return err;
757 }
758
759 size_t n = mExtractor->countTracks();
760 bool haveAudio = false;
761 bool haveVideo = false;
762 for (size_t i = 0; i < n; ++i) {
763 sp<AMessage> format;
764 err = mExtractor->getTrackFormat(i, &format);
765
766 if (err != OK) {
767 continue;
768 }
769
770 AString mime;
771 CHECK(format->findString("mime", &mime));
772
773 bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6);
774 bool isVideo = !strncasecmp(mime.c_str(), "video/", 6);
775
776 if (isAudio && enableAudio && !haveAudio) {
777 haveAudio = true;
778 } else if (isVideo && enableVideo && !haveVideo) {
779 haveVideo = true;
780 } else {
781 continue;
782 }
783
784 err = mExtractor->selectTrack(i);
785
786 size_t trackIndex = mTracks.size();
787
788 sp<AMessage> notify = new AMessage(kWhatTrackNotify, this);
789 notify->setSize("trackIndex", trackIndex);
790
791 sp<Track> track = new Track(notify, format);
792 looper()->registerHandler(track);
793
794 mTracks.add(trackIndex, track);
795
796 mExtractorTrackToInternalTrack.add(i, trackIndex);
797
798 if (isVideo) {
799 mVideoTrackIndex = trackIndex;
800 }
801
802 uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
803
804 ssize_t mediaSenderTrackIndex =
805 mMediaSender->addTrack(format, flags);
806 CHECK_GE(mediaSenderTrackIndex, 0);
807
808 track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
809
810 if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) {
811 break;
812 }
813 }
814
815 return OK;
816 }
817
schedulePullExtractor()818 void WifiDisplaySource::PlaybackSession::schedulePullExtractor() {
819 if (mPullExtractorPending) {
820 return;
821 }
822
823 int64_t delayUs = 1000000; // default delay is 1 sec
824 int64_t sampleTimeUs;
825 status_t err = mExtractor->getSampleTime(&sampleTimeUs);
826
827 if (err == OK) {
828 int64_t nowUs = ALooper::GetNowUs();
829
830 if (mFirstSampleTimeRealUs < 0ll) {
831 mFirstSampleTimeRealUs = nowUs;
832 mFirstSampleTimeUs = sampleTimeUs;
833 }
834
835 int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs;
836 delayUs = whenUs - nowUs;
837 } else {
838 ALOGW("could not get sample time (%d)", err);
839 }
840
841 sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, this);
842 msg->setInt32("generation", mPullExtractorGeneration);
843 msg->post(delayUs);
844
845 mPullExtractorPending = true;
846 }
847
onPullExtractor()848 void WifiDisplaySource::PlaybackSession::onPullExtractor() {
849 sp<ABuffer> accessUnit = new ABuffer(1024 * 1024);
850 status_t err = mExtractor->readSampleData(accessUnit);
851 if (err != OK) {
852 // EOS.
853 return;
854 }
855
856 int64_t timeUs;
857 CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs));
858
859 accessUnit->meta()->setInt64(
860 "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs);
861
862 size_t trackIndex;
863 CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex));
864
865 sp<AMessage> msg = new AMessage(kWhatConverterNotify, this);
866
867 msg->setSize(
868 "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex));
869
870 msg->setInt32("what", Converter::kWhatAccessUnit);
871 msg->setBuffer("accessUnit", accessUnit);
872 msg->post();
873
874 mExtractor->advance();
875
876 schedulePullExtractor();
877 }
878
setupPacketizer(bool enableAudio,bool usePCMAudio,bool enableVideo,VideoFormats::ResolutionType videoResolutionType,size_t videoResolutionIndex,VideoFormats::ProfileType videoProfileType,VideoFormats::LevelType videoLevelType)879 status_t WifiDisplaySource::PlaybackSession::setupPacketizer(
880 bool enableAudio,
881 bool usePCMAudio,
882 bool enableVideo,
883 VideoFormats::ResolutionType videoResolutionType,
884 size_t videoResolutionIndex,
885 VideoFormats::ProfileType videoProfileType,
886 VideoFormats::LevelType videoLevelType) {
887 CHECK(enableAudio || enableVideo);
888
889 if (!mMediaPath.empty()) {
890 return setupMediaPacketizer(enableAudio, enableVideo);
891 }
892
893 if (enableVideo) {
894 status_t err = addVideoSource(
895 videoResolutionType, videoResolutionIndex, videoProfileType,
896 videoLevelType);
897
898 if (err != OK) {
899 return err;
900 }
901 }
902
903 if (!enableAudio) {
904 return OK;
905 }
906
907 return addAudioSource(usePCMAudio);
908 }
909
addSource(bool isVideo,const sp<MediaSource> & source,bool isRepeaterSource,bool usePCMAudio,unsigned profileIdc,unsigned levelIdc,unsigned constraintSet,size_t * numInputBuffers)910 status_t WifiDisplaySource::PlaybackSession::addSource(
911 bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource,
912 bool usePCMAudio, unsigned profileIdc, unsigned levelIdc,
913 unsigned constraintSet, size_t *numInputBuffers) {
914 CHECK(!usePCMAudio || !isVideo);
915 CHECK(!isRepeaterSource || isVideo);
916 CHECK(!profileIdc || isVideo);
917 CHECK(!levelIdc || isVideo);
918 CHECK(!constraintSet || isVideo);
919
920 sp<ALooper> pullLooper = new ALooper;
921 pullLooper->setName("pull_looper");
922
923 pullLooper->start(
924 false /* runOnCallingThread */,
925 false /* canCallJava */,
926 PRIORITY_AUDIO);
927
928 sp<ALooper> codecLooper = new ALooper;
929 codecLooper->setName("codec_looper");
930
931 codecLooper->start(
932 false /* runOnCallingThread */,
933 false /* canCallJava */,
934 PRIORITY_AUDIO);
935
936 size_t trackIndex;
937
938 sp<AMessage> notify;
939
940 trackIndex = mTracks.size();
941
942 sp<AMessage> format;
943 status_t err = convertMetaDataToMessage(source->getFormat(), &format);
944 CHECK_EQ(err, (status_t)OK);
945
946 if (isVideo) {
947 format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC);
948 format->setInt32(
949 "android._input-metadata-buffer-type", kMetadataBufferTypeANWBuffer);
950 format->setInt32("android._store-metadata-in-buffers-output", (mHDCP != NULL)
951 && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE));
952 format->setInt32(
953 "color-format", OMX_COLOR_FormatAndroidOpaque);
954 format->setInt32("profile-idc", profileIdc);
955 format->setInt32("level-idc", levelIdc);
956 format->setInt32("constraint-set", constraintSet);
957 } else {
958 if (usePCMAudio) {
959 format->setInt32("pcm-encoding", kAudioEncodingPcm16bit);
960 format->setString("mime", MEDIA_MIMETYPE_AUDIO_RAW);
961 } else {
962 format->setString("mime", MEDIA_MIMETYPE_AUDIO_AAC);
963 }
964 }
965
966 notify = new AMessage(kWhatConverterNotify, this);
967 notify->setSize("trackIndex", trackIndex);
968
969 sp<Converter> converter = new Converter(notify, codecLooper, format);
970
971 looper()->registerHandler(converter);
972
973 err = converter->init();
974 if (err != OK) {
975 ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err);
976
977 looper()->unregisterHandler(converter->id());
978 return err;
979 }
980
981 notify = new AMessage(Converter::kWhatMediaPullerNotify, converter);
982 notify->setSize("trackIndex", trackIndex);
983
984 sp<MediaPuller> puller = new MediaPuller(source, notify);
985 pullLooper->registerHandler(puller);
986
987 if (numInputBuffers != NULL) {
988 *numInputBuffers = converter->getInputBufferCount();
989 }
990
991 notify = new AMessage(kWhatTrackNotify, this);
992 notify->setSize("trackIndex", trackIndex);
993
994 sp<Track> track = new Track(
995 notify, pullLooper, codecLooper, puller, converter);
996
997 if (isRepeaterSource) {
998 track->setRepeaterSource(static_cast<RepeaterSource *>(source.get()));
999 }
1000
1001 looper()->registerHandler(track);
1002
1003 mTracks.add(trackIndex, track);
1004
1005 if (isVideo) {
1006 mVideoTrackIndex = trackIndex;
1007 }
1008
1009 uint32_t flags = 0;
1010 if (converter->needToManuallyPrependSPSPPS()) {
1011 flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
1012 }
1013
1014 ssize_t mediaSenderTrackIndex =
1015 mMediaSender->addTrack(converter->getOutputFormat(), flags);
1016 CHECK_GE(mediaSenderTrackIndex, 0);
1017
1018 track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
1019
1020 return OK;
1021 }
1022
addVideoSource(VideoFormats::ResolutionType videoResolutionType,size_t videoResolutionIndex,VideoFormats::ProfileType videoProfileType,VideoFormats::LevelType videoLevelType)1023 status_t WifiDisplaySource::PlaybackSession::addVideoSource(
1024 VideoFormats::ResolutionType videoResolutionType,
1025 size_t videoResolutionIndex,
1026 VideoFormats::ProfileType videoProfileType,
1027 VideoFormats::LevelType videoLevelType) {
1028 size_t width, height, framesPerSecond;
1029 bool interlaced;
1030 CHECK(VideoFormats::GetConfiguration(
1031 videoResolutionType,
1032 videoResolutionIndex,
1033 &width,
1034 &height,
1035 &framesPerSecond,
1036 &interlaced));
1037
1038 unsigned profileIdc, levelIdc, constraintSet;
1039 CHECK(VideoFormats::GetProfileLevel(
1040 videoProfileType,
1041 videoLevelType,
1042 &profileIdc,
1043 &levelIdc,
1044 &constraintSet));
1045
1046 sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height);
1047
1048 source->setUseAbsoluteTimestamps();
1049
1050 sp<RepeaterSource> videoSource =
1051 new RepeaterSource(source, framesPerSecond);
1052
1053 size_t numInputBuffers;
1054 status_t err = addSource(
1055 true /* isVideo */, videoSource, true /* isRepeaterSource */,
1056 false /* usePCMAudio */, profileIdc, levelIdc, constraintSet,
1057 &numInputBuffers);
1058
1059 if (err != OK) {
1060 return err;
1061 }
1062
1063 err = source->setMaxAcquiredBufferCount(numInputBuffers);
1064 CHECK_EQ(err, (status_t)OK);
1065
1066 mProducer = source->getProducer();
1067
1068 return OK;
1069 }
1070
addAudioSource(bool usePCMAudio)1071 status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) {
1072 sp<AudioSource> audioSource = new AudioSource(
1073 AUDIO_SOURCE_REMOTE_SUBMIX,
1074 mOpPackageName,
1075 48000 /* sampleRate */,
1076 2 /* channelCount */);
1077
1078 if (audioSource->initCheck() == OK) {
1079 return addSource(
1080 false /* isVideo */, audioSource, false /* isRepeaterSource */,
1081 usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */,
1082 0 /* constraintSet */, NULL /* numInputBuffers */);
1083 }
1084
1085 ALOGW("Unable to instantiate audio source");
1086
1087 return OK;
1088 }
1089
getSurfaceTexture()1090 sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() {
1091 return mProducer;
1092 }
1093
requestIDRFrame()1094 void WifiDisplaySource::PlaybackSession::requestIDRFrame() {
1095 for (size_t i = 0; i < mTracks.size(); ++i) {
1096 const sp<Track> &track = mTracks.valueAt(i);
1097
1098 track->requestIDRFrame();
1099 }
1100 }
1101
notifySessionDead()1102 void WifiDisplaySource::PlaybackSession::notifySessionDead() {
1103 // Inform WifiDisplaySource of our premature death (wish).
1104 sp<AMessage> notify = mNotify->dup();
1105 notify->setInt32("what", kWhatSessionDead);
1106 notify->post();
1107
1108 mWeAreDead = true;
1109 }
1110
1111 } // namespace android
1112
1113