1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef NUPLAYER_RENDERER_H_ 18 19 #define NUPLAYER_RENDERER_H_ 20 21 #include <atomic> 22 23 #include <media/AudioResamplerPublic.h> 24 #include <media/AVSyncSettings.h> 25 26 #include "NuPlayer.h" 27 28 namespace android { 29 30 class AWakeLock; 31 struct MediaClock; 32 class MediaCodecBuffer; 33 struct VideoFrameSchedulerBase; 34 35 struct NuPlayer::Renderer : public AHandler { 36 enum Flags { 37 FLAG_REAL_TIME = 1, 38 FLAG_OFFLOAD_AUDIO = 2, 39 }; 40 Renderer(const sp<MediaPlayerBase::AudioSink> &sink, 41 const sp<MediaClock> &mediaClock, 42 const sp<AMessage> ¬ify, 43 uint32_t flags = 0); 44 45 static size_t AudioSinkCallback( 46 MediaPlayerBase::AudioSink *audioSink, 47 void *data, size_t size, void *me, 48 MediaPlayerBase::AudioSink::cb_event_t event); 49 50 void queueBuffer( 51 bool audio, 52 const sp<MediaCodecBuffer> &buffer, 53 const sp<AMessage> ¬ifyConsumed); 54 55 void queueEOS(bool audio, status_t finalResult); 56 57 status_t setPlaybackSettings(const AudioPlaybackRate &rate /* sanitized */); 58 status_t getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */); 59 status_t setSyncSettings(const AVSyncSettings &sync, float videoFpsHint); 60 status_t getSyncSettings(AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */); 61 62 void flush(bool audio, bool notifyComplete); 63 64 void signalTimeDiscontinuity(); 65 66 void signalDisableOffloadAudio(); 67 void signalEnableOffloadAudio(); 68 69 void pause(); 70 void resume(); 71 72 void setVideoFrameRate(float fps); 73 74 status_t getCurrentPosition(int64_t *mediaUs); 75 int64_t getVideoLateByUs(); 76 77 status_t openAudioSink( 78 const sp<AMessage> &format, 79 bool offloadOnly, 80 bool hasVideo, 81 uint32_t flags, 82 bool *isOffloaded, 83 bool isStreaming); 84 void closeAudioSink(); 85 86 // re-open audio sink after all pending audio buffers played. 87 void changeAudioFormat( 88 const sp<AMessage> &format, 89 bool offloadOnly, 90 bool hasVideo, 91 uint32_t flags, 92 bool isStreaming, 93 const sp<AMessage> ¬ify); 94 95 enum { 96 kWhatEOS = 'eos ', 97 kWhatFlushComplete = 'fluC', 98 kWhatPosition = 'posi', 99 kWhatVideoRenderingStart = 'vdrd', 100 kWhatMediaRenderingStart = 'mdrd', 101 kWhatAudioTearDown = 'adTD', 102 kWhatAudioOffloadPauseTimeout = 'aOPT', 103 kWhatReleaseWakeLock = 'adRL', 104 }; 105 106 enum AudioTearDownReason { 107 kDueToError = 0, // Could restart with either offload or non-offload. 108 kDueToTimeout, 109 kForceNonOffload, // Restart only with non-offload. 110 }; 111 112 protected: 113 virtual ~Renderer(); 114 115 virtual void onMessageReceived(const sp<AMessage> &msg); 116 117 private: 118 enum { 119 kWhatDrainAudioQueue = 'draA', 120 kWhatDrainVideoQueue = 'draV', 121 kWhatPostDrainVideoQueue = 'pDVQ', 122 kWhatQueueBuffer = 'queB', 123 kWhatQueueEOS = 'qEOS', 124 kWhatConfigPlayback = 'cfPB', 125 kWhatConfigSync = 'cfSy', 126 kWhatGetPlaybackSettings = 'gPbS', 127 kWhatGetSyncSettings = 'gSyS', 128 kWhatFlush = 'flus', 129 kWhatPause = 'paus', 130 kWhatResume = 'resm', 131 kWhatOpenAudioSink = 'opnA', 132 kWhatCloseAudioSink = 'clsA', 133 kWhatChangeAudioFormat = 'chgA', 134 kWhatStopAudioSink = 'stpA', 135 kWhatDisableOffloadAudio = 'noOA', 136 kWhatEnableOffloadAudio = 'enOA', 137 kWhatSetVideoFrameRate = 'sVFR', 138 }; 139 140 // if mBuffer != nullptr, it's a buffer containing real data. 141 // else if mNotifyConsumed == nullptr, it's EOS. 142 // else it's a tag for re-opening audio sink in different format. 143 struct QueueEntry { 144 sp<MediaCodecBuffer> mBuffer; 145 sp<AMessage> mMeta; 146 sp<AMessage> mNotifyConsumed; 147 size_t mOffset; 148 status_t mFinalResult; 149 int32_t mBufferOrdinal; 150 }; 151 152 static const int64_t kMinPositionUpdateDelayUs; 153 154 sp<MediaPlayerBase::AudioSink> mAudioSink; 155 bool mUseVirtualAudioSink; 156 sp<AMessage> mNotify; 157 Mutex mLock; 158 uint32_t mFlags; 159 List<QueueEntry> mAudioQueue; 160 List<QueueEntry> mVideoQueue; 161 uint32_t mNumFramesWritten; 162 sp<VideoFrameSchedulerBase> mVideoScheduler; 163 164 bool mDrainAudioQueuePending; 165 bool mDrainVideoQueuePending; 166 int32_t mAudioQueueGeneration; 167 int32_t mVideoQueueGeneration; 168 int32_t mAudioDrainGeneration; 169 int32_t mVideoDrainGeneration; 170 int32_t mAudioEOSGeneration; 171 172 const sp<MediaClock> mMediaClock; 173 float mPlaybackRate; // audio track rate 174 175 AudioPlaybackRate mPlaybackSettings; 176 AVSyncSettings mSyncSettings; 177 float mVideoFpsHint; 178 179 int64_t mAudioFirstAnchorTimeMediaUs; 180 int64_t mAnchorTimeMediaUs; 181 int64_t mAnchorNumFramesWritten; 182 int64_t mVideoLateByUs; 183 int64_t mNextVideoTimeMediaUs; 184 bool mHasAudio; 185 bool mHasVideo; 186 187 bool mNotifyCompleteAudio; 188 bool mNotifyCompleteVideo; 189 190 bool mSyncQueues; 191 192 // modified on only renderer's thread. 193 bool mPaused; 194 int64_t mPauseDrainAudioAllowedUs; // time when we can drain/deliver audio in pause mode. 195 196 bool mVideoSampleReceived; 197 bool mVideoRenderingStarted; 198 int32_t mVideoRenderingStartGeneration; 199 int32_t mAudioRenderingStartGeneration; 200 bool mRenderingDataDelivered; 201 202 int64_t mNextAudioClockUpdateTimeUs; 203 // the media timestamp of last audio sample right before EOS. 204 int64_t mLastAudioMediaTimeUs; 205 206 int32_t mAudioOffloadPauseTimeoutGeneration; 207 bool mAudioTornDown; 208 audio_offload_info_t mCurrentOffloadInfo; 209 210 struct PcmInfo { 211 audio_channel_mask_t mChannelMask; 212 audio_output_flags_t mFlags; 213 audio_format_t mFormat; 214 int32_t mNumChannels; 215 int32_t mSampleRate; 216 }; 217 PcmInfo mCurrentPcmInfo; 218 static const PcmInfo AUDIO_PCMINFO_INITIALIZER; 219 220 int32_t mTotalBuffersQueued; 221 int32_t mLastAudioBufferDrained; 222 bool mUseAudioCallback; 223 224 sp<AWakeLock> mWakeLock; 225 226 std::atomic_flag mSyncFlag = ATOMIC_FLAG_INIT; 227 Mutex mSyncLock; 228 Condition mSyncCondition; 229 int64_t mSyncCount{0}; 230 231 status_t getCurrentPositionOnLooper(int64_t *mediaUs); 232 status_t getCurrentPositionOnLooper( 233 int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo = false); 234 bool getCurrentPositionIfPaused_l(int64_t *mediaUs); 235 status_t getCurrentPositionFromAnchor( 236 int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo = false); 237 238 void notifyEOSCallback(); 239 size_t fillAudioBuffer(void *buffer, size_t size); 240 241 bool onDrainAudioQueue(); 242 void drainAudioQueueUntilLastEOS(); 243 int64_t getPendingAudioPlayoutDurationUs(int64_t nowUs); 244 void postDrainAudioQueue_l(int64_t delayUs = 0); 245 246 void clearAnchorTime(); 247 void clearAudioFirstAnchorTime_l(); 248 void setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs); 249 void setVideoLateByUs(int64_t lateUs); 250 251 void onNewAudioMediaTime(int64_t mediaTimeUs); 252 int64_t getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs); 253 254 void onDrainVideoQueue(); 255 void postDrainVideoQueue(); 256 257 void prepareForMediaRenderingStart_l(); 258 void notifyIfMediaRenderingStarted_l(); 259 260 void onQueueBuffer(const sp<AMessage> &msg); 261 void onQueueEOS(const sp<AMessage> &msg); 262 void onFlush(const sp<AMessage> &msg); 263 void onAudioSinkChanged(); 264 void onDisableOffloadAudio(); 265 void onEnableOffloadAudio(); 266 status_t onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */); 267 status_t onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */); 268 status_t onConfigSync(const AVSyncSettings &sync, float videoFpsHint); 269 status_t onGetSyncSettings(AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */); 270 271 void onPause(); 272 void onResume(); 273 void onSetVideoFrameRate(float fps); 274 int32_t getQueueGeneration(bool audio); 275 int32_t getDrainGeneration(bool audio); 276 bool getSyncQueues(); 277 void onAudioTearDown(AudioTearDownReason reason); 278 status_t onOpenAudioSink( 279 const sp<AMessage> &format, 280 bool offloadOnly, 281 bool hasVideo, 282 uint32_t flags, 283 bool isStreaming); 284 void onCloseAudioSink(); 285 void onChangeAudioFormat(const sp<AMessage> &meta, const sp<AMessage> ¬ify); 286 287 void notifyEOS(bool audio, status_t finalResult, int64_t delayUs = 0); 288 void notifyEOS_l(bool audio, status_t finalResult, int64_t delayUs = 0); 289 void notifyFlushComplete(bool audio); 290 void notifyPosition(); 291 void notifyVideoLateBy(int64_t lateByUs); 292 void notifyVideoRenderingStart(); 293 void notifyAudioTearDown(AudioTearDownReason reason); 294 295 void flushQueue(List<QueueEntry> *queue); 296 bool dropBufferIfStale(bool audio, const sp<AMessage> &msg); 297 void syncQueuesDone_l(); 298 offloadingAudioRenderer299 bool offloadingAudio() const { return (mFlags & FLAG_OFFLOAD_AUDIO) != 0; } 300 301 void startAudioOffloadPauseTimeout(); 302 void cancelAudioOffloadPauseTimeout(); 303 304 int64_t getDurationUsIfPlayedAtSampleRate(uint32_t numFrames); 305 306 DISALLOW_EVIL_CONSTRUCTORS(Renderer); 307 }; 308 309 } // namespace android 310 311 #endif // NUPLAYER_RENDERER_H_ 312