1 /* 2 * Copyright (c) 2023-2025 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef HISTREAMER_AUDIO_SINK_H 17 #define HISTREAMER_AUDIO_SINK_H 18 #include <mutex> 19 #include "common/status.h" 20 #include "meta/meta.h" 21 #include "sink/media_synchronous_sink.h" 22 #include "media_sync_manager.h" 23 #include "buffer/avbuffer_queue.h" 24 #include "buffer/avbuffer_queue_define.h" 25 #include "plugin/audio_sink_plugin.h" 26 #include "filter/filter.h" 27 #include "plugin/plugin_time.h" 28 #include "performance_utils.h" 29 #include <queue> 30 #include "audio_errors.h" 31 32 namespace OHOS { 33 namespace Media { 34 using namespace OHOS::Media::Plugins; 35 36 class AudioSink : public std::enable_shared_from_this<AudioSink>, public Pipeline::MediaSynchronousSink { 37 public: 38 AudioSink(); 39 AudioSink(bool isRenderCallbackMode, bool isProcessInputMerged); 40 ~AudioSink(); 41 Status Init(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver); 42 sptr<AVBufferQueueProducer> GetBufferQueueProducer(); 43 sptr<AVBufferQueueConsumer> GetBufferQueueConsumer(); 44 Status SetParameter(const std::shared_ptr<Meta>& meta); 45 Status GetParameter(std::shared_ptr<Meta>& meta); 46 Status Prepare(); 47 Status Start(); 48 Status Stop(); 49 Status Pause(); 50 Status Freeze(); 51 Status UnFreeze(); 52 Status Resume(); 53 Status Flush(); 54 Status Release(); 55 Status SetPlayRange(int64_t start, int64_t end); 56 Status SetVolume(float volume); 57 Status SetVolumeMode(int32_t mode); 58 void DrainOutputBuffer(bool flushed); 59 void SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver>& receiver); 60 Status GetLatency(uint64_t& nanoSec); 61 void SetSyncCenter(std::shared_ptr<Pipeline::MediaSyncManager> syncCenter); 62 int64_t DoSyncWrite(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer) override; 63 void ResetSyncInfo() override; 64 Status SetSpeed(float speed); 65 Status SetAudioEffectMode(int32_t effectMode); 66 Status GetAudioEffectMode(int32_t &effectMode); 67 int32_t SetVolumeWithRamp(float targetVolume, int32_t duration); 68 void SetThreadGroupId(const std::string& groupId); 69 Status SetIsTransitent(bool isTransitent); 70 Status ChangeTrack(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver); 71 Status HandleFormatChange(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver); 72 Status SetMuted(bool isMuted); 73 virtual void OnInterrupted(bool isInterruptNeeded) override; 74 75 float GetMaxAmplitude(); 76 int32_t SetMaxAmplitudeCbStatus(bool status); 77 Status SetPerfRecEnabled(bool isPerfRecEnabled); 78 79 static const int64_t kMinAudioClockUpdatePeriodUs = 20 * HST_USECOND; 80 81 static const int64_t kMaxAllowedAudioSinkDelayUs = 1500 * HST_MSECOND; 82 HasPlugin()83 bool HasPlugin() const 84 { 85 return plugin_ != nullptr; 86 } 87 IsInitialized()88 bool IsInitialized() const 89 { 90 return state_ == Pipeline::FilterState::INITIALIZED; 91 } 92 Status SetSeekTime(int64_t seekTime); NeedImmediateRender()93 inline bool NeedImmediateRender() const 94 { 95 return isApe_ || isFlac_; 96 } 97 void SetIsInPrePausing(bool isInPrePausing); SetIsAudioDemuxDecodeAsync(bool isAudioDemuxDecodeAsync)98 inline void SetIsAudioDemuxDecodeAsync(bool isAudioDemuxDecodeAsync) 99 { 100 isAudioDemuxDecodeAsync_ = isAudioDemuxDecodeAsync; 101 } 102 bool GetSyncCenterClockTime(int64_t &clockTime); 103 Status SetIsCalledBySystemApp(bool isCalledBySystemApp); 104 Status SetLooping(bool loop); 105 bool IsInputBufferDataEnough(int32_t size, bool isAudioVivid); 106 Status GetBufferDesc(AudioStandard::BufferDesc &bufferDesc); 107 Status EnqueueBufferDesc(const AudioStandard::BufferDesc &bufferDesc); 108 bool HandleAudioRenderRequest(size_t size, bool isAudioVivid, AudioStandard::BufferDesc &bufferDesc); 109 void HandleAudioRenderRequestPost(); 110 Status SetAudioHapticsSyncId(int32_t syncId); 111 112 protected: 113 std::atomic<OHOS::Media::Pipeline::FilterState> state_; 114 private: 115 Status PrepareInputBufferQueue(); 116 std::shared_ptr<Plugins::AudioSinkPlugin> CreatePlugin(); 117 bool OnNewAudioMediaTime(int64_t mediaTimeUs); 118 int64_t getPendingAudioPlayoutDurationUs(int64_t nowUs); 119 int64_t getDurationUsPlayedAtSampleRate(uint32_t numFrames); 120 bool CopyDataToBufferDesc(size_t size, bool isAudioVivid, AudioStandard::BufferDesc &bufferDesc); 121 void SyncWriteByRenderInfo(); 122 void UpdateRenderInfo(); 123 void UpdateAmplitude(); 124 bool IsTimeAnchorNeedUpdate(); 125 bool IsBufferAvailable(std::shared_ptr<AVBuffer> &buffer, size_t &cacheBufferSize); 126 bool IsBufferDataDrained(AudioStandard::BufferDesc &bufferDesc, std::shared_ptr<AVBuffer> &buffer, 127 size_t &size, size_t &cacheBufferSize, bool isAudioVivid, int64_t &bufferPts); 128 void ReleaseCacheBuffer(bool isSwapBuffer = false); 129 int64_t CalculateBufferDuration(int64_t writeDataSize); 130 void WriteDataToRender(std::shared_ptr<AVBuffer> &filledOutputBuffer); 131 void ResetInfo(); 132 bool IsEosBuffer(std::shared_ptr<AVBuffer> &filledOutputBuffer); 133 void HandleEosBuffer(std::shared_ptr<AVBuffer> &filledOutputBuffer); 134 void UpdateAudioWriteTimeMayWait(); 135 bool UpdateTimeAnchorIfNeeded(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer); 136 void DrainAndReportEosEvent(); 137 void HandleEosInner(bool drain); 138 void CalcMaxAmplitude(std::shared_ptr<AVBuffer> filledOutputBuffer); 139 void CheckUpdateState(char *frame, uint64_t replyBytes, int32_t format); 140 bool DropApeBuffer(std::shared_ptr<AVBuffer> filledOutputBuffer); 141 int64_t CalcBufferDuration(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer); 142 void PerfRecord(int64_t audioWriteMs); 143 void ClearInputBuffer(); 144 int32_t GetSampleFormatBytes(); 145 bool CopyBufferData(AudioStandard::BufferDesc &bufferDesc, std::shared_ptr<AVBuffer> &buffer, 146 size_t &size, size_t &cacheBufferSize, int64_t &bufferPts); 147 bool CopyAudioVividBufferData(AudioStandard::BufferDesc &bufferDesc, std::shared_ptr<AVBuffer> &buffer, 148 size_t &size, size_t &cacheBufferSize, int64_t &bufferPts); 149 Status InitAudioSinkPlugin(const std::shared_ptr<Meta>& meta, 150 const std::shared_ptr<Pipeline::EventReceiver>& receiver, 151 const std::shared_ptr<Plugins::AudioSinkPlugin>& plugin); 152 Status InitAudioSinkInfo(std::shared_ptr<Meta>& meta); 153 Status SetAudioSinkPluginParameters(const std::shared_ptr<Plugins::AudioSinkPlugin>& plugin); 154 std::shared_ptr<Plugins::AudioSinkPlugin> PreCreateAndStartNewPlugin(const std::shared_ptr<Meta>& meta, 155 const std::shared_ptr<Pipeline::EventReceiver>& receiver); 156 void FlushForChangeTrack(); 157 Status ChangeTrackForFormatChange(); 158 void GetAvailableOutputBuffers(); 159 void ClearAvailableOutputBuffers(); 160 void DriveBufferCircle(); 161 void WaitForAllBufferConsumed(); 162 std::shared_ptr<AVBuffer> CopyBuffer(const std::shared_ptr<AVBuffer> buffer); 163 Status MuteAudioBuffer(size_t size, AudioStandard::BufferDesc &bufferDesc, bool isEos); 164 165 class UnderrunDetector { 166 public: 167 void DetectAudioUnderrun(int64_t clkTime, int64_t latency); 168 void SetEventReceiver(std::weak_ptr<Pipeline::EventReceiver> eventReceiver); 169 void UpdateBufferTimeNoLock(int64_t clkTime, int64_t latency); 170 void SetLastAudioBufferDuration(int64_t durationUs); 171 void Reset(); 172 private: 173 std::weak_ptr<Pipeline::EventReceiver> eventReceiver_; 174 Mutex mutex_ {}; 175 int64_t lastClkTime_ {HST_TIME_NONE}; 176 int64_t lastLatency_ {HST_TIME_NONE}; 177 int64_t lastBufferDuration_ {HST_TIME_NONE}; 178 }; 179 180 class AudioLagDetector : public Pipeline::LagDetector { 181 public: 182 void Reset() override; 183 184 bool CalcLag(std::shared_ptr<AVBuffer> buffer) override; 185 SetLatency(int64_t latency)186 void SetLatency(int64_t latency) 187 { 188 latency_ = latency; 189 } 190 191 struct AudioDrainTimeGroup { 192 int64_t lastAnchorPts = 0; 193 int64_t anchorDuration = 0; 194 int64_t writeDuration = 0; 195 int64_t nowClockTime = 0; 196 197 AudioDrainTimeGroup() = default; AudioDrainTimeGroupAudioDrainTimeGroup198 AudioDrainTimeGroup(int64_t anchorPts, int64_t duration, int64_t writeDuration, int64_t clockTime) 199 : lastAnchorPts(anchorPts), 200 anchorDuration(duration), 201 writeDuration(writeDuration), 202 nowClockTime(clockTime) {} 203 }; 204 205 void UpdateDrainTimeGroup(AudioDrainTimeGroup group); 206 private: 207 int64_t latency_ = 0; 208 int64_t lastDrainTimeMs_ = 0; 209 AudioDrainTimeGroup lastDrainTimeGroup_ {}; 210 }; 211 212 class AudioSinkDataCallbackImpl : public AudioSinkDataCallback { 213 public: 214 explicit AudioSinkDataCallbackImpl(std::shared_ptr<AudioSink> sink); 215 void OnWriteData(int32_t size, bool isAudioVivid) override; 216 private: 217 std::weak_ptr<AudioSink> audioSink_; 218 }; 219 std::shared_ptr<Plugins::AudioSinkPlugin> plugin_ {}; 220 std::shared_ptr<Pipeline::EventReceiver> playerEventReceiver_; 221 int32_t appUid_{0}; 222 int32_t appPid_{0}; 223 int64_t numFramesWritten_ {0}; 224 int64_t firstAudioAnchorTimeMediaUs_ {HST_TIME_NONE}; 225 int64_t nextAudioClockUpdateTimeUs_ {HST_TIME_NONE}; 226 int64_t lastAnchorClockTime_ {HST_TIME_NONE}; 227 int64_t latestBufferPts_ {HST_TIME_NONE}; 228 int64_t latestBufferDuration_ {0}; 229 int64_t bufferDurationSinceLastAnchor_ {0}; 230 std::atomic<bool> forceUpdateTimeAnchorNextTime_ {true}; 231 const std::string INPUT_BUFFER_QUEUE_NAME = "AudioSinkInputBufferQueue"; 232 std::shared_ptr<AVBufferQueue> inputBufferQueue_; 233 sptr<AVBufferQueueProducer> inputBufferQueueProducer_; 234 sptr<AVBufferQueueConsumer> inputBufferQueueConsumer_; 235 int64_t firstPts_ {HST_TIME_NONE}; 236 int32_t sampleRate_ {0}; 237 int32_t samplePerFrame_ {0}; 238 int32_t audioChannelCount_ = 0; 239 int64_t fixDelay_ {0}; 240 bool isTransitent_ {false}; 241 bool isEos_ {false}; 242 std::mutex pluginMutex_; 243 float volume_ {-1.0f}; 244 std::unique_ptr<Task> eosTask_ {nullptr}; 245 enum class EosInterruptState : int { 246 NONE, 247 INITIAL, 248 PAUSE, 249 RESUME, 250 STOP, 251 }; 252 Mutex eosMutex_ {}; 253 std::atomic<bool> eosDraining_ {false}; 254 std::atomic<EosInterruptState> eosInterruptType_ {EosInterruptState::NONE}; 255 float speed_ {1.0f}; 256 int32_t effectMode_ {-1}; 257 bool isApe_ {false}; 258 bool isFlac_ {false}; 259 int64_t playRangeStartTime_ = -1; 260 int64_t playRangeEndTime_ = -1; 261 // vars for audio progress optimization 262 int64_t playingBufferDurationUs_ {0}; 263 int64_t lastBufferWriteTime_ {0}; 264 bool lastBufferWriteSuccess_ {true}; 265 bool isMuted_ = false; 266 Mutex amplitudeMutex_ {}; 267 float maxAmplitude_ = 0; 268 float currentMaxAmplitude_ {0}; 269 270 bool calMaxAmplitudeCbStatus_ = false; 271 UnderrunDetector underrunDetector_; 272 AudioLagDetector lagDetector_; 273 std::atomic<int64_t> seekTimeUs_ {HST_TIME_NONE}; 274 PerfRecorder perfRecorder_ {}; 275 bool isPerfRecEnabled_ { false }; 276 bool isCalledBySystemApp_ { false }; 277 bool isLoop_ { false }; 278 bool isRenderCallbackMode_ {true}; 279 bool isProcessInputMerged_ {true}; 280 bool isAudioDemuxDecodeAsync_ {true}; 281 std::atomic<bool> isInPrePausing_ {false}; 282 std::shared_ptr<AudioSinkDataCallback> audioSinkDataCallback_ {nullptr}; 283 std::mutex availBufferMutex_; 284 std::atomic<size_t> availDataSize_ {0}; 285 std::atomic<size_t> remainingDataSize_ {0}; 286 std::queue<std::shared_ptr<AVBuffer>> availOutputBuffers_; 287 int32_t currentQueuedBufferOffset_ {0}; 288 bool isEosBuffer_ {false}; 289 std::mutex eosCbMutex_ {}; 290 bool hangeOnEosCb_ {false}; 291 std::condition_variable eosCbCond_ {}; 292 293 std::atomic<bool> formatChange_ {false}; 294 std::mutex formatChangeMutex_ {}; 295 std::condition_variable formatChangeCond_ {}; 296 std::unique_ptr<Task> changeTrackTask_ {nullptr}; 297 std::shared_ptr<Plugins::AudioSinkPlugin> newPlugin_ {}; 298 std::atomic<bool> hasPluginCreateTaskFinished_ {false}; 299 std::mutex preCreatePluginMutex_ {}; 300 std::condition_variable preCreatePluginCond_ {}; 301 class AudioDataSynchroizer { 302 public: 303 void UpdateCurrentBufferInfo(int64_t bufferPts, int64_t bufferDuration); 304 int64_t GetLastReportedClockTime() const; 305 int64_t GetLastBufferPTS() const; 306 int64_t GetBufferDuration() const; 307 int64_t CalculateAudioLatency(); 308 void UpdateReportTime(int64_t nowClockTime); 309 void UpdateLastBufferPTS(int64_t bufferOffset, float speed); 310 void OnRenderPositionUpdated(int64_t currentRenderPTS, int64_t currentRenderClockTime); 311 void Reset(); 312 private: 313 int64_t lastBufferPTS_ {HST_TIME_NONE}; 314 int64_t startPTS_ {HST_TIME_NONE}; 315 int64_t curBufferPTS_ {HST_TIME_NONE}; 316 int64_t bufferDuration_ {0}; 317 int64_t currentRenderClockTime_ {0}; 318 int64_t currentRenderPTS_ {0}; 319 int64_t lastReportedClockTime_ {HST_TIME_NONE}; 320 int64_t lastBufferOffset_ {0}; 321 int64_t compensateDuration_ {0}; 322 int64_t sumDuration_ {0}; 323 }; 324 std::unique_ptr<AudioDataSynchroizer> innerSynchroizer_ = std::make_unique<AudioDataSynchroizer>(); 325 MemoryType bufferMemoryType_ {MemoryType::UNKNOWN_MEMORY}; 326 int32_t maxCbDataSize_ {0}; 327 std::queue<std::shared_ptr<AVBuffer>> swapOutputBuffers_ {}; 328 }; 329 } 330 } 331 332 #endif // HISTREAMER_AUDIO_SINK_H 333