1 /* 2 * Copyright (c) 2023-2025 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef HISTREAMER_AUDIO_SINK_H 17 #define HISTREAMER_AUDIO_SINK_H 18 #include <mutex> 19 #include "common/status.h" 20 #include "meta/meta.h" 21 #include "sink/media_synchronous_sink.h" 22 #include "media_sync_manager.h" 23 #include "buffer/avbuffer_queue.h" 24 #include "buffer/avbuffer_queue_define.h" 25 #include "plugin/audio_sink_plugin.h" 26 #include "filter/filter.h" 27 #include "plugin/plugin_time.h" 28 #include "performance_utils.h" 29 #include <queue> 30 #include "audio_errors.h" 31 32 namespace OHOS { 33 namespace Media { 34 using namespace OHOS::Media::Plugins; 35 36 class AudioSink : public std::enable_shared_from_this<AudioSink>, public Pipeline::MediaSynchronousSink { 37 public: 38 AudioSink(); 39 ~AudioSink(); 40 Status Init(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver); 41 sptr<AVBufferQueueProducer> GetBufferQueueProducer(); 42 sptr<AVBufferQueueConsumer> GetBufferQueueConsumer(); 43 Status SetParameter(const std::shared_ptr<Meta>& meta); 44 Status GetParameter(std::shared_ptr<Meta>& meta); 45 Status Prepare(); 46 Status Start(); 47 Status Stop(); 48 Status Pause(); 49 Status Resume(); 50 Status Flush(); 51 Status Release(); 52 Status SetPlayRange(int64_t start, int64_t end); 53 Status SetVolume(float volume); 54 Status SetVolumeMode(int32_t mode); 55 void DrainOutputBuffer(bool flushed); 56 void SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver>& receiver); 57 Status GetLatency(uint64_t& nanoSec); 58 void SetSyncCenter(std::shared_ptr<Pipeline::MediaSyncManager> syncCenter); 59 int64_t DoSyncWrite(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer) override; 60 void ResetSyncInfo() override; 61 Status SetSpeed(float speed); 62 Status SetAudioEffectMode(int32_t effectMode); 63 Status GetAudioEffectMode(int32_t &effectMode); 64 int32_t SetVolumeWithRamp(float targetVolume, int32_t duration); 65 void SetThreadGroupId(const std::string& groupId); 66 Status SetIsTransitent(bool isTransitent); 67 Status ChangeTrack(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver); 68 Status SetMuted(bool isMuted); 69 virtual void OnInterrupted(bool isInterruptNeeded) override; 70 71 float GetMaxAmplitude(); 72 int32_t SetMaxAmplitudeCbStatus(bool status); 73 Status SetPerfRecEnabled(bool isPerfRecEnabled); 74 75 static const int64_t kMinAudioClockUpdatePeriodUs = 20 * HST_USECOND; 76 77 static const int64_t kMaxAllowedAudioSinkDelayUs = 1500 * HST_MSECOND; 78 HasPlugin()79 bool HasPlugin() const 80 { 81 return plugin_ != nullptr; 82 } 83 IsInitialized()84 bool IsInitialized() const 85 { 86 return state_ == Pipeline::FilterState::INITIALIZED; 87 } 88 Status SetSeekTime(int64_t seekTime); 89 bool NeedImmediateRender(); 90 bool GetSyncCenterClockTime(int64_t &clockTime); 91 Status SetIsCalledBySystemApp(bool isCalledBySystemApp); 92 Status SetLooping(bool loop); 93 bool IsInputBufferDataEnough(int32_t size, bool isAudioVivid); 94 bool CopyDataToBufferDesc(size_t size, bool isAudioVivid, AudioStandard::BufferDesc &bufferDesc); 95 Status GetBufferDesc(AudioStandard::BufferDesc &bufferDesc); 96 Status EnqueueBufferDesc(const AudioStandard::BufferDesc &bufferDesc); 97 void SyncWriteByRenderInfo(); 98 void UpdateRenderInfo(); 99 void UpdateAmplitude(); 100 bool IsTimeAnchorNeedUpdate(); 101 bool IsBufferAvailable(std::shared_ptr<AVBuffer> &buffer, size_t &cacheBufferSize); 102 bool IsBufferDataDrained(AudioStandard::BufferDesc &bufferDesc, std::shared_ptr<AVBuffer> &buffer, 103 size_t &size, size_t &cacheBufferSize, bool isAudioVivid, int64_t &bufferPts); 104 void ReleaseCacheBuffer(bool isSwapBuffer = false); 105 int64_t CalculateBufferDuration(int64_t writeDataSize); 106 void WriteDataToRender(std::shared_ptr<AVBuffer> &filledOutputBuffer); 107 void ResetInfo(); 108 bool IsEosBuffer(std::shared_ptr<AVBuffer> &filledOutputBuffer); 109 void HandleEosBuffer(std::shared_ptr<AVBuffer> &filledOutputBuffer); 110 bool HandleAudioRenderRequest(size_t size, bool isAudioVivid, AudioStandard::BufferDesc &bufferDesc); 111 void HandleAudioRenderRequestPost(); 112 113 protected: 114 std::atomic<OHOS::Media::Pipeline::FilterState> state_; 115 private: 116 Status PrepareInputBufferQueue(); 117 std::shared_ptr<Plugins::AudioSinkPlugin> CreatePlugin(); 118 bool OnNewAudioMediaTime(int64_t mediaTimeUs); 119 int64_t getPendingAudioPlayoutDurationUs(int64_t nowUs); 120 int64_t getDurationUsPlayedAtSampleRate(uint32_t numFrames); 121 void UpdateAudioWriteTimeMayWait(); 122 bool UpdateTimeAnchorIfNeeded(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer); 123 void DrainAndReportEosEvent(); 124 void HandleEosInner(bool drain); 125 void CalcMaxAmplitude(std::shared_ptr<AVBuffer> filledOutputBuffer); 126 void CheckUpdateState(char *frame, uint64_t replyBytes, int32_t format); 127 bool DropApeBuffer(std::shared_ptr<AVBuffer> filledOutputBuffer); 128 int64_t CalcBufferDuration(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer); 129 void PerfRecord(int64_t audioWriteMs); 130 void ClearInputBuffer(); 131 int32_t GetSampleFormatBytes(); 132 bool CopyBufferData(AudioStandard::BufferDesc &bufferDesc, std::shared_ptr<AVBuffer> &buffer, 133 size_t &size, size_t &cacheBufferSize, int64_t &bufferPts); 134 bool CopyAudioVividBufferData(AudioStandard::BufferDesc &bufferDesc, std::shared_ptr<AVBuffer> &buffer, 135 size_t &size, size_t &cacheBufferSize, int64_t &bufferPts); 136 Status InitAudioSinkPlugin(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver); 137 Status InitAudioSinkInfo(std::shared_ptr<Meta>& meta); 138 Status SetAudioSinkPluginParameters(); 139 void GetAvailableOutputBuffers(); 140 void ClearAvailableOutputBuffers(); 141 void DriveBufferCircle(); 142 std::shared_ptr<AVBuffer> CopyBuffer(const std::shared_ptr<AVBuffer> buffer); 143 144 class UnderrunDetector { 145 public: 146 void DetectAudioUnderrun(int64_t clkTime, int64_t latency); 147 void SetEventReceiver(std::weak_ptr<Pipeline::EventReceiver> eventReceiver); 148 void UpdateBufferTimeNoLock(int64_t clkTime, int64_t latency); 149 void SetLastAudioBufferDuration(int64_t durationUs); 150 void Reset(); 151 private: 152 std::weak_ptr<Pipeline::EventReceiver> eventReceiver_; 153 Mutex mutex_ {}; 154 int64_t lastClkTime_ {HST_TIME_NONE}; 155 int64_t lastLatency_ {HST_TIME_NONE}; 156 int64_t lastBufferDuration_ {HST_TIME_NONE}; 157 }; 158 159 class AudioLagDetector : public Pipeline::LagDetector { 160 public: 161 void Reset() override; 162 163 bool CalcLag(std::shared_ptr<AVBuffer> buffer) override; 164 SetLatency(int64_t latency)165 void SetLatency(int64_t latency) 166 { 167 latency_ = latency; 168 } 169 170 struct AudioDrainTimeGroup { 171 int64_t lastAnchorPts = 0; 172 int64_t anchorDuration = 0; 173 int64_t writeDuration = 0; 174 int64_t nowClockTime = 0; 175 176 AudioDrainTimeGroup() = default; AudioDrainTimeGroupAudioDrainTimeGroup177 AudioDrainTimeGroup(int64_t anchorPts, int64_t duration, int64_t writeDuration, int64_t clockTime) 178 : lastAnchorPts(anchorPts), 179 anchorDuration(duration), 180 writeDuration(writeDuration), 181 nowClockTime(clockTime) {} 182 }; 183 184 void UpdateDrainTimeGroup(AudioDrainTimeGroup group); 185 private: 186 int64_t latency_ = 0; 187 int64_t lastDrainTimeMs_ = 0; 188 AudioDrainTimeGroup lastDrainTimeGroup_ {}; 189 }; 190 191 class AudioSinkDataCallbackImpl : public AudioSinkDataCallback { 192 public: 193 explicit AudioSinkDataCallbackImpl(std::shared_ptr<AudioSink> sink); 194 void OnWriteData(int32_t size, bool isAudioVivid) override; 195 private: 196 std::weak_ptr<AudioSink> audioSink_; 197 }; 198 std::shared_ptr<Plugins::AudioSinkPlugin> plugin_ {}; 199 std::shared_ptr<Pipeline::EventReceiver> playerEventReceiver_; 200 int32_t appUid_{0}; 201 int32_t appPid_{0}; 202 int64_t numFramesWritten_ {0}; 203 int64_t firstAudioAnchorTimeMediaUs_ {HST_TIME_NONE}; 204 int64_t nextAudioClockUpdateTimeUs_ {HST_TIME_NONE}; 205 int64_t lastAnchorClockTime_ {HST_TIME_NONE}; 206 int64_t latestBufferPts_ {HST_TIME_NONE}; 207 int64_t latestBufferDuration_ {0}; 208 int64_t bufferDurationSinceLastAnchor_ {0}; 209 std::atomic<bool> forceUpdateTimeAnchorNextTime_ {true}; 210 const std::string INPUT_BUFFER_QUEUE_NAME = "AudioSinkInputBufferQueue"; 211 std::shared_ptr<AVBufferQueue> inputBufferQueue_; 212 sptr<AVBufferQueueProducer> inputBufferQueueProducer_; 213 sptr<AVBufferQueueConsumer> inputBufferQueueConsumer_; 214 int64_t firstPts_ {HST_TIME_NONE}; 215 int32_t sampleRate_ {0}; 216 int32_t samplePerFrame_ {0}; 217 int32_t audioChannelCount_ = 0; 218 int64_t fixDelay_ {0}; 219 bool isTransitent_ {false}; 220 bool isEos_ {false}; 221 std::mutex pluginMutex_; 222 float volume_ {-1.0f}; 223 std::unique_ptr<Task> eosTask_ {nullptr}; 224 enum class EosInterruptState : int { 225 NONE, 226 INITIAL, 227 PAUSE, 228 RESUME, 229 STOP, 230 }; 231 Mutex eosMutex_ {}; 232 std::atomic<bool> eosDraining_ {false}; 233 std::atomic<EosInterruptState> eosInterruptType_ {EosInterruptState::NONE}; 234 float speed_ {1.0f}; 235 int32_t effectMode_ {-1}; 236 bool isApe_ {false}; 237 bool isFlac_ {false}; 238 int64_t playRangeStartTime_ = -1; 239 int64_t playRangeEndTime_ = -1; 240 // vars for audio progress optimization 241 int64_t playingBufferDurationUs_ {0}; 242 int64_t lastBufferWriteTime_ {0}; 243 bool lastBufferWriteSuccess_ {true}; 244 bool isMuted_ = false; 245 Mutex amplitudeMutex_ {}; 246 float maxAmplitude_ = 0; 247 float currentMaxAmplitude_ {0}; 248 249 bool calMaxAmplitudeCbStatus_ = false; 250 UnderrunDetector underrunDetector_; 251 AudioLagDetector lagDetector_; 252 std::atomic<int64_t> seekTimeUs_ {HST_TIME_NONE}; 253 PerfRecorder perfRecorder_ {}; 254 bool isPerfRecEnabled_ { false }; 255 bool isCalledBySystemApp_ { false }; 256 bool isLoop_ { false }; 257 bool isCallbackMode_ {true}; 258 std::shared_ptr<AudioSinkDataCallback> audioSinkDataCallback_ {nullptr}; 259 std::mutex availBufferMutex_; 260 std::atomic<size_t> availDataSize_ {0}; 261 std::atomic<size_t> remainingDataSize_ {0}; 262 std::queue<std::shared_ptr<AVBuffer>> availOutputBuffers_; 263 int32_t currentQueuedBufferOffset_ {0}; 264 bool isEosBuffer_ {false}; 265 std::mutex eosCbMutex_ {}; 266 bool hangeOnEosCb_ {false}; 267 std::condition_variable eosCbCond_ {}; 268 class AudioDataSynchroizer { 269 public: 270 void UpdateCurrentBufferInfo(int64_t bufferPts, int64_t bufferDuration); 271 int64_t GetLastReportedClockTime() const; 272 int64_t GetLastBufferPTS() const; 273 int64_t GetBufferDuration() const; 274 int64_t CalculateAudioLatency(); 275 void UpdateReportTime(int64_t nowClockTime); 276 void UpdateLastBufferPTS(int64_t bufferOffset, float speed); 277 void OnRenderPositionUpdated(int64_t currentRenderPTS, int64_t currentRenderClockTime); 278 void Reset(); 279 private: 280 int64_t lastBufferPTS_ {HST_TIME_NONE}; 281 int64_t startPTS_ {HST_TIME_NONE}; 282 int64_t curBufferPTS_ {HST_TIME_NONE}; 283 int64_t bufferDuration_ {0}; 284 int64_t currentRenderClockTime_ {0}; 285 int64_t currentRenderPTS_ {0}; 286 int64_t lastReportedClockTime_ {HST_TIME_NONE}; 287 int64_t lastBufferOffset_ {0}; 288 int64_t compensateDuration_ {0}; 289 int64_t sumDuration_ {0}; 290 }; 291 std::unique_ptr<AudioDataSynchroizer> innerSynchroizer_ = std::make_unique<AudioDataSynchroizer>(); 292 MemoryType bufferMemoryType_ {MemoryType::UNKNOWN_MEMORY}; 293 int32_t maxCbDataSize_ {0}; 294 std::queue<std::shared_ptr<AVBuffer>> swapOutputBuffers_ {}; 295 }; 296 } 297 } 298 299 #endif // HISTREAMER_AUDIO_SINK_H 300