1 /*
2 * Copyright (c) 2023-2025 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "audio_sink.h"
17 #include "avcodec_trace.h"
18 #include "syspara/parameters.h"
19 #include "plugin/plugin_manager_v2.h"
20 #include "common/log.h"
21 #include "calc_max_amplitude.h"
22 #include "scoped_timer.h"
23 #include "avcodec_info.h"
24
25 namespace {
26 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "AudioSink" };
27 constexpr int64_t MAX_BUFFER_DURATION_US = 200000; // Max buffer duration is 200 ms
28 constexpr int64_t US_TO_MS = 1000; // 1000 us per ms
29 constexpr int64_t ANCHOR_UPDATE_PERIOD_US = 200000; // Update time anchor every 200 ms
30 constexpr int64_t DRAIN_TIME_DIFF_WARN_MS = 40;
31 constexpr int64_t DRAIN_TIME_DIFF_INFO_MS = 20;
32 constexpr int64_t AUDIO_SAMPLE_8_BIT = 1;
33 constexpr int64_t AUDIO_SAMPLE_16_BIT = 2;
34 constexpr int64_t AUDIO_SAMPLE_24_BIT = 3;
35 constexpr int64_t AUDIO_SAMPLE_32_BIT = 4;
36 constexpr int64_t SEC_TO_US = 1000 * 1000;
37 constexpr int64_t EOS_CALLBACK_WAIT_MS = 500;
38 constexpr int32_t BOOT_APP_UID = 1003;
39 constexpr int64_t INIT_PLUGIN_WARNING_MS = 20;
40 constexpr int64_t OVERTIME_WARNING_MS = 50;
41 constexpr int64_t FORMAT_CHANGE_MS = 100;
42 constexpr int64_t BUFFER_CONSUME_MS = 50;
43 constexpr int64_t FIX_DELAY_MS_AUDIO_VIVID = 80;
44 }
45
46 namespace OHOS {
47 namespace Media {
48
49 const int32_t DEFAULT_BUFFER_QUEUE_SIZE = 8;
50 const int32_t APE_BUFFER_QUEUE_SIZE = 30;
51 const int64_t DEFAULT_PLAY_RANGE_VALUE = -1;
52 const int64_t MICROSECONDS_CONVERT_UNITS = 1000;
53
GetAudioLatencyFixDelay()54 int64_t GetAudioLatencyFixDelay()
55 {
56 constexpr int64_t defaultOffset = -1;
57 const std::string audioLatencyOffsetKey = "const.multimedia.audio.latency_offset";
58 static int64_t audioLatencyOffset = OHOS::system::GetIntParameter(audioLatencyOffsetKey, defaultOffset);
59 MEDIA_LOG_I("audio.latency_offset, pid:%{public}d , offset: " PUBLIC_LOG_D64, getprocpid(), audioLatencyOffset);
60 FALSE_RETURN_V_NOLOG(audioLatencyOffset < 0, audioLatencyOffset * HST_USECOND);
61
62 constexpr uint64_t defaultValue = 120 * HST_USECOND;
63 static uint64_t fixDelay = OHOS::system::GetUintParameter("debug.media_service.audio_sync_fix_delay", defaultValue);
64 MEDIA_LOG_I("audio_sync_fix_delay, pid:%{public}d, fixdelay: " PUBLIC_LOG_U64, getprocpid(), fixDelay);
65 return static_cast<int64_t>(fixDelay);
66 }
67
AudioSink()68 AudioSink::AudioSink()
69 {
70 bool isRenderCallbackMode =
71 OHOS::system::GetParameter("debug.media_service.audio.audiosink_callback", "1") == "1";
72 bool isProcessInputMerged =
73 OHOS::system::GetParameter("debug.media_service.audio.audiosink_processinput_merged", "1") == "1";
74 MEDIA_LOG_I("AudioSink ctor isRenderCallbackMode: " PUBLIC_LOG_D32 ", isProcessInputMerged: " PUBLIC_LOG_D32,
75 isRenderCallbackMode, isProcessInputMerged);
76 isRenderCallbackMode_ = isRenderCallbackMode;
77 isProcessInputMerged_ = isProcessInputMerged;
78 syncerPriority_ = IMediaSynchronizer::AUDIO_SINK;
79 fixDelay_ = GetAudioLatencyFixDelay();
80 plugin_ = CreatePlugin();
81 }
82
AudioSink(bool isRenderCallbackMode,bool isProcessInputMerged)83 AudioSink::AudioSink(bool isRenderCallbackMode, bool isProcessInputMerged)
84 : isRenderCallbackMode_(isRenderCallbackMode), isProcessInputMerged_(isProcessInputMerged)
85 {
86 MEDIA_LOG_I("AudioSink ctor default isRenderCallbackMode: " PUBLIC_LOG_D32
87 ", isProcessInputMerged: " PUBLIC_LOG_D32, isRenderCallbackMode, isProcessInputMerged);
88 syncerPriority_ = IMediaSynchronizer::AUDIO_SINK;
89 fixDelay_ = GetAudioLatencyFixDelay();
90 plugin_ = CreatePlugin();
91 }
92
~AudioSink()93 AudioSink::~AudioSink()
94 {
95 MEDIA_LOG_D("AudioSink dtor");
96 }
97
AudioSinkDataCallbackImpl(std::shared_ptr<AudioSink> sink)98 AudioSink::AudioSinkDataCallbackImpl::AudioSinkDataCallbackImpl(std::shared_ptr<AudioSink> sink): audioSink_(sink) {}
99
OnWriteData(int32_t size,bool isAudioVivid)100 void AudioSink::AudioSinkDataCallbackImpl::OnWriteData(int32_t size, bool isAudioVivid)
101 {
102 auto sink = audioSink_.lock();
103 FALSE_RETURN_MSG(sink != nullptr, "audioSink_ is nullptr");
104 AudioStandard::BufferDesc bufferDesc;
105 MediaAVCodec::AVCodecTrace trace("AudioSink::OnWriteData");
106 MEDIA_LOG_DD("GetBufferDesc in");
107 Status ret = sink->GetBufferDesc(bufferDesc);
108 FALSE_RETURN_MSG(ret == Status::OK, "GetBufferDesc fail, ret=" PUBLIC_LOG_D32, ret);
109 bufferDesc.dataLength = 0;
110
111 if (sink->IsInputBufferDataEnough(size, isAudioVivid)) {
112 bool isCopySucess = sink->HandleAudioRenderRequest(static_cast<size_t>(size),
113 isAudioVivid, bufferDesc);
114 bufferDesc.dataLength = isCopySucess ? bufferDesc.dataLength : 0;
115 }
116 ret = sink->EnqueueBufferDesc(bufferDesc);
117 sink->HandleAudioRenderRequestPost();
118 FALSE_RETURN_MSG(ret == Status::OK, "enqueue failed, ret=" PUBLIC_LOG_D32, ret);
119 }
120
HandleAudioRenderRequest(size_t size,bool isAudioVivid,AudioStandard::BufferDesc & bufferDesc)121 bool AudioSink::HandleAudioRenderRequest(size_t size, bool isAudioVivid, AudioStandard::BufferDesc &bufferDesc)
122 {
123 FALSE_RETURN_V(!eosDraining_, false);
124 bool isCopySucess = CopyDataToBufferDesc(static_cast<size_t>(size), isAudioVivid, bufferDesc);
125 FALSE_RETURN_V_MSG_D(isCopySucess, false, "CopyDataToBufferDesc failed");
126 UpdateAudioWriteTimeMayWait();
127 SyncWriteByRenderInfo();
128 UpdateAmplitude();
129 return true;
130 }
131
HandleAudioRenderRequestPost()132 void AudioSink::HandleAudioRenderRequestPost()
133 {
134 std::lock_guard<std::mutex> lock(availBufferMutex_);
135 if (appUid_ == BOOT_APP_UID && isEosBuffer_ && availOutputBuffers_.empty()) {
136 std::unique_lock<std::mutex> eosCbLock(eosCbMutex_);
137 hangeOnEosCb_ = true;
138 eosCbCond_.wait_for(eosCbLock, std::chrono::milliseconds(EOS_CALLBACK_WAIT_MS),
139 [this] () { return !hangeOnEosCb_; });
140 }
141 FALSE_RETURN_NOLOG(isEosBuffer_);
142 FALSE_RETURN_NOLOG(!availOutputBuffers_.empty());
143 auto cacheBuffer = availOutputBuffers_.front();
144 FALSE_RETURN(cacheBuffer != nullptr);
145 FALSE_RETURN_NOLOG(IsEosBuffer(cacheBuffer));
146 availOutputBuffers_.pop();
147 HandleEosBuffer(cacheBuffer);
148 }
149
GetBufferDesc(AudioStandard::BufferDesc & bufferDesc)150 Status AudioSink::GetBufferDesc(AudioStandard::BufferDesc &bufferDesc)
151 {
152 FALSE_RETURN_V_MSG(plugin_ != nullptr, Status::ERROR_UNKNOWN, "GetBufferDesc audioSinkPlugin is nullptr");
153 MEDIA_TRACE_DEBUG("AudioSink::GetBufferDesc");
154 return plugin_->GetBufferDesc(bufferDesc);
155 }
156
EnqueueBufferDesc(const AudioStandard::BufferDesc & bufferDesc)157 Status AudioSink::EnqueueBufferDesc(const AudioStandard::BufferDesc &bufferDesc)
158 {
159 FALSE_RETURN_V_MSG(plugin_ != nullptr, Status::ERROR_UNKNOWN, "Enqueue audioSinkPlugin is nullptr");
160 MEDIA_TRACE_DEBUG("AudioSink::EnqueueBufferDesc");
161 return plugin_->EnqueueBufferDesc(bufferDesc);
162 }
163
IsInputBufferDataEnough(int32_t size,bool isAudioVivid)164 bool AudioSink::IsInputBufferDataEnough(int32_t size, bool isAudioVivid)
165 {
166 std::lock_guard<std::mutex> lock(availBufferMutex_);
167 if (!isAudioVivid) {
168 maxCbDataSize_ = std::max(maxCbDataSize_, size);
169 } else {
170 // audioVivid use min(size, availDataSize) as maxDataSize to ensure that audioVivid dont use swapBuffers
171 size_t availDataSize = availDataSize_.load();
172 int32_t availDataSizeInt32 = availDataSize <= static_cast<size_t>(INT32_MAX) ?
173 static_cast<int32_t>(availDataSize): INT32_MAX;
174 maxCbDataSize_ = std::min(size, availDataSizeInt32);
175 }
176 DriveBufferCircle();
177 std::unique_lock<std::mutex> formatLock(formatChangeMutex_);
178 return availDataSize_.load() >= static_cast<size_t>(size) || isEosBuffer_ || formatChange_.load();
179 }
180
Init(std::shared_ptr<Meta> & meta,const std::shared_ptr<Pipeline::EventReceiver> & receiver)181 Status AudioSink::Init(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver)
182 {
183 state_ = Pipeline::FilterState::INITIALIZED;
184 Status ret = InitAudioSinkPlugin(meta, receiver, plugin_);
185 FALSE_RETURN_V(ret == Status::OK, ret);
186 ret = InitAudioSinkInfo(meta);
187 FALSE_RETURN_V(ret == Status::OK, ret);
188 return Status::OK;
189 }
190
InitAudioSinkPlugin(const std::shared_ptr<Meta> & meta,const std::shared_ptr<Pipeline::EventReceiver> & receiver,const std::shared_ptr<Plugins::AudioSinkPlugin> & plugin)191 Status AudioSink::InitAudioSinkPlugin(const std::shared_ptr<Meta>& meta,
192 const std::shared_ptr<Pipeline::EventReceiver>& receiver,
193 const std::shared_ptr<Plugins::AudioSinkPlugin>& plugin)
194 {
195 FALSE_RETURN_V(plugin != nullptr, Status::ERROR_NULL_POINTER);
196 FALSE_RETURN_V(meta != nullptr, Status::ERROR_NULL_POINTER);
197 meta->SetData(Tag::APP_PID, appPid_);
198 meta->SetData(Tag::APP_UID, appUid_);
199 plugin->SetEventReceiver(receiver);
200 plugin->SetParameter(meta);
201 {
202 ScopedTimer timer("InitAudioSinkPlugin", INIT_PLUGIN_WARNING_MS);
203 plugin->Init();
204 }
205 if (isRenderCallbackMode_) {
206 if (audioSinkDataCallback_ == nullptr) {
207 audioSinkDataCallback_ = std::make_shared<AudioSinkDataCallbackImpl>(shared_from_this());
208 }
209 Status ret = plugin->SetRequestDataCallback(audioSinkDataCallback_);
210 isRenderCallbackMode_ = ret == Status::OK ? true : false;
211 }
212 plugin->Prepare();
213 plugin->SetMuted(isMuted_);
214 return Status::OK;
215 }
216
InitAudioSinkInfo(std::shared_ptr<Meta> & meta)217 Status AudioSink::InitAudioSinkInfo(std::shared_ptr<Meta>& meta)
218 {
219 FALSE_RETURN_V(meta != nullptr, Status::ERROR_NULL_POINTER);
220 meta->GetData(Tag::AUDIO_SAMPLE_RATE, sampleRate_);
221 meta->GetData(Tag::AUDIO_SAMPLE_PER_FRAME, samplePerFrame_);
222 meta->GetData(Tag::AUDIO_CHANNEL_COUNT, audioChannelCount_);
223 if (samplePerFrame_ > 0 && sampleRate_ > 0) {
224 playingBufferDurationUs_ = static_cast<int64_t>(samplePerFrame_) * SEC_TO_US / sampleRate_;
225 }
226 MEDIA_LOG_I("Audiosink playingBufferDurationUs_ = " PUBLIC_LOG_D64, playingBufferDurationUs_);
227 std::string mime;
228 if (!meta->Get<Tag::MIME_TYPE>(mime)) {
229 return Status::OK;
230 }
231 if (mime == MediaAVCodec::CodecMimeType::AUDIO_APE) {
232 isApe_ = true;
233 MEDIA_LOG_I("AudioSink::Init is ape");
234 } else if (mime == MediaAVCodec::CodecMimeType::AUDIO_FLAC) {
235 isFlac_ = true;
236 MEDIA_LOG_I("AudioSink::Init is flac");
237 } else if (mime == MediaAVCodec::CodecMimeType::AUDIO_VIVID) {
238 fixDelay_ = FIX_DELAY_MS_AUDIO_VIVID * HST_USECOND;
239 MEDIA_LOG_I("Audio vivid update fix delay to" PUBLIC_LOG_D64, fixDelay_);
240 }
241 return Status::OK;
242 }
243
GetBufferQueueProducer()244 sptr<AVBufferQueueProducer> AudioSink::GetBufferQueueProducer()
245 {
246 if (state_ != Pipeline::FilterState::READY) {
247 return nullptr;
248 }
249 return inputBufferQueueProducer_;
250 }
251
GetBufferQueueConsumer()252 sptr<AVBufferQueueConsumer> AudioSink::GetBufferQueueConsumer()
253 {
254 if (state_ != Pipeline::FilterState::READY) {
255 return nullptr;
256 }
257 return inputBufferQueueConsumer_;
258 }
259
SetParameter(const std::shared_ptr<Meta> & meta)260 Status AudioSink::SetParameter(const std::shared_ptr<Meta>& meta)
261 {
262 UpdateMediaTimeRange(meta);
263 FALSE_RETURN_V(meta != nullptr, Status::ERROR_NULL_POINTER);
264 meta->GetData(Tag::APP_PID, appPid_);
265 meta->GetData(Tag::APP_UID, appUid_);
266 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
267 plugin_->SetParameter(meta);
268 return Status::OK;
269 }
270
GetParameter(std::shared_ptr<Meta> & meta)271 Status AudioSink::GetParameter(std::shared_ptr<Meta>& meta)
272 {
273 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
274 return plugin_->GetParameter(meta);
275 }
276
Prepare()277 Status AudioSink::Prepare()
278 {
279 state_ = Pipeline::FilterState::PREPARING;
280 Status ret = PrepareInputBufferQueue();
281 if (ret != Status::OK) {
282 state_ = Pipeline::FilterState::INITIALIZED;
283 return ret;
284 }
285 state_ = Pipeline::FilterState::READY;
286 {
287 AutoLock lock(eosMutex_);
288 eosInterruptType_ = EosInterruptState::NONE;
289 eosDraining_ = false;
290 }
291 return ret;
292 }
293
Start()294 Status AudioSink::Start()
295 {
296 Status ret = Status::OK;
297 {
298 ScopedTimer timer("AudioSinkPlugin Start", OVERTIME_WARNING_MS);
299 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
300 ret = plugin_->Start();
301 }
302 if (ret != Status::OK) {
303 MEDIA_LOG_I("AudioSink start error " PUBLIC_LOG_D32, ret);
304 return ret;
305 }
306 isEos_ = false;
307 state_ = Pipeline::FilterState::RUNNING;
308 FALSE_RETURN_V_NOLOG(playerEventReceiver_ != nullptr, Status::OK);
309 playerEventReceiver_->OnMemoryUsageEvent({"AUDIO_SINK_BQ",
310 DfxEventType::DFX_INFO_MEMORY_USAGE, inputBufferQueue_->GetMemoryUsage()});
311 return ret;
312 }
313
Stop()314 Status AudioSink::Stop()
315 {
316 std::lock_guard<std::mutex> lockPlugin(pluginMutex_);
317 playRangeStartTime_ = DEFAULT_PLAY_RANGE_VALUE;
318 playRangeEndTime_ = DEFAULT_PLAY_RANGE_VALUE;
319 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
320 Status ret = plugin_->Stop();
321 underrunDetector_.Reset();
322 lagDetector_.Reset();
323 ResetInfo();
324 if (ret != Status::OK) {
325 return ret;
326 }
327 state_ = Pipeline::FilterState::INITIALIZED;
328 AutoLock lock(eosMutex_);
329 if (eosInterruptType_ != EosInterruptState::NONE) {
330 eosInterruptType_ = EosInterruptState::STOP;
331 }
332 return ret;
333 }
334
Pause()335 Status AudioSink::Pause()
336 {
337 Status ret = Status::OK;
338 underrunDetector_.Reset();
339 lagDetector_.Reset();
340 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
341 if (appUid_ == BOOT_APP_UID) {
342 if (eosTask_ != nullptr) {
343 eosTask_->SubmitJobOnce([this] {
344 ScopedTimer timer("AudioSinkPlugin Pause BOOT", OVERTIME_WARNING_MS);
345 {
346 std::unique_lock<std::mutex> eosCbLock(eosCbMutex_);
347 hangeOnEosCb_ = false;
348 eosCbCond_.notify_all();
349 }
350 plugin_->PauseTransitent();
351 });
352 }
353 } else if (isTransitent_ || (isEos_ && isLoop_)) {
354 ScopedTimer timer("AudioSinkPlugin PauseTransitent", OVERTIME_WARNING_MS);
355 ret = plugin_->PauseTransitent();
356 } else {
357 ScopedTimer timer("AudioSinkPlugin Pause", OVERTIME_WARNING_MS);
358 ret = plugin_->Pause();
359 }
360 forceUpdateTimeAnchorNextTime_ = true;
361 if (ret != Status::OK) {
362 return ret;
363 }
364 state_ = Pipeline::FilterState::PAUSED;
365 AutoLock lock(eosMutex_);
366 if (eosInterruptType_ == EosInterruptState::INITIAL || eosInterruptType_ == EosInterruptState::RESUME) {
367 eosInterruptType_ = EosInterruptState::PAUSE;
368 }
369 return ret;
370 }
371
Resume()372 Status AudioSink::Resume()
373 {
374 lagDetector_.Reset();
375 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
376 Status ret = plugin_->Resume();
377 if (ret != Status::OK) {
378 MEDIA_LOG_I("AudioSink resume error " PUBLIC_LOG_D32, ret);
379 return ret;
380 }
381 isEos_ = false;
382 state_ = Pipeline::FilterState::RUNNING;
383 AutoLock lock(eosMutex_);
384 if (eosInterruptType_ == EosInterruptState::PAUSE) {
385 eosInterruptType_ = EosInterruptState::RESUME;
386 if (!eosDraining_ && eosTask_ != nullptr) {
387 eosTask_->SubmitJobOnce([this] {
388 HandleEosInner(false);
389 });
390 }
391 }
392 return ret;
393 }
394
Flush()395 Status AudioSink::Flush()
396 {
397 std::lock_guard<std::mutex> lockPlugin(pluginMutex_);
398 MEDIA_LOG_D("do audioSink flush");
399 underrunDetector_.Reset();
400 lagDetector_.Reset();
401
402 {
403 AutoLock lock(eosMutex_);
404 eosInterruptType_ = EosInterruptState::NONE;
405 eosDraining_ = false;
406 }
407 Status ret = Status::OK;
408 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
409 ret = plugin_->Flush();
410 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "plugin flush failed");
411 ResetInfo();
412 return Status::OK;
413 }
414
FlushForChangeTrack()415 void AudioSink::FlushForChangeTrack()
416 {
417 MEDIA_LOG_D("do audioSink flush for change track");
418 underrunDetector_.Reset();
419 lagDetector_.Reset();
420
421 {
422 AutoLock lock(eosMutex_);
423 eosInterruptType_ = EosInterruptState::NONE;
424 eosDraining_ = false;
425 }
426 ResetInfo();
427 }
428
Release()429 Status AudioSink::Release()
430 {
431 underrunDetector_.Reset();
432 lagDetector_.Reset();
433 ResetInfo();
434 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
435 return plugin_->Deinit();
436 }
437
SetPlayRange(int64_t start,int64_t end)438 Status AudioSink::SetPlayRange(int64_t start, int64_t end)
439 {
440 MEDIA_LOG_I("SetPlayRange enter.");
441 playRangeStartTime_ = start;
442 playRangeEndTime_ = end;
443 return Status::OK;
444 }
445
SetVolumeMode(int32_t mode)446 Status AudioSink::SetVolumeMode(int32_t mode)
447 {
448 if (plugin_ == nullptr) {
449 return Status::ERROR_NULL_POINTER;
450 }
451 return plugin_->SetVolumeMode(mode);
452 }
453
SetVolume(float volume)454 Status AudioSink::SetVolume(float volume)
455 {
456 if (plugin_ == nullptr) {
457 return Status::ERROR_NULL_POINTER;
458 }
459 if (volume < 0) {
460 return Status::ERROR_INVALID_PARAMETER;
461 }
462 volume_ = volume;
463 return plugin_->SetVolume(volume);
464 }
465
SetVolumeWithRamp(float targetVolume,int32_t duration)466 int32_t AudioSink::SetVolumeWithRamp(float targetVolume, int32_t duration)
467 {
468 MEDIA_LOG_I("SetVolumeWithRamp");
469 FALSE_RETURN_V(plugin_ != nullptr, static_cast<int32_t>(Status::ERROR_NULL_POINTER));
470 return plugin_->SetVolumeWithRamp(targetVolume, duration);
471 }
472
SetIsTransitent(bool isTransitent)473 Status AudioSink::SetIsTransitent(bool isTransitent)
474 {
475 MEDIA_LOG_I("SetIsTransitent");
476 isTransitent_ = isTransitent;
477 return Status::OK;
478 }
479
Freeze()480 Status AudioSink::Freeze()
481 {
482 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
483 plugin_->Freeze();
484 return Status::OK;
485 }
486
UnFreeze()487 Status AudioSink::UnFreeze()
488 {
489 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
490 plugin_->UnFreeze();
491 return Status::OK;
492 }
493
PrepareInputBufferQueue()494 Status AudioSink::PrepareInputBufferQueue()
495 {
496 if (inputBufferQueue_ != nullptr && inputBufferQueue_-> GetQueueSize() > 0) {
497 MEDIA_LOG_I("InputBufferQueue already create");
498 return Status::ERROR_INVALID_OPERATION;
499 }
500 int32_t inputBufferSize = isApe_ ? APE_BUFFER_QUEUE_SIZE : DEFAULT_BUFFER_QUEUE_SIZE;
501 MemoryType memoryType = MemoryType::SHARED_MEMORY;
502 #ifndef MEDIA_OHOS
503 memoryType = MemoryType::VIRTUAL_MEMORY;
504 #endif
505 bufferMemoryType_ = memoryType;
506 MEDIA_LOG_I("PrepareInputBufferQueue ");
507 inputBufferQueue_ = AVBufferQueue::Create(inputBufferSize, memoryType, INPUT_BUFFER_QUEUE_NAME);
508 inputBufferQueueProducer_ = inputBufferQueue_->GetProducer();
509 inputBufferQueueConsumer_ = inputBufferQueue_->GetConsumer();
510 return Status::OK;
511 }
512
CreatePlugin()513 std::shared_ptr<Plugins::AudioSinkPlugin> AudioSink::CreatePlugin()
514 {
515 auto plugin = Plugins::PluginManagerV2::Instance().CreatePluginByMime(Plugins::PluginType::AUDIO_SINK, "audio/raw");
516 if (plugin == nullptr) {
517 return nullptr;
518 }
519 return std::reinterpret_pointer_cast<Plugins::AudioSinkPlugin>(plugin);
520 }
521
UpdateAudioWriteTimeMayWait()522 void AudioSink::UpdateAudioWriteTimeMayWait()
523 {
524 if (latestBufferDuration_ <= 0) {
525 return;
526 }
527 if (latestBufferDuration_ > MAX_BUFFER_DURATION_US) {
528 latestBufferDuration_ = MAX_BUFFER_DURATION_US; // wait at most MAX_DURATION
529 }
530 int64_t timeNow = Plugins::HstTime2Us(SteadyClock::GetCurrentTimeNanoSec());
531 if (!lastBufferWriteSuccess_) {
532 int64_t writeSleepTime = latestBufferDuration_ - (timeNow - lastBufferWriteTime_);
533 MEDIA_LOG_W("Last buffer write fail, sleep time is " PUBLIC_LOG_D64 "us", writeSleepTime);
534 if (writeSleepTime > 0) {
535 usleep(writeSleepTime);
536 timeNow = Plugins::HstTime2Us(SteadyClock::GetCurrentTimeNanoSec());
537 }
538 }
539 lastBufferWriteTime_ = timeNow;
540 }
541
SetThreadGroupId(const std::string & groupId)542 void AudioSink::SetThreadGroupId(const std::string& groupId)
543 {
544 eosTask_ = std::make_unique<Task>("OS_EOSa", groupId, TaskType::AUDIO, TaskPriority::HIGH, false);
545 changeTrackTask_ = std::make_unique<Task>("CHANGE_TRACK", groupId, TaskType::AUDIO, TaskPriority::HIGH, false);
546 }
547
HandleEosInner(bool drain)548 void AudioSink::HandleEosInner(bool drain)
549 {
550 AutoLock lock(eosMutex_);
551 eosDraining_ = true; // start draining task
552 switch (eosInterruptType_) {
553 case EosInterruptState::INITIAL: // No user operation during EOS drain, complete drain normally
554 break;
555 case EosInterruptState::RESUME: // EOS drain is resumed after pause, do necessary changes
556 if (drain) {
557 // pause and resume happened before this task, audiosink latency should be updated
558 drain = false;
559 }
560 eosInterruptType_ = EosInterruptState::INITIAL; // Reset EOS draining state
561 break;
562 default: // EOS drain is interrupted by pause or stop, and not resumed
563 MEDIA_LOG_W("Drain audiosink interrupted");
564 eosDraining_ = false; // abort draining task
565 return;
566 }
567 FALSE_RETURN(plugin_ != nullptr);
568 if (drain || !plugin_->IsOffloading()) {
569 MEDIA_LOG_I("Drain audiosink and report EOS");
570 DrainAndReportEosEvent();
571 return;
572 }
573 uint64_t latency = 0;
574 if (plugin_->GetLatency(latency) != Status::OK) {
575 MEDIA_LOG_W("Failed to get latency, drain audiosink directly");
576 DrainAndReportEosEvent();
577 return;
578 }
579 if (eosTask_ == nullptr) {
580 MEDIA_LOG_W("Drain audiosink, eosTask_ is nullptr");
581 DrainAndReportEosEvent();
582 return;
583 }
584 MEDIA_LOG_I("Drain audiosink wait latency = " PUBLIC_LOG_U64, latency);
585 eosTask_->SubmitJobOnce([this] {
586 HandleEosInner(true);
587 }, latency, false);
588 }
589
DrainAndReportEosEvent()590 void AudioSink::DrainAndReportEosEvent()
591 {
592 FALSE_RETURN(plugin_ != nullptr);
593 plugin_->Drain();
594 eosInterruptType_ = EosInterruptState::NONE;
595 eosDraining_ = false; // finish draining task
596 isEos_ = true;
597 auto syncCenter = syncCenter_.lock();
598 if (syncCenter) {
599 syncCenter->ReportEos(this);
600 }
601 Event event {
602 .srcFilter = "AudioSink",
603 .type = EventType::EVENT_COMPLETE,
604 };
605 FALSE_RETURN(playerEventReceiver_ != nullptr);
606 playerEventReceiver_->OnEvent(event);
607 }
608
CheckUpdateState(char * frame,uint64_t replyBytes,int32_t format)609 void AudioSink::CheckUpdateState(char *frame, uint64_t replyBytes, int32_t format)
610 {
611 FALSE_RETURN(frame != nullptr && replyBytes != 0);
612 auto currentMaxAmplitude = OHOS::Media::CalcMaxAmplitude::UpdateMaxAmplitude(frame, replyBytes, format);
613 currentMaxAmplitude_ = currentMaxAmplitude_ > currentMaxAmplitude ? currentMaxAmplitude_ : currentMaxAmplitude;
614 }
615
UpdateAmplitude()616 void AudioSink::UpdateAmplitude()
617 {
618 AutoLock amplitudeLock(amplitudeMutex_);
619 maxAmplitude_ = currentMaxAmplitude_ > maxAmplitude_ ? currentMaxAmplitude_ : maxAmplitude_;
620 }
621
GetMaxAmplitude()622 float AudioSink::GetMaxAmplitude()
623 {
624 AutoLock amplitudeLock(amplitudeMutex_);
625 auto ret = maxAmplitude_;
626 maxAmplitude_ = 0;
627 return ret;
628 }
629
CalcMaxAmplitude(std::shared_ptr<AVBuffer> filledOutputBuffer)630 void AudioSink::CalcMaxAmplitude(std::shared_ptr<AVBuffer> filledOutputBuffer)
631 {
632 FALSE_RETURN(filledOutputBuffer != nullptr);
633 auto mem = filledOutputBuffer->memory_;
634 FALSE_RETURN(mem != nullptr);
635 auto srcBuffer = mem->GetAddr();
636 auto destBuffer = const_cast<uint8_t *>(srcBuffer);
637 auto srcLength = mem->GetSize();
638 size_t destLength = static_cast<size_t>(srcLength);
639 int32_t format = plugin_->GetSampleFormat();
640 CheckUpdateState(reinterpret_cast<char *>(destBuffer), destLength, format);
641 }
642
DropApeBuffer(std::shared_ptr<AVBuffer> filledOutputBuffer)643 bool AudioSink::DropApeBuffer(std::shared_ptr<AVBuffer> filledOutputBuffer)
644 {
645 if (!isApe_ || seekTimeUs_ == HST_TIME_NONE) {
646 return false;
647 }
648 if (filledOutputBuffer->pts_ < seekTimeUs_) {
649 MEDIA_LOG_D("Drop ape buffer pts = " PUBLIC_LOG_D64, filledOutputBuffer->pts_);
650 inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer);
651 return true;
652 } else {
653 seekTimeUs_ = HST_TIME_NONE;
654 }
655 return false;
656 }
657
ClearInputBuffer()658 void AudioSink::ClearInputBuffer()
659 {
660 MEDIA_LOG_D("AudioSink::ClearInputBuffer enter");
661 if (!inputBufferQueueConsumer_) {
662 return;
663 }
664 std::shared_ptr<AVBuffer> filledInputBuffer;
665 Status ret = Status::OK;
666 while (ret == Status::OK) {
667 ret = inputBufferQueueConsumer_->AcquireBuffer(filledInputBuffer);
668 if (ret != Status::OK) {
669 MEDIA_LOG_I("AudioSink::ClearInputBuffer clear input Buffer");
670 return;
671 }
672 inputBufferQueueConsumer_->ReleaseBuffer(filledInputBuffer);
673 }
674 }
675
GetSampleFormatBytes()676 int32_t AudioSink::GetSampleFormatBytes()
677 {
678 int32_t format = 0;
679 FALSE_RETURN_V(plugin_ != nullptr, static_cast<int32_t>(Status::ERROR_NULL_POINTER));
680 switch (plugin_->GetSampleFormat()) {
681 case AudioSampleFormat::SAMPLE_U8:
682 format = AUDIO_SAMPLE_8_BIT;
683 break;
684 case AudioSampleFormat::SAMPLE_S16LE:
685 format = AUDIO_SAMPLE_16_BIT;
686 break;
687 case AudioSampleFormat::SAMPLE_S24LE:
688 format = AUDIO_SAMPLE_24_BIT;
689 break;
690 case AudioSampleFormat::SAMPLE_S32LE:
691 case AudioSampleFormat::SAMPLE_F32LE:
692 format = AUDIO_SAMPLE_32_BIT;
693 break;
694 default:
695 break;
696 }
697 return format;
698 }
699
IsBufferAvailable(std::shared_ptr<AVBuffer> & buffer,size_t & cacheBufferSize)700 bool AudioSink::IsBufferAvailable(std::shared_ptr<AVBuffer> &buffer, size_t &cacheBufferSize)
701 {
702 FALSE_RETURN_V_MSG_D(buffer != nullptr && buffer->memory_ != nullptr, false, "buffer is null.");
703 int32_t bufferSize = buffer->memory_->GetSize();
704 FALSE_RETURN_V_MSG_D(bufferSize >= currentQueuedBufferOffset_, false, "buffer is empty, skip this buffer.");
705 cacheBufferSize = static_cast<size_t>(bufferSize - currentQueuedBufferOffset_);
706 return true;
707 }
708
CopyBufferData(AudioStandard::BufferDesc & bufferDesc,std::shared_ptr<AVBuffer> & buffer,size_t & size,size_t & cacheBufferSize,int64_t & bufferPts)709 bool AudioSink::CopyBufferData(AudioStandard::BufferDesc &bufferDesc, std::shared_ptr<AVBuffer> &buffer,
710 size_t &size, size_t &cacheBufferSize, int64_t &bufferPts)
711 {
712 FALSE_RETURN_V_MSG(bufferDesc.buffer != nullptr, false, "Audio bufferDesc is nullptr");
713 FALSE_RETURN_V_MSG(buffer != nullptr && buffer->memory_ != nullptr, false, "AVBuffer is nullptr");
714 size_t availableSize = cacheBufferSize > size ? size : cacheBufferSize;
715 auto ret = memcpy_s(bufferDesc.buffer + bufferDesc.dataLength, availableSize,
716 buffer->memory_->GetAddr() + currentQueuedBufferOffset_, availableSize);
717 FALSE_RETURN_V_MSG(ret == 0, false, "copy from cache buffer may fail.");
718 bufferPts = (bufferPts == HST_TIME_NONE) ? buffer->pts_ : bufferPts;
719 bufferDesc.dataLength += availableSize;
720 availDataSize_.fetch_sub(availableSize);
721 if (cacheBufferSize > size) {
722 currentQueuedBufferOffset_ += static_cast<int32_t>(size);
723 size = 0;
724 return false;
725 }
726 currentQueuedBufferOffset_ = 0;
727 size -= cacheBufferSize;
728 return true;
729 }
730
CopyAudioVividBufferData(AudioStandard::BufferDesc & bufferDesc,std::shared_ptr<AVBuffer> & buffer,size_t & size,size_t & cacheBufferSize,int64_t & bufferPts)731 bool AudioSink::CopyAudioVividBufferData(AudioStandard::BufferDesc &bufferDesc, std::shared_ptr<AVBuffer> &buffer,
732 size_t &size, size_t &cacheBufferSize, int64_t &bufferPts)
733 {
734 FALSE_RETURN_V_MSG(bufferDesc.buffer != nullptr, false, "Audio bufferDesc is nullptr");
735 FALSE_RETURN_V_MSG(buffer != nullptr && buffer->memory_ != nullptr, false, "AVBuffer is nullptr");
736 auto ret = memcpy_s(bufferDesc.buffer + bufferDesc.dataLength, cacheBufferSize,
737 buffer->memory_->GetAddr() + currentQueuedBufferOffset_, cacheBufferSize);
738 FALSE_RETURN_V_MSG(ret == 0, false, "copy from cache buffer may fail.");
739 bufferPts = (bufferPts == HST_TIME_NONE) ? buffer->pts_ : bufferPts;
740 bufferDesc.dataLength += cacheBufferSize;
741 size -= cacheBufferSize;
742 availDataSize_.fetch_sub(cacheBufferSize);
743 currentQueuedBufferOffset_ = 0;
744 auto meta = buffer->meta_;
745 std::vector<uint8_t> metaData;
746 meta->GetData(Tag::OH_MD_KEY_AUDIO_VIVID_METADATA, metaData);
747 if (metaData.size() == bufferDesc.metaLength && bufferDesc.metaLength > 0) {
748 ret = memcpy_s(bufferDesc.metaBuffer, bufferDesc.metaLength,
749 metaData.data(), bufferDesc.metaLength);
750 FALSE_RETURN_V_MSG(ret == 0, false, "copy from cache buffer may fail.");
751 } else {
752 MEDIA_LOG_E("CopyAudioVividBufferData error: size: " PUBLIC_LOG_ZU " metaLength:" PUBLIC_LOG_ZU,
753 metaData.size(), bufferDesc.metaLength);
754 }
755 return true;
756 }
757
IsBufferDataDrained(AudioStandard::BufferDesc & bufferDesc,std::shared_ptr<AVBuffer> & buffer,size_t & size,size_t & cacheBufferSize,bool isAudioVivid,int64_t & bufferPts)758 bool AudioSink::IsBufferDataDrained(AudioStandard::BufferDesc &bufferDesc, std::shared_ptr<AVBuffer> &buffer,
759 size_t &size, size_t &cacheBufferSize, bool isAudioVivid, int64_t &bufferPts)
760 {
761 FALSE_RETURN_V_MSG(cacheBufferSize <= size || !isAudioVivid, false, "copy from cache buffer may fail.");
762 MEDIA_TRACE_DEBUG("AudioSink::CopyBuffer");
763 bool ret = isAudioVivid ? CopyAudioVividBufferData(bufferDesc, buffer, size, cacheBufferSize, bufferPts) :
764 CopyBufferData(bufferDesc, buffer, size, cacheBufferSize, bufferPts);
765 return ret;
766 }
767
CopyDataToBufferDesc(size_t size,bool isAudioVivid,AudioStandard::BufferDesc & bufferDesc)768 bool AudioSink::CopyDataToBufferDesc(size_t size, bool isAudioVivid, AudioStandard::BufferDesc &bufferDesc)
769 {
770 FALSE_RETURN_V_MSG(size != 0 && size == bufferDesc.bufLength, false,
771 "bufferDesc or request size is unavailable");
772 bufferDesc.dataLength = 0;
773 std::lock_guard<std::mutex> lock(availBufferMutex_);
774 MEDIA_TRACE_DEBUG("AudioSink::CopyDataToBufferDesc");
775 int64_t bufferPts = HST_TIME_NONE;
776 do {
777 bool isSwapBuffer = false;
778 FALSE_RETURN_V_MSG(!swapOutputBuffers_.empty() || !availOutputBuffers_.empty(), false, "buffer queue is empty");
779 std::shared_ptr<AVBuffer> cacheBuffer;
780 if (!swapOutputBuffers_.empty()) {
781 cacheBuffer = swapOutputBuffers_.front();
782 isSwapBuffer = true;
783 } else {
784 cacheBuffer = availOutputBuffers_.front();
785 }
786 if (MuteAudioBuffer(size, bufferDesc, IsEosBuffer(cacheBuffer)) != Status::OK) {
787 break;
788 }
789 size_t cacheBufferSize = 0;
790 if (IsBufferAvailable(cacheBuffer, cacheBufferSize)) {
791 if (!IsBufferDataDrained(bufferDesc, cacheBuffer, size, cacheBufferSize, isAudioVivid, bufferPts)) {
792 break;
793 }
794 CalcMaxAmplitude(cacheBuffer);
795 }
796 ReleaseCacheBuffer(isSwapBuffer);
797 } while (size > 0 && !isAudioVivid);
798 if (bufferPts >= 0) {
799 int64_t bufferDuration = CalculateBufferDuration(bufferDesc.dataLength);
800 innerSynchroizer_->UpdateCurrentBufferInfo(bufferPts, bufferDuration);
801 return true;
802 }
803 return false;
804 }
805
MuteAudioBuffer(size_t size,AudioStandard::BufferDesc & bufferDesc,bool isEos)806 Status AudioSink::MuteAudioBuffer(size_t size, AudioStandard::BufferDesc &bufferDesc, bool isEos)
807 {
808 std::unique_lock<std::mutex> lock(formatChangeMutex_);
809 FALSE_RETURN_V_NOLOG(formatChange_ || isEos, Status::OK);
810 MEDIA_LOG_I("AudioSink mute audio buffer isEos %{public}d size %{public}zu dataLength %{public}zu"
811 "bufferLength %{public}zu", isEos, size, bufferDesc.dataLength, bufferDesc.bufLength);
812 FALSE_RETURN_V_NOLOG(plugin_ && size > 0, Status::ERROR_INVALID_STATE);
813 (void)plugin_->MuteAudioBuffer(bufferDesc.buffer, bufferDesc.dataLength, size);
814 return isEos ? Status::ERROR_INVALID_STATE : Status::OK;
815 }
816
CalculateBufferDuration(int64_t writeDataSize)817 int64_t AudioSink::CalculateBufferDuration(int64_t writeDataSize)
818 {
819 int32_t format = GetSampleFormatBytes();
820 FALSE_RETURN_V(format > 0 && audioChannelCount_ > 0, 0);
821 int64_t sampleDataNums = writeDataSize / format / audioChannelCount_;
822 FALSE_RETURN_V(sampleRate_ > 0, 0);
823 int64_t sampleDataDuration = sampleDataNums * SEC_TO_US / sampleRate_;
824 return sampleDataDuration;
825 }
826
UpdateCurrentBufferInfo(int64_t bufferPts,int64_t bufferDuration)827 void AudioSink::AudioDataSynchroizer::UpdateCurrentBufferInfo(int64_t bufferPts, int64_t bufferDuration)
828 {
829 bufferDuration_ = bufferDuration;
830 startPTS_ = startPTS_ == HST_TIME_NONE ? bufferPts : startPTS_;
831 lastBufferPTS_ = lastBufferPTS_ == HST_TIME_NONE ? bufferPts : lastBufferPTS_;
832 curBufferPTS_ = bufferPts;
833 }
834
UpdateLastBufferPTS(int64_t bufferOffset,float speed)835 void AudioSink::AudioDataSynchroizer::UpdateLastBufferPTS(int64_t bufferOffset, float speed)
836 {
837 MEDIA_LOG_DD("lastBuffer Info: lastBufferPTS_ is " PUBLIC_LOG_D64 " lastBufferOffset_ is " PUBLIC_LOG_D64
838 " and compensateDuration_ is " PUBLIC_LOG_D64, lastBufferPTS_, lastBufferOffset_, compensateDuration_);
839 curBufferPTS_ = curBufferPTS_ == HST_TIME_NONE ? 0 : curBufferPTS_;
840 int64_t tempPTS = curBufferPTS_ + lastBufferOffset_ + bufferDuration_;
841 if (tempPTS < lastBufferPTS_) {
842 MEDIA_LOG_W("audio pts is not increasing, last pts: " PUBLIC_LOG_D64 ", current pts: " PUBLIC_LOG_D64,
843 lastBufferPTS_, tempPTS);
844 }
845 lastBufferPTS_ = tempPTS;
846 lastBufferOffset_ = bufferOffset;
847 sumDuration_ += bufferDuration_;
848 FALSE_RETURN_MSG(speed != 0, "speed is 0");
849 compensateDuration_ += static_cast<int64_t>(bufferDuration_ - bufferDuration_ / speed);
850 }
851
UpdateRenderInfo()852 void AudioSink::UpdateRenderInfo()
853 {
854 timespec time;
855 uint32_t position;
856 MEDIA_TRACE_DEBUG("AudioSink::UpdateRenderInfo");
857 FALSE_RETURN(plugin_ != nullptr);
858 FALSE_RETURN_MSG(plugin_->GetAudioPosition(time, position), "GetAudioPosition from audioRender failed");
859 int64_t currentRenderClockTime = time.tv_sec * SEC_TO_US + time.tv_nsec / US_TO_MS; // convert to us
860 FALSE_RETURN(sampleRate_ > 0);
861 int64_t currentRenderPTS = static_cast<int64_t>(position) * SEC_TO_US / sampleRate_;
862 MEDIA_LOG_DD("currentRenderPTS is " PUBLIC_LOG_D64 " and currentRenderClockTime is " PUBLIC_LOG_D64,
863 currentRenderPTS, currentRenderClockTime);
864 innerSynchroizer_->OnRenderPositionUpdated(currentRenderPTS, currentRenderClockTime);
865 }
866
OnRenderPositionUpdated(int64_t currentRenderPTS,int64_t currentRenderClockTime)867 void AudioSink::AudioDataSynchroizer::OnRenderPositionUpdated(int64_t currentRenderPTS, int64_t currentRenderClockTime)
868 {
869 currentRenderClockTime_ = currentRenderClockTime;
870 currentRenderPTS_ = currentRenderPTS;
871 }
872
CalculateAudioLatency()873 int64_t AudioSink::AudioDataSynchroizer::CalculateAudioLatency()
874 {
875 int64_t latency = 0;
876 if (lastBufferPTS_ == startPTS_) {
877 return latency;
878 }
879 int64_t nowClockTime = Plugins::HstTime2Us(SteadyClock::GetCurrentTimeNanoSec());
880 MEDIA_LOG_DD("PTS diff is " PUBLIC_LOG_D64, ((sumDuration_ - compensateDuration_) -
881 currentRenderPTS_));
882 latency = ((sumDuration_ - compensateDuration_) - currentRenderPTS_) -
883 (nowClockTime - currentRenderClockTime_);
884 FALSE_RETURN_V_MSG(latency >= 0, 0, "calculate latency failed");
885 return latency;
886 }
887
GetLastBufferPTS() const888 int64_t AudioSink::AudioDataSynchroizer::GetLastBufferPTS() const
889 {
890 return lastBufferPTS_;
891 }
892
GetLastReportedClockTime() const893 int64_t AudioSink::AudioDataSynchroizer::GetLastReportedClockTime() const
894 {
895 return lastReportedClockTime_;
896 }
897
GetBufferDuration() const898 int64_t AudioSink::AudioDataSynchroizer::GetBufferDuration() const
899 {
900 return bufferDuration_;
901 }
902
UpdateReportTime(int64_t nowClockTime)903 void AudioSink::AudioDataSynchroizer::UpdateReportTime(int64_t nowClockTime)
904 {
905 lastReportedClockTime_ = nowClockTime;
906 }
907
Reset()908 void AudioSink::AudioDataSynchroizer::Reset()
909 {
910 lastBufferPTS_ = HST_TIME_NONE;
911 bufferDuration_ = 0;
912 currentRenderClockTime_ = 0;
913 currentRenderPTS_ = 0;
914 lastReportedClockTime_ = HST_TIME_NONE;
915 startPTS_ = HST_TIME_NONE;
916 curBufferPTS_ = HST_TIME_NONE;
917 lastBufferOffset_ = 0;
918 compensateDuration_ = 0;
919 sumDuration_ = 0;
920 }
921
IsTimeAnchorNeedUpdate()922 bool AudioSink::IsTimeAnchorNeedUpdate()
923 {
924 auto syncCenter = syncCenter_.lock();
925 FALSE_RETURN_V(syncCenter != nullptr, false);
926 int64_t lastAnchorClockTime = innerSynchroizer_->GetLastReportedClockTime();
927 int64_t nowCt = syncCenter->GetClockTimeNow();
928 bool needUpdate = forceUpdateTimeAnchorNextTime_ ||
929 (lastAnchorClockTime == HST_TIME_NONE) ||
930 (nowCt - lastAnchorClockTime >= ANCHOR_UPDATE_PERIOD_US);
931 FALSE_RETURN_V_MSG_D(needUpdate, false, "No need to update time anchor this time.");
932 UpdateRenderInfo();
933 int64_t latency = innerSynchroizer_->CalculateAudioLatency();
934 MEDIA_LOG_DD("Calculate latency = " PUBLIC_LOG_U64, latency);
935 int64_t lastBufferPTS = innerSynchroizer_->GetLastBufferPTS();
936 int64_t lastBufferDuration = innerSynchroizer_->GetBufferDuration();
937 Pipeline::IMediaSyncCenter::IMediaTime iMediaTime = {lastBufferPTS - firstPts_, lastBufferPTS,
938 lastBufferDuration};
939 syncCenter->UpdateTimeAnchor(nowCt, latency + fixDelay_, iMediaTime, this);
940 MEDIA_LOG_I("AudioSink fixDelay_: " PUBLIC_LOG_D64
941 " us, latency: " PUBLIC_LOG_D64
942 " us, pts-f: " PUBLIC_LOG_D64
943 " us, nowCt: " PUBLIC_LOG_D64 " us",
944 fixDelay_, latency, lastBufferPTS - firstPts_, nowCt);
945 forceUpdateTimeAnchorNextTime_ = false;
946 innerSynchroizer_->UpdateReportTime(nowCt);
947 return true;
948 }
949
SyncWriteByRenderInfo()950 void AudioSink::SyncWriteByRenderInfo()
951 {
952 auto syncCenter = syncCenter_.lock();
953 FALSE_RETURN(syncCenter != nullptr);
954 if (firstPts_ == HST_TIME_NONE) {
955 if (syncCenter && syncCenter->GetMediaStartPts() != HST_TIME_NONE) {
956 firstPts_ = syncCenter->GetMediaStartPts();
957 } else {
958 firstPts_ = innerSynchroizer_->GetLastBufferPTS();
959 }
960 }
961 bool anchorUpdated = IsTimeAnchorNeedUpdate();
962 innerSynchroizer_->UpdateLastBufferPTS(CalculateBufferDuration(currentQueuedBufferOffset_), speed_);
963 latestBufferDuration_ = innerSynchroizer_->GetBufferDuration() / speed_;
964 if (anchorUpdated) {
965 bufferDurationSinceLastAnchor_ = latestBufferDuration_;
966 } else {
967 bufferDurationSinceLastAnchor_ += latestBufferDuration_;
968 }
969 underrunDetector_.SetLastAudioBufferDuration(bufferDurationSinceLastAnchor_);
970 if (syncCenter) {
971 syncCenter->SetLastAudioBufferDuration(bufferDurationSinceLastAnchor_);
972 }
973 }
974
ReleaseCacheBuffer(bool isSwapBuffer)975 void AudioSink::ReleaseCacheBuffer(bool isSwapBuffer)
976 {
977 if (isSwapBuffer) {
978 FALSE_RETURN_MSG(!swapOutputBuffers_.empty(), "swapOutputBuffers_ has no buffer");
979 swapOutputBuffers_.pop();
980 std::unique_lock<std::mutex> formatLock(formatChangeMutex_);
981 FALSE_RETURN_NOLOG(formatChange_.load() && swapOutputBuffers_.empty() && availOutputBuffers_.empty());
982 formatChange_ = false;
983 formatChangeCond_.notify_all();
984 return;
985 }
986 auto buffer = availOutputBuffers_.front();
987 availOutputBuffers_.pop();
988 FALSE_RETURN_MSG(buffer != nullptr, "release buffer, but buffer is null");
989 MEDIA_LOG_DD("the pts is " PUBLIC_LOG_D64 " buffer is release", buffer->pts_);
990 Status ret = inputBufferQueueConsumer_->ReleaseBuffer(buffer);
991 FALSE_RETURN_MSG(ret == Status::OK, "release avbuffer failed");
992 std::unique_lock<std::mutex> formatLock(formatChangeMutex_);
993 FALSE_RETURN_NOLOG(formatChange_.load() && swapOutputBuffers_.empty() && availOutputBuffers_.empty());
994 formatChange_ = false;
995 formatChangeCond_.notify_all();
996 return;
997 }
998
ResetInfo()999 void AudioSink::ResetInfo()
1000 {
1001 if (appUid_ == BOOT_APP_UID) {
1002 std::unique_lock<std::mutex> eosCbLock(eosCbMutex_);
1003 hangeOnEosCb_ = false;
1004 eosCbCond_.notify_all();
1005 }
1006 innerSynchroizer_->Reset();
1007 maxAmplitude_ = 0;
1008 currentMaxAmplitude_ = 0;
1009 currentQueuedBufferOffset_ = 0;
1010 forceUpdateTimeAnchorNextTime_ = true;
1011 isEosBuffer_ = false;
1012 availDataSize_.store(0);
1013 ClearAvailableOutputBuffers();
1014 ClearInputBuffer();
1015 }
1016
ClearAvailableOutputBuffers()1017 void AudioSink::ClearAvailableOutputBuffers()
1018 {
1019 FALSE_RETURN(inputBufferQueueConsumer_ != nullptr);
1020 std::lock_guard<std::mutex> lock(availBufferMutex_);
1021 while (!swapOutputBuffers_.empty()) {
1022 swapOutputBuffers_.pop();
1023 }
1024 while (!availOutputBuffers_.empty()) {
1025 ReleaseCacheBuffer();
1026 }
1027 std::unique_lock<std::mutex> formatLock(formatChangeMutex_);
1028 formatChange_ = false;
1029 formatChangeCond_.notify_all();
1030 }
1031
GetAvailableOutputBuffers()1032 void AudioSink::GetAvailableOutputBuffers()
1033 {
1034 FALSE_RETURN(inputBufferQueueConsumer_ != nullptr);
1035 std::lock_guard<std::mutex> lock(availBufferMutex_);
1036 std::shared_ptr<AVBuffer> filledInputBuffer;
1037 Status ret = Status::OK;
1038 MEDIA_TRACE_DEBUG("AudioSink::GetBufferFromUpstream");
1039 while (ret == Status::OK) {
1040 ret = inputBufferQueueConsumer_->AcquireBuffer(filledInputBuffer);
1041 if (ret != Status::OK || filledInputBuffer == nullptr) {
1042 break;
1043 }
1044 if (filledInputBuffer->memory_ == nullptr || filledInputBuffer->pts_ < 0) {
1045 inputBufferQueueConsumer_->ReleaseBuffer(filledInputBuffer);
1046 continue;
1047 }
1048 if (DropApeBuffer(filledInputBuffer)) {
1049 break;
1050 }
1051 if (IsEosBuffer(filledInputBuffer)) {
1052 MEDIA_LOG_I("AudioSink Recv EOS");
1053 isEosBuffer_ = true;
1054 }
1055 availOutputBuffers_.push(filledInputBuffer);
1056 MEDIA_LOG_DD("the pts is " PUBLIC_LOG_D64 " buffer is push", filledInputBuffer->pts_);
1057 availDataSize_.fetch_add(filledInputBuffer->memory_->GetSize());
1058 }
1059 DriveBufferCircle();
1060 return;
1061 }
1062
SetIsInPrePausing(bool isInPrePausing)1063 void AudioSink::SetIsInPrePausing(bool isInPrePausing)
1064 {
1065 MediaAVCodec::AVCodecTrace trace("AudioSink::SetIsInPrePausing" + std::to_string(isInPrePausing));
1066 isInPrePausing_.store(isInPrePausing, std::memory_order_relaxed);
1067 FALSE_RETURN_NOLOG(isInPrePausing && (isRenderCallbackMode_ && !isAudioDemuxDecodeAsync_));
1068
1069 std::lock_guard<std::mutex> lock(availBufferMutex_);
1070 DriveBufferCircle();
1071 }
1072
DriveBufferCircle()1073 void AudioSink::DriveBufferCircle()
1074 {
1075 FALSE_RETURN_NOLOG(!isEosBuffer_);
1076 FALSE_RETURN_NOLOG(!availOutputBuffers_.empty() && inputBufferQueue_ != nullptr);
1077 FALSE_RETURN_NOLOG(availOutputBuffers_.size() >= inputBufferQueue_->GetQueueSize());
1078 size_t availDataSize = availDataSize_.load();
1079 int32_t availDataSizeInt32 = availDataSize <= static_cast<size_t>(INT32_MAX) ?
1080 static_cast<int32_t>(availDataSize): INT32_MAX;
1081 bool isInPrePausing = isInPrePausing_.load(std::memory_order_relaxed);
1082 FALSE_RETURN_NOLOG(isInPrePausing || availDataSizeInt32 < maxCbDataSize_);
1083 std::shared_ptr<AVBuffer> oldestBuffer = availOutputBuffers_.front();
1084 FALSE_RETURN_MSG(oldestBuffer != nullptr && oldestBuffer->memory_->GetSize() > 0, "buffer or memory is nullptr");
1085 std::shared_ptr<AVBuffer> swapBuffer = CopyBuffer(oldestBuffer);
1086 FALSE_RETURN_MSG(swapBuffer != nullptr, "CopyBuffer failed, swapBuffer is nullptr");
1087 availOutputBuffers_.pop();
1088 swapOutputBuffers_.push(swapBuffer);
1089 if (isInPrePausing) {
1090 MEDIA_LOG_I("DriveBufferCircle availOutputBuffers_ size:%{public}d, swapOutputBuffers_ size:%{public}d",
1091 static_cast<int>(availOutputBuffers_.size()), static_cast<int>(swapOutputBuffers_.size()));
1092 }
1093 FALSE_RETURN_MSG(inputBufferQueueConsumer_ != nullptr, "bufferQueue consumer is nullptr");
1094 inputBufferQueueConsumer_->ReleaseBuffer(oldestBuffer);
1095 }
1096
CopyBuffer(const std::shared_ptr<AVBuffer> buffer)1097 std::shared_ptr<AVBuffer> AudioSink::CopyBuffer(const std::shared_ptr<AVBuffer> buffer)
1098 {
1099 FALSE_RETURN_V_MSG_E(buffer != nullptr && buffer->memory_->GetSize() > 0, nullptr, "buffer or memory is nullptr");
1100 std::shared_ptr<Meta> meta = buffer->meta_;
1101 std::vector<uint8_t> metaData;
1102 FALSE_RETURN_V_MSG_W(meta == nullptr || !meta->GetData(Tag::OH_MD_KEY_AUDIO_VIVID_METADATA, metaData), nullptr,
1103 "copy buffer not support for audiovivid");
1104 AVBufferConfig avBufferConfig;
1105 avBufferConfig.capacity = static_cast<int32_t>(buffer->memory_->GetSize());
1106 avBufferConfig.memoryType = bufferMemoryType_;
1107 std::shared_ptr<AVBuffer> swapBuffer = AVBuffer::CreateAVBuffer(avBufferConfig);
1108 FALSE_RETURN_V_MSG_E(swapBuffer != nullptr && swapBuffer->memory_ != nullptr, nullptr, "create swapBuffer failed");
1109 swapBuffer->pts_ = buffer->pts_;
1110 swapBuffer->dts_ = buffer->dts_;
1111 swapBuffer->duration_ = buffer->duration_;
1112 swapBuffer->flag_ = buffer->flag_;
1113 FALSE_RETURN_V_MSG_E(swapBuffer->memory_->GetCapacity() >= buffer->memory_->GetSize(), nullptr, "no enough memory");
1114 errno_t res = memcpy_s(swapBuffer->memory_->GetAddr(),
1115 swapBuffer->memory_->GetCapacity(),
1116 buffer->memory_->GetAddr(),
1117 buffer->memory_->GetSize());
1118 FALSE_RETURN_V_MSG_E(res == EOK, nullptr, "copy data failed");
1119 swapBuffer->memory_->SetSize(buffer->memory_->GetSize());
1120 return swapBuffer;
1121 }
1122
WriteDataToRender(std::shared_ptr<AVBuffer> & filledOutputBuffer)1123 void AudioSink::WriteDataToRender(std::shared_ptr<AVBuffer> &filledOutputBuffer)
1124 {
1125 FALSE_RETURN(DropApeBuffer(filledOutputBuffer) == false);
1126 FALSE_RETURN(filledOutputBuffer != nullptr);
1127 if (IsEosBuffer(filledOutputBuffer)) {
1128 HandleEosBuffer(filledOutputBuffer);
1129 return;
1130 }
1131 FALSE_RETURN(plugin_ != nullptr);
1132 UpdateAudioWriteTimeMayWait();
1133 DoSyncWrite(filledOutputBuffer);
1134 if (calMaxAmplitudeCbStatus_) {
1135 CalcMaxAmplitude(filledOutputBuffer);
1136 UpdateAmplitude();
1137 } else {
1138 maxAmplitude_ = 0.0f;
1139 }
1140 lastBufferWriteSuccess_ = (plugin_->Write(filledOutputBuffer) == Status::OK);
1141 int64_t nowClockTime = 0;
1142 GetSyncCenterClockTime(nowClockTime);
1143 auto audioWriteMs = plugin_->GetWriteDurationMs();
1144 lagDetector_.UpdateDrainTimeGroup(
1145 { lastAnchorClockTime_, bufferDurationSinceLastAnchor_, audioWriteMs, nowClockTime });
1146 PerfRecord(audioWriteMs);
1147 lagDetector_.CalcLag(filledOutputBuffer);
1148 MEDIA_LOG_DD("audio DrainOutputBuffer pts = " PUBLIC_LOG_D64, filledOutputBuffer->pts_);
1149 numFramesWritten_++;
1150 FALSE_RETURN(inputBufferQueueConsumer_ != nullptr);
1151 inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer);
1152 }
1153
IsEosBuffer(std::shared_ptr<AVBuffer> & filledOutputBuffer)1154 bool AudioSink::IsEosBuffer(std::shared_ptr<AVBuffer> &filledOutputBuffer)
1155 {
1156 FALSE_RETURN_V(filledOutputBuffer != nullptr, false);
1157 return (filledOutputBuffer->flag_ & BUFFER_FLAG_EOS) ||
1158 ((playRangeEndTime_ != DEFAULT_PLAY_RANGE_VALUE) &&
1159 (filledOutputBuffer->pts_ > playRangeEndTime_ * MICROSECONDS_CONVERT_UNITS));
1160 }
1161
HandleEosBuffer(std::shared_ptr<AVBuffer> & filledOutputBuffer)1162 void AudioSink::HandleEosBuffer(std::shared_ptr<AVBuffer> &filledOutputBuffer)
1163 {
1164 FALSE_RETURN(inputBufferQueueConsumer_ != nullptr);
1165 inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer);
1166 AutoLock eosLock(eosMutex_);
1167 // avoid submit handle eos task multiple times
1168 FALSE_RETURN(!eosDraining_);
1169 eosInterruptType_ = EosInterruptState::INITIAL;
1170 if (eosTask_ == nullptr) {
1171 DrainAndReportEosEvent();
1172 return;
1173 }
1174 MEDIA_LOG_I("DrainOutputBuffer Recv EOS PTS");
1175 eosTask_->SubmitJobOnce([this] {
1176 HandleEosInner(false);
1177 });
1178 return;
1179 }
1180
DrainOutputBuffer(bool flushed)1181 void AudioSink::DrainOutputBuffer(bool flushed)
1182 {
1183 std::lock_guard<std::mutex> lock(pluginMutex_);
1184 FALSE_RETURN(plugin_ != nullptr && inputBufferQueueConsumer_ != nullptr);
1185
1186 if (isRenderCallbackMode_) {
1187 if (state_ != Pipeline::FilterState::RUNNING) {
1188 MEDIA_LOG_W("DrainOutputBuffer ignore temporarily for not RUNNINT state: " PUBLIC_LOG_D32
1189 ", isProcessInputMerged: " PUBLIC_LOG_D32, state_.load(), isProcessInputMerged_);
1190
1191 /*
1192 * As the AudioRender START and PAUSE procedure may consume a long time about 200 ms,
1193 * if the consumption of inputbuffer queue is not excuted in AudioSinkFilter's task working thread,
1194 * this RETURN ACTION will cause the filled buffers not been consumed,
1195 * which cause the upstream filter RequesetBuffer failed and audio track playback stuck.
1196 */
1197 if (!isProcessInputMerged_) {
1198 return;
1199 }
1200 }
1201 GetAvailableOutputBuffers();
1202 } else {
1203 std::shared_ptr<AVBuffer> filledOutputBuffer = nullptr;
1204 Status ret = inputBufferQueueConsumer_->AcquireBuffer(filledOutputBuffer);
1205 FALSE_RETURN(ret == Status::OK && filledOutputBuffer != nullptr);
1206 if (state_ != Pipeline::FilterState::RUNNING || flushed) {
1207 MEDIA_LOG_W("DrainOutputBuffer, drop audio buffer pts = " PUBLIC_LOG_D64 ", state: " PUBLIC_LOG_D32
1208 ", flushed: " PUBLIC_LOG_D32, filledOutputBuffer->pts_, state_.load(), flushed);
1209
1210 /*
1211 * As START and PAUSE procedure of AudioDecoderFilter and AudioSinkFilter run concurrently
1212 * in different working thread, AudioSinkFilter may change to RUNNING state after AudioDecoderFilter
1213 * or AudioSinkFilter may change to PAUSED state before AudioDecoderFilter.
1214 * This ReleaseBuffer ACTION will cause audio buffer droped, which may cause audio discontinuity.
1215 *
1216 * So, How to deal with this case?
1217 * 1. Try to cache buffers in non RUNNING state, prioritize consuming the cached buffers in RUNNING state?
1218 * 2. There may be some buffers that cannot be processed, i.e. do Pause/Resume repeatedly around the ending
1219 * and cause the EOS buffer being cached which may cause that the Audio Track can't end.
1220 */
1221 inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer);
1222 return;
1223 }
1224 WriteDataToRender(filledOutputBuffer);
1225 }
1226 }
1227
SetPerfRecEnabled(bool isPerfRecEnabled)1228 Status AudioSink::SetPerfRecEnabled(bool isPerfRecEnabled)
1229 {
1230 isPerfRecEnabled_ = isPerfRecEnabled;
1231 return Status::OK;
1232 }
1233
PerfRecord(int64_t audioWriteMs)1234 void AudioSink::PerfRecord(int64_t audioWriteMs)
1235 {
1236 FALSE_RETURN_NOLOG(isPerfRecEnabled_);
1237 FALSE_RETURN_MSG(playerEventReceiver_ != nullptr, "Report perf failed, event receiver is nullptr");
1238 FALSE_RETURN_NOLOG(perfRecorder_.Record(audioWriteMs) == PerfRecorder::FULL);
1239 playerEventReceiver_->OnDfxEvent(
1240 { "ASINK", DfxEventType::DFX_INFO_PERF_REPORT, perfRecorder_.GetMainPerfData() });
1241 perfRecorder_.Reset();
1242 }
1243
ResetSyncInfo()1244 void AudioSink::ResetSyncInfo()
1245 {
1246 lastAnchorClockTime_ = HST_TIME_NONE;
1247 forceUpdateTimeAnchorNextTime_ = true;
1248 }
1249
Reset()1250 void AudioSink::UnderrunDetector::Reset()
1251 {
1252 AutoLock lock(mutex_);
1253 lastClkTime_ = HST_TIME_NONE;
1254 lastLatency_ = HST_TIME_NONE;
1255 lastBufferDuration_ = HST_TIME_NONE;
1256 }
1257
SetEventReceiver(std::weak_ptr<Pipeline::EventReceiver> eventReceiver)1258 void AudioSink::UnderrunDetector::SetEventReceiver(std::weak_ptr<Pipeline::EventReceiver> eventReceiver)
1259 {
1260 eventReceiver_ = eventReceiver;
1261 }
1262
UpdateBufferTimeNoLock(int64_t clkTime,int64_t latency)1263 void AudioSink::UnderrunDetector::UpdateBufferTimeNoLock(int64_t clkTime, int64_t latency)
1264 {
1265 lastClkTime_ = clkTime;
1266 lastLatency_ = latency;
1267 }
1268
SetLastAudioBufferDuration(int64_t durationUs)1269 void AudioSink::UnderrunDetector::SetLastAudioBufferDuration(int64_t durationUs)
1270 {
1271 AutoLock lock(mutex_);
1272 lastBufferDuration_ = durationUs;
1273 }
1274
DetectAudioUnderrun(int64_t clkTime,int64_t latency)1275 void AudioSink::UnderrunDetector::DetectAudioUnderrun(int64_t clkTime, int64_t latency)
1276 {
1277 if (lastClkTime_ == HST_TIME_NONE) {
1278 AutoLock lock(mutex_);
1279 UpdateBufferTimeNoLock(clkTime, latency);
1280 return;
1281 }
1282 int64_t underrunTimeUs = 0;
1283 {
1284 AutoLock lock(mutex_);
1285 int64_t elapsedClk = clkTime - lastClkTime_;
1286 underrunTimeUs = elapsedClk - (lastLatency_ + lastBufferDuration_);
1287 UpdateBufferTimeNoLock(clkTime, latency);
1288 }
1289 if (underrunTimeUs > 0) {
1290 MEDIA_LOG_D("AudioSink maybe underrun, underrunTimeUs=" PUBLIC_LOG_D64, underrunTimeUs);
1291 auto eventReceiver = eventReceiver_.lock();
1292 FALSE_RETURN(eventReceiver != nullptr);
1293 eventReceiver->OnDfxEvent({"AudioSink", DfxEventType::DFX_INFO_PLAYER_AUDIO_LAG, underrunTimeUs / US_TO_MS});
1294 }
1295 }
1296
CalcLag(std::shared_ptr<AVBuffer> buffer)1297 bool AudioSink::AudioLagDetector::CalcLag(std::shared_ptr<AVBuffer> buffer)
1298 {
1299 (void)buffer;
1300 int64_t maxMediaTime = lastDrainTimeGroup_.anchorDuration + lastDrainTimeGroup_.lastAnchorPts + latency_;
1301 auto currentMediaTime = lastDrainTimeGroup_.nowClockTime;
1302 auto writeTimeMs = lastDrainTimeGroup_.writeDuration;
1303
1304 MEDIA_LOG_DD("maxMediaTime " PUBLIC_LOG_D64 " currentMediaTime " PUBLIC_LOG_D64 " latency_ " PUBLIC_LOG_D64,
1305 maxMediaTime, currentMediaTime, latency_);
1306
1307 if (maxMediaTime < currentMediaTime) {
1308 MEDIA_LOG_W("renderer write cost " PUBLIC_LOG_D64, writeTimeMs);
1309 }
1310
1311 // Calc time delays except plugin write
1312 auto currentTimeMs = Plugins::GetCurrentMillisecond();
1313 auto totalTimeDiff = lastDrainTimeMs_ == 0 ? 0 : currentTimeMs - lastDrainTimeMs_;
1314 lastDrainTimeMs_ = currentTimeMs;
1315 auto drainTimeDiff = totalTimeDiff - writeTimeMs;
1316 if (drainTimeDiff > DRAIN_TIME_DIFF_WARN_MS) {
1317 MEDIA_LOG_W("Audio drain cost " PUBLIC_LOG_D64, drainTimeDiff);
1318 } else if (drainTimeDiff > DRAIN_TIME_DIFF_INFO_MS) {
1319 MEDIA_LOG_I("Audio drain cost " PUBLIC_LOG_D64, drainTimeDiff);
1320 } else {
1321 MEDIA_LOG_DD("Audio drain cost " PUBLIC_LOG_D64, drainTimeDiff);
1322 }
1323
1324 return maxMediaTime < currentMediaTime;
1325 }
1326
Reset()1327 void AudioSink::AudioLagDetector::Reset()
1328 {
1329 latency_ = 0;
1330 lastDrainTimeMs_ = 0;
1331 lastDrainTimeGroup_.lastAnchorPts = 0;
1332 lastDrainTimeGroup_.anchorDuration = 0;
1333 lastDrainTimeGroup_.writeDuration = 0;
1334 lastDrainTimeGroup_.nowClockTime = 0;
1335 }
1336
GetSyncCenterClockTime(int64_t & clockTime)1337 bool AudioSink::GetSyncCenterClockTime(int64_t &clockTime)
1338 {
1339 auto syncCenter = syncCenter_.lock();
1340 FALSE_RETURN_V(syncCenter != nullptr, false);
1341 clockTime = syncCenter->GetClockTimeNow();
1342 return true;
1343 }
1344
UpdateDrainTimeGroup(AudioDrainTimeGroup group)1345 void AudioSink::AudioLagDetector::UpdateDrainTimeGroup(AudioDrainTimeGroup group)
1346 {
1347 lastDrainTimeGroup_.lastAnchorPts = group.lastAnchorPts;
1348 lastDrainTimeGroup_.anchorDuration = group.anchorDuration;
1349 lastDrainTimeGroup_.writeDuration = group.writeDuration;
1350 lastDrainTimeGroup_.nowClockTime = group.nowClockTime;
1351 }
1352
UpdateTimeAnchorIfNeeded(const std::shared_ptr<OHOS::Media::AVBuffer> & buffer)1353 bool AudioSink::UpdateTimeAnchorIfNeeded(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer)
1354 {
1355 auto syncCenter = syncCenter_.lock();
1356 FALSE_RETURN_V(syncCenter != nullptr, false);
1357 int64_t nowCt = syncCenter->GetClockTimeNow();
1358 bool needUpdate = forceUpdateTimeAnchorNextTime_ ||
1359 (lastAnchorClockTime_ == HST_TIME_NONE) ||
1360 (nowCt - lastAnchorClockTime_ >= ANCHOR_UPDATE_PERIOD_US);
1361 if (!needUpdate) {
1362 MEDIA_LOG_DD("No need to update time anchor this time.");
1363 return false;
1364 }
1365 uint64_t latency = 0;
1366 FALSE_LOG_MSG(plugin_->GetLatency(latency) == Status::OK, "failed to get latency");
1367 Pipeline::IMediaSyncCenter::IMediaTime iMediaTime = {buffer->pts_ - firstPts_, buffer->pts_, buffer->duration_};
1368 syncCenter->UpdateTimeAnchor(nowCt, latency + fixDelay_, iMediaTime, this);
1369 lagDetector_.SetLatency(latency + fixDelay_);
1370 MEDIA_LOG_I("AudioSink fixDelay_: " PUBLIC_LOG_D64
1371 " us, latency: " PUBLIC_LOG_D64
1372 " us, pts-f: " PUBLIC_LOG_D64
1373 " us, pts: " PUBLIC_LOG_D64
1374 " us, nowCt: " PUBLIC_LOG_D64 " us",
1375 fixDelay_, latency, buffer->pts_ - firstPts_, buffer->pts_, nowCt);
1376 forceUpdateTimeAnchorNextTime_ = false;
1377 lastAnchorClockTime_ = nowCt;
1378 return true;
1379 }
1380
DoSyncWrite(const std::shared_ptr<OHOS::Media::AVBuffer> & buffer)1381 int64_t AudioSink::DoSyncWrite(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer)
1382 {
1383 bool render = true; // audio sink always report time anchor and do not drop
1384 auto syncCenter = syncCenter_.lock();
1385 FALSE_RETURN_V(syncCenter != nullptr, 0);
1386 if (firstPts_ == HST_TIME_NONE) {
1387 if (syncCenter && syncCenter->GetMediaStartPts() != HST_TIME_NONE) {
1388 firstPts_ = syncCenter->GetMediaStartPts();
1389 } else {
1390 firstPts_ = buffer->pts_;
1391 }
1392 MEDIA_LOG_I("audio DoSyncWrite set firstPts = " PUBLIC_LOG_D64, firstPts_);
1393 }
1394 bool anchorUpdated = UpdateTimeAnchorIfNeeded(buffer);
1395 latestBufferDuration_ = CalcBufferDuration(buffer) / speed_;
1396 if (anchorUpdated) {
1397 bufferDurationSinceLastAnchor_ = latestBufferDuration_;
1398 } else {
1399 bufferDurationSinceLastAnchor_ += latestBufferDuration_;
1400 }
1401 underrunDetector_.SetLastAudioBufferDuration(bufferDurationSinceLastAnchor_);
1402 if (syncCenter) {
1403 syncCenter->SetLastAudioBufferDuration(bufferDurationSinceLastAnchor_);
1404 }
1405 return render ? 0 : -1;
1406 }
1407
CalcBufferDuration(const std::shared_ptr<OHOS::Media::AVBuffer> & buffer)1408 int64_t AudioSink::CalcBufferDuration(const std::shared_ptr<OHOS::Media::AVBuffer>& buffer)
1409 {
1410 FALSE_RETURN_V(buffer != nullptr && buffer->memory_ != nullptr && sampleRate_ != 0 && audioChannelCount_ != 0, 0);
1411 int64_t size = static_cast<int64_t>(buffer->memory_->GetSize());
1412 int32_t format = GetSampleFormatBytes();
1413 FALSE_RETURN_V(format > 0 && audioChannelCount_ > 0, 0);
1414 return SEC_TO_US * size / format / sampleRate_ / audioChannelCount_;
1415 }
1416
SetSpeed(float speed)1417 Status AudioSink::SetSpeed(float speed)
1418 {
1419 if (plugin_ == nullptr) {
1420 return Status::ERROR_NULL_POINTER;
1421 }
1422 if (speed <= 0) {
1423 return Status::ERROR_INVALID_PARAMETER;
1424 }
1425 auto ret = plugin_->SetSpeed(speed);
1426 if (ret == Status::OK) {
1427 speed_ = speed;
1428 }
1429 forceUpdateTimeAnchorNextTime_ = true;
1430 return ret;
1431 }
1432
SetAudioEffectMode(int32_t effectMode)1433 Status AudioSink::SetAudioEffectMode(int32_t effectMode)
1434 {
1435 MEDIA_LOG_I("SetAudioEffectMode");
1436 if (plugin_ == nullptr) {
1437 return Status::ERROR_NULL_POINTER;
1438 }
1439 effectMode_ = effectMode;
1440 return plugin_->SetAudioEffectMode(effectMode);
1441 }
1442
GetAudioEffectMode(int32_t & effectMode)1443 Status AudioSink::GetAudioEffectMode(int32_t &effectMode)
1444 {
1445 MEDIA_LOG_I("GetAudioEffectMode");
1446 if (plugin_ == nullptr) {
1447 return Status::ERROR_NULL_POINTER;
1448 }
1449 return plugin_->GetAudioEffectMode(effectMode);
1450 }
1451
OnNewAudioMediaTime(int64_t mediaTimeUs)1452 bool AudioSink::OnNewAudioMediaTime(int64_t mediaTimeUs)
1453 {
1454 bool render = true;
1455 if (firstAudioAnchorTimeMediaUs_ == Plugins::HST_TIME_NONE) {
1456 firstAudioAnchorTimeMediaUs_ = mediaTimeUs;
1457 }
1458 int64_t nowUs = 0;
1459 auto syncCenter = syncCenter_.lock();
1460 if (syncCenter) {
1461 nowUs = syncCenter->GetClockTimeNow();
1462 }
1463 int64_t pendingTimeUs = getPendingAudioPlayoutDurationUs(nowUs);
1464 render = syncCenter->UpdateTimeAnchor(nowUs, pendingTimeUs, {mediaTimeUs, mediaTimeUs, mediaTimeUs}, this);
1465 return render;
1466 }
1467
getPendingAudioPlayoutDurationUs(int64_t nowUs)1468 int64_t AudioSink::getPendingAudioPlayoutDurationUs(int64_t nowUs)
1469 {
1470 int64_t writtenSamples = numFramesWritten_ * samplePerFrame_;
1471 const int64_t numFramesPlayed = plugin_->GetPlayedOutDurationUs(nowUs);
1472 FALSE_RETURN_V(sampleRate_ > 0, 0);
1473 int64_t pendingUs = (writtenSamples - numFramesPlayed) * HST_MSECOND / sampleRate_;
1474 MEDIA_LOG_DD("pendingUs: " PUBLIC_LOG_D64, pendingUs);
1475 if (pendingUs < 0) {
1476 pendingUs = 0;
1477 }
1478 return pendingUs;
1479 }
1480
getDurationUsPlayedAtSampleRate(uint32_t numFrames)1481 int64_t AudioSink::getDurationUsPlayedAtSampleRate(uint32_t numFrames)
1482 {
1483 std::shared_ptr<Meta> parameter;
1484 plugin_->GetParameter(parameter);
1485 int32_t sampleRate = 0;
1486 if (parameter) {
1487 parameter->GetData(Tag::AUDIO_SAMPLE_RATE, sampleRate);
1488 }
1489 if (sampleRate == 0) {
1490 MEDIA_LOG_W("cannot get sampleRate");
1491 return 0;
1492 }
1493 return (int64_t)(static_cast<int32_t>(numFrames) * HST_MSECOND / sampleRate);
1494 }
1495
SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> & receiver)1496 void AudioSink::SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver>& receiver)
1497 {
1498 FALSE_RETURN(receiver != nullptr);
1499 playerEventReceiver_ = receiver;
1500 FALSE_RETURN(plugin_ != nullptr);
1501 plugin_->SetEventReceiver(receiver);
1502 }
1503
SetSyncCenter(std::shared_ptr<Pipeline::MediaSyncManager> syncCenter)1504 void AudioSink::SetSyncCenter(std::shared_ptr<Pipeline::MediaSyncManager> syncCenter)
1505 {
1506 syncCenter_ = syncCenter;
1507 MediaSynchronousSink::Init();
1508 }
1509
ChangeTrackForFormatChange()1510 Status AudioSink::ChangeTrackForFormatChange()
1511 {
1512 MEDIA_LOG_I("ChangeTrackForFormatChange.");
1513 std::lock_guard<std::mutex> lock(pluginMutex_);
1514 std::shared_ptr<Plugins::AudioSinkPlugin> plugin = std::move(plugin_);
1515 FALSE_RETURN_V(plugin != nullptr && changeTrackTask_ != nullptr, Status::ERROR_NULL_POINTER);
1516 changeTrackTask_->SubmitJobOnce([plugin] {
1517 plugin->Stop();
1518 plugin->Deinit();
1519 });
1520
1521 std::unique_lock<std::mutex> preCreateLock(preCreatePluginMutex_);
1522 preCreatePluginCond_.wait(preCreateLock,
1523 [this]() { return isInterruptNeeded_ || hasPluginCreateTaskFinished_.load(); });
1524 hasPluginCreateTaskFinished_ = false;
1525 FALSE_RETURN_V(newPlugin_ != nullptr, Status::ERROR_NULL_POINTER);
1526 plugin_ = std::move(newPlugin_);
1527
1528 forceUpdateTimeAnchorNextTime_ = true;
1529
1530 return Status::OK;
1531 }
1532
ChangeTrack(std::shared_ptr<Meta> & meta,const std::shared_ptr<Pipeline::EventReceiver> & receiver)1533 Status AudioSink::ChangeTrack(std::shared_ptr<Meta>& meta, const std::shared_ptr<Pipeline::EventReceiver>& receiver)
1534 {
1535 MEDIA_LOG_I("GetAudioEffectMode ChangeTrack. ");
1536 std::lock_guard<std::mutex> lock(pluginMutex_);
1537 if (plugin_) {
1538 plugin_->Stop();
1539 plugin_->Deinit();
1540 plugin_ = nullptr;
1541 }
1542 plugin_ = CreatePlugin();
1543 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
1544 Status ret = Status::OK;
1545 ret = InitAudioSinkPlugin(meta, receiver, plugin_);
1546 FALSE_RETURN_V(ret == Status::OK, ret);
1547
1548 ret = InitAudioSinkInfo(meta);
1549 FALSE_RETURN_V(ret == Status::OK, ret);
1550
1551 ret = SetAudioSinkPluginParameters(plugin_);
1552 FALSE_RETURN_V(ret == Status::OK, ret);
1553
1554 forceUpdateTimeAnchorNextTime_ = true;
1555
1556 return Status::OK;
1557 }
1558
HandleFormatChange(std::shared_ptr<Meta> & meta,const std::shared_ptr<Pipeline::EventReceiver> & receiver)1559 Status AudioSink::HandleFormatChange(std::shared_ptr<Meta>& meta,
1560 const std::shared_ptr<Pipeline::EventReceiver>& receiver)
1561 {
1562 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_INVALID_STATE);
1563 ScopedTimer timer("HandleFormatChange", FORMAT_CHANGE_MS);
1564 FALSE_RETURN_V_NOLOG(meta && plugin_ && plugin_->IsFormatSupported(meta), Status::ERROR_INVALID_PARAMETER);
1565 WaitForAllBufferConsumed();
1566 FALSE_RETURN_V_MSG(!isInterruptNeeded_, Status::OK, "Abandon format change operation becaused of interruption");
1567 FALSE_RETURN_V(changeTrackTask_ != nullptr, Status::ERROR_NULL_POINTER);
1568 std::weak_ptr<AudioSink> weakPtr(shared_from_this());
1569 changeTrackTask_->SubmitJobOnce([weakPtr, meta, receiver] {
1570 auto strongPtr = weakPtr.lock();
1571 FALSE_RETURN(strongPtr != nullptr);
1572 std::unique_lock<std::mutex> preCreateLock(strongPtr->preCreatePluginMutex_);
1573 strongPtr->newPlugin_ = strongPtr->PreCreateAndStartNewPlugin(meta, receiver);
1574 strongPtr->hasPluginCreateTaskFinished_ = true;
1575 strongPtr->preCreatePluginCond_.notify_one();
1576 });
1577 plugin_->Drain();
1578 FlushForChangeTrack();
1579 return ChangeTrackForFormatChange();
1580 }
1581
WaitForAllBufferConsumed()1582 void AudioSink::WaitForAllBufferConsumed()
1583 {
1584 ScopedTimer timer("WaitForAllBufferConsumed", BUFFER_CONSUME_MS);
1585 MediaAVCodec::AVCodecTrace trace("AudioSink::WaitForAllBufferConsumed");
1586 std::unique_lock<std::mutex> formatLock(formatChangeMutex_);
1587 formatChange_ = true;
1588 formatChangeCond_.wait(formatLock,
1589 [this]() { return isInterruptNeeded_ || (swapOutputBuffers_.empty() && availOutputBuffers_.empty()); });
1590 }
1591
SetAudioSinkPluginParameters(const std::shared_ptr<Plugins::AudioSinkPlugin> & plugin)1592 Status AudioSink::SetAudioSinkPluginParameters(const std::shared_ptr<Plugins::AudioSinkPlugin>& plugin)
1593 {
1594 FALSE_RETURN_V(plugin != nullptr, Status::ERROR_NULL_POINTER);
1595 Status ret = Status::OK;
1596 if (volume_ >= 0) {
1597 plugin->SetVolume(volume_);
1598 }
1599 if (speed_ >= 0) {
1600 plugin->SetSpeed(speed_);
1601 }
1602 if (effectMode_ >= 0) {
1603 plugin->SetAudioEffectMode(effectMode_);
1604 }
1605 if (state_ == Pipeline::FilterState::RUNNING) {
1606 ret = plugin->Start();
1607 }
1608 return ret;
1609 }
1610
SetMuted(bool isMuted)1611 Status AudioSink::SetMuted(bool isMuted)
1612 {
1613 isMuted_ = isMuted;
1614 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
1615 return plugin_->SetMuted(isMuted);
1616 }
1617
SetMaxAmplitudeCbStatus(bool status)1618 int32_t AudioSink::SetMaxAmplitudeCbStatus(bool status)
1619 {
1620 calMaxAmplitudeCbStatus_ = status;
1621 MEDIA_LOG_I("audio SetMaxAmplitudeCbStatus = " PUBLIC_LOG_D32, calMaxAmplitudeCbStatus_);
1622 return 0;
1623 }
1624
SetSeekTime(int64_t seekTime)1625 Status AudioSink::SetSeekTime(int64_t seekTime)
1626 {
1627 MEDIA_LOG_I("AudioSink SetSeekTime pts = " PUBLIC_LOG_D64, seekTime);
1628 seekTimeUs_ = seekTime;
1629 return Status::OK;
1630 }
1631
OnInterrupted(bool isInterruptNeeded)1632 void AudioSink::OnInterrupted(bool isInterruptNeeded)
1633 {
1634 MEDIA_LOG_D("OnInterrupted %{public}d", isInterruptNeeded);
1635 {
1636 std::unique_lock<std::mutex> lock(formatChangeMutex_);
1637 isInterruptNeeded_ = isInterruptNeeded;
1638 formatChangeCond_.notify_all();
1639 }
1640 if (plugin_ != nullptr) {
1641 plugin_->SetInterruptState(isInterruptNeeded);
1642 }
1643 }
1644
SetIsCalledBySystemApp(bool isCalledBySystemApp)1645 Status AudioSink::SetIsCalledBySystemApp(bool isCalledBySystemApp)
1646 {
1647 MEDIA_LOG_I("AudioSink isCalledBySystemApp = " PUBLIC_LOG_D32, isCalledBySystemApp);
1648 isCalledBySystemApp_ = isCalledBySystemApp;
1649 return Status::OK;
1650 }
1651
SetLooping(bool loop)1652 Status AudioSink::SetLooping(bool loop)
1653 {
1654 isLoop_ = loop;
1655 MEDIA_LOG_I("AudioSink SetLooping isLoop_ = " PUBLIC_LOG_D32, isLoop_);
1656 return Status::OK;
1657 }
1658
SetAudioHapticsSyncId(int32_t syncId)1659 Status AudioSink::SetAudioHapticsSyncId(int32_t syncId)
1660 {
1661 FALSE_RETURN_V(plugin_ != nullptr, Status::ERROR_NULL_POINTER);
1662 return plugin_->SetAudioHapticsSyncId(syncId);
1663 }
1664
PreCreateAndStartNewPlugin(const std::shared_ptr<Meta> & meta,const std::shared_ptr<Pipeline::EventReceiver> & receiver)1665 std::shared_ptr<Plugins::AudioSinkPlugin> AudioSink::PreCreateAndStartNewPlugin(const std::shared_ptr<Meta>& meta,
1666 const std::shared_ptr<Pipeline::EventReceiver>& receiver)
1667 {
1668 MEDIA_LOG_I("AudioSink PreCreateAndStartNewPlugin.");
1669 auto plugin = CreatePlugin();
1670 FALSE_RETURN_V(plugin != nullptr, nullptr);
1671 Status ret = Status::OK;
1672 ret = InitAudioSinkPlugin(meta, receiver, plugin);
1673 FALSE_RETURN_V(ret == Status::OK, nullptr);
1674
1675 ret = SetAudioSinkPluginParameters(plugin);
1676 FALSE_RETURN_V(ret == Status::OK, nullptr);
1677 return plugin;
1678 }
1679 } // namespace MEDIA
1680 } // namespace OHOS
1681