1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "audio_capture_as_impl.h"
17 #include <vector>
18 #include <cmath>
19 #include "media_log.h"
20 #include "audio_errors.h"
21 #include "media_errors.h"
22
23 namespace {
24 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AudioCaptureAsImpl"};
25 constexpr size_t MAXIMUM_BUFFER_SIZE = 100000;
26 constexpr uint64_t SEC_TO_NANOSECOND = 1000000000;
27 }
28
29 namespace OHOS {
30 namespace Media {
AudioCaptureAsImpl()31 AudioCaptureAsImpl::AudioCaptureAsImpl()
32 {
33 }
34
~AudioCaptureAsImpl()35 AudioCaptureAsImpl::~AudioCaptureAsImpl()
36 {
37 if (captureLoop_ != nullptr && audioCacheCtrl_ != nullptr && captureLoop_->joinable()) {
38 MEDIA_LOGE("Clear thread resources");
39 curState_.store(RECORDER_INITIALIZED);
40 audioCacheCtrl_->pauseCond_.notify_all();
41 captureLoop_->join();
42 captureLoop_.reset();
43 captureLoop_ = nullptr;
44 }
45
46 if (audioCapturer_ != nullptr) {
47 (void)audioCapturer_->Release();
48 audioCapturer_ = nullptr;
49 }
50 }
51
CheckAndGetCaptureParameter(uint32_t bitrate,uint32_t channels,uint32_t sampleRate,AudioStandard::AudioCapturerParams & params)52 bool AudioCaptureAsImpl::CheckAndGetCaptureParameter(uint32_t bitrate, uint32_t channels, uint32_t sampleRate,
53 AudioStandard::AudioCapturerParams ¶ms)
54 {
55 (void)bitrate;
56 auto supportedSampleList = AudioStandard::AudioCapturer::GetSupportedSamplingRates();
57 CHECK_AND_RETURN_RET(supportedSampleList.size() > 0, false);
58 bool isValidSampleRate = false;
59 for (auto iter = supportedSampleList.cbegin(); iter != supportedSampleList.cend(); ++iter) {
60 CHECK_AND_RETURN_RET(static_cast<int32_t>(*iter) > 0, false);
61 uint32_t supportedSampleRate = static_cast<uint32_t>(*iter);
62 if (sampleRate <= supportedSampleRate) {
63 params.samplingRate = *iter;
64 isValidSampleRate = true;
65 break;
66 }
67 }
68 CHECK_AND_RETURN_RET(isValidSampleRate, false);
69
70 auto supportedChannelsList = AudioStandard::AudioCapturer::GetSupportedChannels();
71 CHECK_AND_RETURN_RET(supportedChannelsList.size() > 0, false);
72 bool isValidChannels = false;
73 for (auto iter = supportedChannelsList.cbegin(); iter != supportedChannelsList.cend(); ++iter) {
74 CHECK_AND_RETURN_RET(static_cast<int32_t>(*iter) > 0, false);
75 uint32_t supportedChannels = static_cast<uint32_t>(*iter);
76 if (channels == supportedChannels) {
77 params.audioChannel = *iter;
78 isValidChannels = true;
79 break;
80 }
81 }
82 CHECK_AND_RETURN_RET(isValidChannels, false);
83
84 return true;
85 }
86
IsSupportedCaptureParameter(uint32_t bitrate,uint32_t channels,uint32_t sampleRate)87 bool AudioCaptureAsImpl::IsSupportedCaptureParameter(uint32_t bitrate, uint32_t channels, uint32_t sampleRate)
88 {
89 AudioStandard::AudioCapturerParams params;
90 CHECK_AND_RETURN_RET_LOG(CheckAndGetCaptureParameter(bitrate, channels, sampleRate, params), false,
91 "unsupport audio params");
92
93 return true;
94 }
95
SetCaptureParameter(uint32_t bitrate,uint32_t channels,uint32_t sampleRate,const AppInfo & appInfo)96 int32_t AudioCaptureAsImpl::SetCaptureParameter(uint32_t bitrate, uint32_t channels, uint32_t sampleRate,
97 const AppInfo &appInfo)
98 {
99 if (audioCapturer_ == nullptr) {
100 AudioStandard::AppInfo audioAppInfo = {};
101 audioAppInfo.appUid = appInfo.appUid;
102 audioAppInfo.appPid = appInfo.appPid;
103 audioAppInfo.appTokenId = appInfo.appTokenId;
104 audioAppInfo.appFullTokenId = appInfo.appFullTokenId;
105
106 audioCapturer_ = AudioStandard::AudioCapturer::Create(AudioStandard::AudioStreamType::STREAM_MUSIC,
107 audioAppInfo);
108 CHECK_AND_RETURN_RET_LOG(audioCapturer_ != nullptr, MSERR_NO_MEMORY, "create audio capturer failed");
109 }
110 audioCacheCtrl_ = std::make_unique<AudioCacheCtrl>();
111 CHECK_AND_RETURN_RET_LOG(audioCacheCtrl_ != nullptr, MSERR_NO_MEMORY, "create audio cache ctrl failed");
112
113 AudioStandard::AudioCapturerParams params;
114 CHECK_AND_RETURN_RET_LOG(CheckAndGetCaptureParameter(bitrate, channels, sampleRate, params),
115 MSERR_UNSUPPORT_AUD_PARAMS, "unsupport audio params");
116
117 params.audioSampleFormat = AudioStandard::SAMPLE_S16LE;
118 params.audioEncoding = AudioStandard::ENCODING_PCM;
119 MEDIA_LOGD("SetCaptureParameter out, channels:%{public}d, sampleRate:%{public}d",
120 params.audioChannel, params.samplingRate);
121 CHECK_AND_RETURN_RET(audioCapturer_->SetParams(params) == AudioStandard::SUCCESS, MSERR_UNKNOWN);
122 CHECK_AND_RETURN_RET(audioCapturer_->GetBufferSize(bufferSize_) == AudioStandard::SUCCESS, MSERR_UNKNOWN);
123 MEDIA_LOGD("audio buffer size is: %{public}zu", bufferSize_);
124 CHECK_AND_RETURN_RET_LOG(bufferSize_ < MAXIMUM_BUFFER_SIZE, MSERR_UNKNOWN, "audio buffer size too long");
125 return MSERR_OK;
126 }
127
GetCaptureParameter(uint32_t & bitrate,uint32_t & channels,uint32_t & sampleRate)128 int32_t AudioCaptureAsImpl::GetCaptureParameter(uint32_t &bitrate, uint32_t &channels, uint32_t &sampleRate)
129 {
130 (void)bitrate;
131 MEDIA_LOGD("GetCaptureParameter");
132 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
133 AudioStandard::AudioCapturerParams params;
134 CHECK_AND_RETURN_RET(audioCapturer_->GetParams(params) == AudioStandard::SUCCESS, MSERR_UNKNOWN);
135 channels = params.audioChannel;
136 sampleRate = params.samplingRate;
137 MEDIA_LOGD("get channels:%{public}u, sampleRate:%{public}u from audio server", channels, sampleRate);
138 CHECK_AND_RETURN_RET(bufferSize_ > 0 && channels > 0 && sampleRate > 0, MSERR_UNKNOWN);
139 constexpr uint32_t bitsPerByte = 8;
140 constexpr uint32_t audioSampleS16 = 16;
141 bufferDurationNs_ = (bufferSize_ * SEC_TO_NANOSECOND) / (sampleRate * (audioSampleS16 / bitsPerByte) * channels);
142
143 MEDIA_LOGD("audio frame duration is (%{public}" PRIu64 ") ns", bufferDurationNs_);
144 return MSERR_OK;
145 }
146
GetSegmentInfo(uint64_t & start)147 int32_t AudioCaptureAsImpl::GetSegmentInfo(uint64_t &start)
148 {
149 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
150 AudioStandard::Timestamp timeStamp;
151 auto timestampBase = AudioStandard::Timestamp::Timestampbase::MONOTONIC;
152 CHECK_AND_RETURN_RET_LOG(audioCapturer_->GetAudioTime(timeStamp, timestampBase), MSERR_UNKNOWN,
153 "failed to GetAudioTime");
154 CHECK_AND_RETURN_RET(timeStamp.time.tv_nsec >= 0 && timeStamp.time.tv_sec >= 0, MSERR_UNKNOWN);
155 if (((UINT64_MAX - timeStamp.time.tv_nsec) / SEC_TO_NANOSECOND) <= static_cast<uint64_t>(timeStamp.time.tv_sec)) {
156 MEDIA_LOGW("audio frame pts too long, this shouldn't happen");
157 }
158 start = timeStamp.time.tv_nsec + timeStamp.time.tv_sec * SEC_TO_NANOSECOND;
159 MEDIA_LOGD("timestamp from audioCapturer: %{public}" PRIu64 "", start);
160 MEDIA_LOGD("audioCapturer timestamp has increased: %{public}" PRIu64 "", start - lastInputTime_);
161 lastInputTime_ = start;
162
163 return MSERR_OK;
164 }
165
GetSegmentData()166 std::shared_ptr<AudioBuffer> AudioCaptureAsImpl::GetSegmentData()
167 {
168 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, nullptr);
169 std::shared_ptr<AudioBuffer> tempBuffer = std::make_shared<AudioBuffer>();
170 CHECK_AND_RETURN_RET(tempBuffer != nullptr, nullptr);
171 CHECK_AND_RETURN_RET(bufferSize_ > 0 && bufferSize_ < MAXIMUM_BUFFER_SIZE, nullptr);
172 tempBuffer->gstBuffer = gst_buffer_new_allocate(nullptr, bufferSize_, nullptr);
173 CHECK_AND_RETURN_RET(tempBuffer->gstBuffer != nullptr, nullptr);
174
175 GstMapInfo map = GST_MAP_INFO_INIT;
176 if (gst_buffer_map(tempBuffer->gstBuffer, &map, GST_MAP_READ) != TRUE) {
177 gst_buffer_unref(tempBuffer->gstBuffer);
178 return nullptr;
179 }
180 bool isBlocking = true;
181 int32_t bytesRead = audioCapturer_->Read(*(map.data), map.size, isBlocking);
182 gst_buffer_unmap(tempBuffer->gstBuffer, &map);
183 if (bytesRead <= 0) {
184 gst_buffer_unref(tempBuffer->gstBuffer);
185 MEDIA_LOGE("audioCapturer read size %{public}d!", bytesRead);
186 return nullptr;
187 }
188
189 return tempBuffer;
190 }
191
AudioCaptureLoop()192 int32_t AudioCaptureAsImpl::AudioCaptureLoop()
193 {
194 CHECK_AND_RETURN_RET_LOG(audioCacheCtrl_ != nullptr, MSERR_UNKNOWN, "audioCacheCtrl_ is nullptr!");
195 while (true) {
196 if ((curState_.load() != RECORDER_RUNNING) && (curState_ != RECORDER_RESUME)) {
197 return MSERR_OK;
198 }
199
200 {
201 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
202 if (audioCacheCtrl_->captureQueue_.size() >= MAX_QUEUE_SIZE) {
203 audioCacheCtrl_->captureCond_.notify_all();
204 continue;
205 }
206 }
207
208 std::shared_ptr<AudioBuffer> tempBuffer = GetSegmentData();
209 CHECK_AND_RETURN_RET_LOG(tempBuffer != nullptr, MSERR_UNKNOWN, "tempBuffer is nullptr!");
210
211 uint64_t curTimeStamp = 0;
212 if (GetSegmentInfo(curTimeStamp) != MSERR_OK) {
213 gst_buffer_unref(tempBuffer->gstBuffer);
214 MEDIA_LOGD("failed to GetSegmentInfo!");
215 return MSERR_UNKNOWN;
216 }
217
218 tempBuffer->timestamp = curTimeStamp;
219 tempBuffer->duration = bufferDurationNs_;
220 tempBuffer->dataLen = bufferSize_;
221
222 {
223 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
224 audioCacheCtrl_->captureQueue_.push(tempBuffer);
225 MEDIA_LOGD("audio cache queue size is %{public}zu", audioCacheCtrl_->captureQueue_.size());
226 audioCacheCtrl_->captureCond_.notify_all();
227 }
228 }
229 }
230
GetAudioCaptureBuffer()231 void AudioCaptureAsImpl::GetAudioCaptureBuffer()
232 {
233 MEDIA_LOGD("GetAudioCaptureBuffer");
234 CHECK_AND_RETURN_LOG(audioCacheCtrl_ != nullptr, "audioCacheCtrl_ is nullptr!");
235 pthread_setname_np(pthread_self(), "AudioCapture");
236 while (true) {
237 {
238 std::unique_lock<std::mutex> lock(pauseMutex_);
239 audioCacheCtrl_->pauseCond_.wait(lock, [this]() {
240 return curState_.load() != RECORDER_PAUSED;
241 });
242 }
243
244 if ((curState_.load() == RECORDER_STOP) || (curState_.load() == RECORDER_INITIALIZED)) {
245 MEDIA_LOGD("exit GetAudioCaptureBuffer!");
246 return;
247 }
248
249 int32_t ret = AudioCaptureLoop();
250 if (ret != MSERR_OK && ((curState_.load() == RECORDER_RUNNING) || (curState_ == RECORDER_RESUME))) {
251 // runtime error
252 captureLoopErr_.store(true);
253 audioCacheCtrl_->captureCond_.notify_all();
254 MEDIA_LOGE("failed to AudioCaptureLoop");
255 return;
256 }
257 }
258 }
259
GetBuffer()260 std::shared_ptr<AudioBuffer> AudioCaptureAsImpl::GetBuffer()
261 {
262 CHECK_AND_RETURN_RET_LOG(audioCacheCtrl_ != nullptr, nullptr, "audioCacheCtrl_ is nullptr!");
263 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
264
265 if (curState_.load() == RECORDER_RESUME && audioCacheCtrl_->pausedTime_ == 1) {
266 EmptyCaptureQueue();
267 }
268
269 audioCacheCtrl_->captureCond_.wait(loopLock, [this]() {
270 return ((audioCacheCtrl_->captureQueue_.size() > 0) || (curState_.load() == RECORDER_STOP) ||
271 captureLoopErr_.load());
272 });
273 if (curState_.load() == RECORDER_STOP) {
274 return nullptr;
275 }
276
277 CHECK_AND_RETURN_RET((audioCacheCtrl_->captureQueue_.size() > 0) || (!captureLoopErr_.load()), nullptr);
278
279 std::shared_ptr<AudioBuffer> bufferOut = audioCacheCtrl_->captureQueue_.front();
280 audioCacheCtrl_->captureQueue_.pop();
281
282 if (curState_.load() == RECORDER_PAUSED) {
283 audioCacheCtrl_->pausedTime_ = bufferOut->timestamp;
284 MEDIA_LOGD("audio pause timestamp %{public}" PRIu64 "", audioCacheCtrl_->pausedTime_);
285 EmptyCaptureQueue();
286 }
287 if (curState_.load() == RECORDER_RESUME) {
288 curState_.store(RECORDER_RUNNING);
289 if (audioCacheCtrl_->pausedTime_ == 1) {
290 audioCacheCtrl_->pausedTime_ = audioCacheCtrl_->lastTimeStamp_;
291 MEDIA_LOGD("audio pause timestamp %{public}" PRIu64 "", audioCacheCtrl_->pausedTime_);
292 }
293 audioCacheCtrl_->resumeTime_ = bufferOut->timestamp;
294 MEDIA_LOGD("audio resume timestamp %{public}" PRIu64 "", audioCacheCtrl_->resumeTime_);
295 audioCacheCtrl_->persistTime_ = std::fabs(audioCacheCtrl_->resumeTime_ - audioCacheCtrl_->pausedTime_);
296 if (audioCacheCtrl_->persistTime_ >= bufferDurationNs_) {
297 audioCacheCtrl_->persistTime_ -= bufferDurationNs_;
298 }
299 audioCacheCtrl_->pausedTime_ = 1; // reset the pause time
300 audioCacheCtrl_->totalPauseTime_ += audioCacheCtrl_->persistTime_;
301 MEDIA_LOGD("audio has %{public}d times pause, total PauseTime: %{public}" PRIu64 "",
302 audioCacheCtrl_->pausedCount_, audioCacheCtrl_->totalPauseTime_);
303 }
304 audioCacheCtrl_->lastTimeStamp_ = bufferOut->timestamp;
305 bufferOut->timestamp -= audioCacheCtrl_->totalPauseTime_;
306 return bufferOut;
307 }
308
StartAudioCapture()309 int32_t AudioCaptureAsImpl::StartAudioCapture()
310 {
311 MEDIA_LOGD("StartAudioCapture");
312
313 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
314 CHECK_AND_RETURN_RET(audioCapturer_->Start(), MSERR_UNKNOWN);
315
316 curState_.store(RECORDER_RUNNING);
317 captureLoop_ = std::make_unique<std::thread>(&AudioCaptureAsImpl::GetAudioCaptureBuffer, this);
318 CHECK_AND_RETURN_RET_LOG(captureLoop_ != nullptr, MSERR_INVALID_OPERATION, "create audio cache thread failed");
319
320 return MSERR_OK;
321 }
322
StopAudioCapture()323 int32_t AudioCaptureAsImpl::StopAudioCapture()
324 {
325 MEDIA_LOGD("StopAudioCapture");
326 CHECK_AND_RETURN_RET_LOG(audioCacheCtrl_ != nullptr, MSERR_UNKNOWN, "audioCacheCtrl_ is nullptr!");
327 curState_.store(RECORDER_STOP);
328
329 if (captureLoop_ != nullptr && captureLoop_->joinable()) {
330 audioCacheCtrl_->pauseCond_.notify_all();
331 captureLoop_->join();
332 captureLoop_.reset();
333 captureLoop_ = nullptr;
334 }
335
336 audioCacheCtrl_->captureCond_.notify_all();
337
338 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
339 if (audioCapturer_->GetStatus() == AudioStandard::CapturerState::CAPTURER_RUNNING) {
340 CHECK_AND_RETURN_RET(audioCapturer_->Stop(), MSERR_UNKNOWN);
341 }
342 if (audioCapturer_->GetStatus() != AudioStandard::CapturerState::CAPTURER_RELEASED) {
343 CHECK_AND_RETURN_RET(audioCapturer_->Release(), MSERR_UNKNOWN);
344 }
345
346 {
347 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
348 EmptyCaptureQueue();
349 }
350
351 audioCapturer_ = nullptr;
352 audioCacheCtrl_ = nullptr;
353 curState_.store(RECORDER_INITIALIZED);
354 MEDIA_LOGD("exit StopAudioCapture");
355 return MSERR_OK;
356 }
357
PauseAudioCapture()358 int32_t AudioCaptureAsImpl::PauseAudioCapture()
359 {
360 MEDIA_LOGD("PauseAudioCapture");
361 CHECK_AND_RETURN_RET_LOG(audioCacheCtrl_ != nullptr, MSERR_UNKNOWN, "audioCacheCtrl_ is nullptr!");
362 curState_.store(RECORDER_PAUSED);
363 audioCacheCtrl_->pausedCount_++;
364
365 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
366 if (audioCapturer_->GetStatus() == AudioStandard::CapturerState::CAPTURER_RUNNING) {
367 CHECK_AND_RETURN_RET(audioCapturer_->Stop(), MSERR_UNKNOWN);
368 }
369
370 MEDIA_LOGD("exit PauseAudioCapture");
371 return MSERR_OK;
372 }
373
ResumeAudioCapture()374 int32_t AudioCaptureAsImpl::ResumeAudioCapture()
375 {
376 MEDIA_LOGD("ResumeAudioCapture");
377
378 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
379 CHECK_AND_RETURN_RET(audioCapturer_->Start(), MSERR_UNKNOWN);
380 CHECK_AND_RETURN_RET_LOG(audioCacheCtrl_ != nullptr, MSERR_UNKNOWN, "audioCacheCtrl_ is nullptr!");
381
382 curState_.store(RECORDER_RESUME);
383 audioCacheCtrl_->pauseCond_.notify_all();
384
385 MEDIA_LOGD("exit ResumeAudioCapture");
386 return MSERR_OK;
387 }
388
WakeUpAudioThreads()389 int32_t AudioCaptureAsImpl::WakeUpAudioThreads()
390 {
391 MEDIA_LOGD("wake up threads when paused state");
392
393 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
394 CHECK_AND_RETURN_RET_LOG(audioCacheCtrl_ != nullptr, MSERR_UNKNOWN, "audioCacheCtrl_ is nullptr!");
395
396 curState_.store(RECORDER_STOP);
397 audioCacheCtrl_->pauseCond_.notify_all();
398
399 {
400 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
401 audioCacheCtrl_->captureCond_.notify_all();
402 }
403
404 return MSERR_OK;
405 }
406
EmptyCaptureQueue()407 void AudioCaptureAsImpl::EmptyCaptureQueue()
408 {
409 MEDIA_LOGD("%{public}zu audio buffer has been dropped", audioCacheCtrl_->captureQueue_.size());
410 CHECK_AND_RETURN_LOG(audioCacheCtrl_ != nullptr, "audioCacheCtrl_ is nullptr!");
411 while (!audioCacheCtrl_->captureQueue_.empty()) {
412 auto iter = audioCacheCtrl_->captureQueue_.front();
413 if (iter != nullptr) {
414 gst_buffer_unref(iter->gstBuffer);
415 }
416 audioCacheCtrl_->captureQueue_.pop();
417 }
418 }
419 } // namespace Media
420 } // namespace OHOS
421