1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "audio_capture_as_impl.h"
17 #include <vector>
18 #include <cmath>
19 #include "media_log.h"
20 #include "audio_errors.h"
21 #include "media_errors.h"
22
23 namespace {
24 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AudioCaptureAsImpl"};
25 constexpr size_t MAXIMUM_BUFFER_SIZE = 100000;
26 constexpr uint64_t SEC_TO_NANOSECOND = 1000000000;
27 }
28
29 namespace OHOS {
30 namespace Media {
AudioCaptureAsImpl()31 AudioCaptureAsImpl::AudioCaptureAsImpl()
32 {
33 }
34
~AudioCaptureAsImpl()35 AudioCaptureAsImpl::~AudioCaptureAsImpl()
36 {
37 if (audioCapturer_ != nullptr) {
38 (void)audioCapturer_->Release();
39 audioCapturer_ = nullptr;
40 }
41 }
42
CheckAndGetCaptureParameter(uint32_t bitrate,uint32_t channels,uint32_t sampleRate,AudioStandard::AudioCapturerParams & params)43 bool AudioCaptureAsImpl::CheckAndGetCaptureParameter(uint32_t bitrate, uint32_t channels, uint32_t sampleRate,
44 AudioStandard::AudioCapturerParams ¶ms)
45 {
46 (void)bitrate;
47 auto supportedSampleList = AudioStandard::AudioCapturer::GetSupportedSamplingRates();
48 CHECK_AND_RETURN_RET(supportedSampleList.size() > 0, false);
49 bool isValidSampleRate = false;
50 for (auto iter = supportedSampleList.cbegin(); iter != supportedSampleList.cend(); ++iter) {
51 CHECK_AND_RETURN_RET(static_cast<int32_t>(*iter) > 0, false);
52 uint32_t supportedSampleRate = static_cast<uint32_t>(*iter);
53 if (sampleRate <= supportedSampleRate) {
54 params.samplingRate = *iter;
55 isValidSampleRate = true;
56 break;
57 }
58 }
59 CHECK_AND_RETURN_RET(isValidSampleRate, false);
60
61 auto supportedChannelsList = AudioStandard::AudioCapturer::GetSupportedChannels();
62 CHECK_AND_RETURN_RET(supportedChannelsList.size() > 0, false);
63 bool isValidChannels = false;
64 for (auto iter = supportedChannelsList.cbegin(); iter != supportedChannelsList.cend(); ++iter) {
65 CHECK_AND_RETURN_RET(static_cast<int32_t>(*iter) > 0, false);
66 uint32_t supportedChannels = static_cast<uint32_t>(*iter);
67 if (channels == supportedChannels) {
68 params.audioChannel = *iter;
69 isValidChannels = true;
70 break;
71 }
72 }
73 CHECK_AND_RETURN_RET(isValidChannels, false);
74
75 return true;
76 }
77
IsSupportedCaptureParameter(uint32_t bitrate,uint32_t channels,uint32_t sampleRate)78 bool AudioCaptureAsImpl::IsSupportedCaptureParameter(uint32_t bitrate, uint32_t channels, uint32_t sampleRate)
79 {
80 AudioStandard::AudioCapturerParams params;
81 CHECK_AND_RETURN_RET_LOG(CheckAndGetCaptureParameter(bitrate, channels, sampleRate, params), false,
82 "unsupport audio params");
83
84 return true;
85 }
86
SetCaptureParameter(uint32_t bitrate,uint32_t channels,uint32_t sampleRate,const AppInfo & appInfo)87 int32_t AudioCaptureAsImpl::SetCaptureParameter(uint32_t bitrate, uint32_t channels, uint32_t sampleRate,
88 const AppInfo &appInfo)
89 {
90 if (audioCapturer_ == nullptr) {
91 AudioStandard::AppInfo audioAppInfo = {};
92 audioAppInfo.appUid = appInfo.appUid;
93 audioAppInfo.appPid = appInfo.appPid;
94 audioAppInfo.appTokenId = appInfo.appTokenId;
95
96 audioCapturer_ = AudioStandard::AudioCapturer::Create(AudioStandard::AudioStreamType::STREAM_MUSIC,
97 audioAppInfo);
98 CHECK_AND_RETURN_RET_LOG(audioCapturer_ != nullptr, MSERR_NO_MEMORY, "create audio capturer failed");
99 }
100 audioCacheCtrl_ = std::make_unique<AudioCacheCtrl>();
101 CHECK_AND_RETURN_RET_LOG(audioCacheCtrl_ != nullptr, MSERR_NO_MEMORY, "create audio cache ctrl failed");
102
103 AudioStandard::AudioCapturerParams params;
104 CHECK_AND_RETURN_RET_LOG(CheckAndGetCaptureParameter(bitrate, channels, sampleRate, params),
105 MSERR_UNSUPPORT_AUD_PARAMS, "unsupport audio params");
106
107 params.audioSampleFormat = AudioStandard::SAMPLE_S16LE;
108 params.audioEncoding = AudioStandard::ENCODING_PCM;
109 MEDIA_LOGD("SetCaptureParameter out, channels:%{public}d, sampleRate:%{public}d",
110 params.audioChannel, params.samplingRate);
111 CHECK_AND_RETURN_RET(audioCapturer_->SetParams(params) == AudioStandard::SUCCESS, MSERR_UNKNOWN);
112 CHECK_AND_RETURN_RET(audioCapturer_->GetBufferSize(bufferSize_) == AudioStandard::SUCCESS, MSERR_UNKNOWN);
113 MEDIA_LOGD("audio buffer size is: %{public}zu", bufferSize_);
114 CHECK_AND_RETURN_RET_LOG(bufferSize_ < MAXIMUM_BUFFER_SIZE, MSERR_UNKNOWN, "audio buffer size too long");
115 return MSERR_OK;
116 }
117
GetCaptureParameter(uint32_t & bitrate,uint32_t & channels,uint32_t & sampleRate)118 int32_t AudioCaptureAsImpl::GetCaptureParameter(uint32_t &bitrate, uint32_t &channels, uint32_t &sampleRate)
119 {
120 (void)bitrate;
121 MEDIA_LOGD("GetCaptureParameter");
122 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
123 AudioStandard::AudioCapturerParams params;
124 CHECK_AND_RETURN_RET(audioCapturer_->GetParams(params) == AudioStandard::SUCCESS, MSERR_UNKNOWN);
125 channels = params.audioChannel;
126 sampleRate = params.samplingRate;
127 MEDIA_LOGD("get channels:%{public}u, sampleRate:%{public}u from audio server", channels, sampleRate);
128 CHECK_AND_RETURN_RET(bufferSize_ > 0 && channels > 0 && sampleRate > 0, MSERR_UNKNOWN);
129 constexpr uint32_t bitsPerByte = 8;
130 constexpr uint32_t audioSampleS16 = 16;
131 bufferDurationNs_ = (bufferSize_ * SEC_TO_NANOSECOND) / (sampleRate * (audioSampleS16 / bitsPerByte) * channels);
132
133 MEDIA_LOGD("audio frame duration is (%{public}" PRIu64 ") ns", bufferDurationNs_);
134 return MSERR_OK;
135 }
136
GetSegmentInfo(uint64_t & start)137 int32_t AudioCaptureAsImpl::GetSegmentInfo(uint64_t &start)
138 {
139 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
140 AudioStandard::Timestamp timeStamp;
141 auto timestampBase = AudioStandard::Timestamp::Timestampbase::MONOTONIC;
142 CHECK_AND_RETURN_RET_LOG(audioCapturer_->GetAudioTime(timeStamp, timestampBase), MSERR_UNKNOWN,
143 "failed to GetAudioTime");
144 CHECK_AND_RETURN_RET(timeStamp.time.tv_nsec >= 0 && timeStamp.time.tv_sec >= 0, MSERR_UNKNOWN);
145 if (((UINT64_MAX - timeStamp.time.tv_nsec) / SEC_TO_NANOSECOND) <= static_cast<uint64_t>(timeStamp.time.tv_sec)) {
146 MEDIA_LOGW("audio frame pts too long, this shouldn't happen");
147 }
148 start = timeStamp.time.tv_nsec + timeStamp.time.tv_sec * SEC_TO_NANOSECOND;
149 MEDIA_LOGD("timestamp from audioCapturer: %{public}" PRIu64 "", start);
150 MEDIA_LOGD("audioCapturer timestamp has increased: %{public}" PRIu64 "", start - lastInputTime_);
151 lastInputTime_ = start;
152
153 return MSERR_OK;
154 }
155
GetSegmentData()156 std::shared_ptr<AudioBuffer> AudioCaptureAsImpl::GetSegmentData()
157 {
158 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, nullptr);
159 std::shared_ptr<AudioBuffer> tempBuffer = std::make_shared<AudioBuffer>();
160 CHECK_AND_RETURN_RET(tempBuffer != nullptr, nullptr);
161 CHECK_AND_RETURN_RET(bufferSize_ > 0 && bufferSize_ < MAXIMUM_BUFFER_SIZE, nullptr);
162 tempBuffer->gstBuffer = gst_buffer_new_allocate(nullptr, bufferSize_, nullptr);
163 CHECK_AND_RETURN_RET(tempBuffer->gstBuffer != nullptr, nullptr);
164
165 GstMapInfo map = GST_MAP_INFO_INIT;
166 if (gst_buffer_map(tempBuffer->gstBuffer, &map, GST_MAP_READ) != TRUE) {
167 gst_buffer_unref(tempBuffer->gstBuffer);
168 return nullptr;
169 }
170 bool isBlocking = true;
171 int32_t bytesRead = audioCapturer_->Read(*(map.data), map.size, isBlocking);
172 gst_buffer_unmap(tempBuffer->gstBuffer, &map);
173 if (bytesRead <= 0) {
174 gst_buffer_unref(tempBuffer->gstBuffer);
175 MEDIA_LOGE("audioCapturer read size %{public}d!", bytesRead);
176 return nullptr;
177 }
178
179 return tempBuffer;
180 }
181
AudioCaptureLoop()182 int32_t AudioCaptureAsImpl::AudioCaptureLoop()
183 {
184 while (true) {
185 if ((curState_.load() != RECORDER_RUNNING) && (curState_ != RECORDER_RESUME)) {
186 return MSERR_OK;
187 }
188
189 {
190 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
191 if (audioCacheCtrl_->captureQueue_.size() >= MAX_QUEUE_SIZE) {
192 audioCacheCtrl_->captureCond_.notify_all();
193 continue;
194 }
195 }
196
197 std::shared_ptr<AudioBuffer> tempBuffer = GetSegmentData();
198 CHECK_AND_RETURN_RET_LOG(tempBuffer != nullptr, MSERR_UNKNOWN, "tempBuffer is nullptr!");
199
200 uint64_t curTimeStamp = 0;
201 if (GetSegmentInfo(curTimeStamp) != MSERR_OK) {
202 gst_buffer_unref(tempBuffer->gstBuffer);
203 MEDIA_LOGD("failed to GetSegmentInfo!");
204 return MSERR_UNKNOWN;
205 }
206
207 tempBuffer->timestamp = curTimeStamp;
208 tempBuffer->duration = bufferDurationNs_;
209 tempBuffer->dataLen = bufferSize_;
210
211 {
212 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
213 audioCacheCtrl_->captureQueue_.push(tempBuffer);
214 MEDIA_LOGD("audio cache queue size is %{public}zu", audioCacheCtrl_->captureQueue_.size());
215 audioCacheCtrl_->captureCond_.notify_all();
216 }
217 }
218 }
219
GetAudioCaptureBuffer()220 void AudioCaptureAsImpl::GetAudioCaptureBuffer()
221 {
222 MEDIA_LOGD("GetAudioCaptureBuffer");
223 while (true) {
224 {
225 std::unique_lock<std::mutex> lock(pauseMutex_);
226 audioCacheCtrl_->pauseCond_.wait(lock, [this]() {
227 return curState_.load() != RECORDER_PAUSED;
228 });
229 }
230
231 if ((curState_.load() == RECORDER_STOP) || (curState_.load() == RECORDER_INITIALIZED)) {
232 MEDIA_LOGD("exit GetAudioCaptureBuffer!");
233 return;
234 }
235
236 int32_t ret = AudioCaptureLoop();
237 if (ret != MSERR_OK && ((curState_.load() == RECORDER_RUNNING) || (curState_ == RECORDER_RESUME))) {
238 // runtime error
239 captureLoopErr_.store(true);
240 audioCacheCtrl_->captureCond_.notify_all();
241 MEDIA_LOGE("failed to AudioCaptureLoop");
242 return;
243 }
244 }
245 }
246
GetBuffer()247 std::shared_ptr<AudioBuffer> AudioCaptureAsImpl::GetBuffer()
248 {
249 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
250
251 if (curState_.load() == RECORDER_RESUME && audioCacheCtrl_->pausedTime_ == 1) {
252 EmptyCaptureQueue();
253 }
254
255 audioCacheCtrl_->captureCond_.wait(loopLock, [this]() {
256 return ((audioCacheCtrl_->captureQueue_.size() > 0) || (curState_.load() == RECORDER_STOP) ||
257 captureLoopErr_.load());
258 });
259 if (curState_.load() == RECORDER_STOP) {
260 return nullptr;
261 }
262
263 CHECK_AND_RETURN_RET((audioCacheCtrl_->captureQueue_.size() > 0) || (!captureLoopErr_.load()), nullptr);
264
265 std::shared_ptr<AudioBuffer> bufferOut = audioCacheCtrl_->captureQueue_.front();
266 audioCacheCtrl_->captureQueue_.pop();
267
268 if (curState_.load() == RECORDER_PAUSED) {
269 audioCacheCtrl_->pausedTime_ = bufferOut->timestamp;
270 MEDIA_LOGD("audio pause timestamp %{public}" PRIu64 "", audioCacheCtrl_->pausedTime_);
271 EmptyCaptureQueue();
272 }
273 if (curState_.load() == RECORDER_RESUME) {
274 curState_.store(RECORDER_RUNNING);
275 if (audioCacheCtrl_->pausedTime_ == 1) {
276 audioCacheCtrl_->pausedTime_ = audioCacheCtrl_->lastTimeStamp_;
277 MEDIA_LOGD("audio pause timestamp %{public}" PRIu64 "", audioCacheCtrl_->pausedTime_);
278 }
279 audioCacheCtrl_->resumeTime_ = bufferOut->timestamp;
280 MEDIA_LOGD("audio resume timestamp %{public}" PRIu64 "", audioCacheCtrl_->resumeTime_);
281 audioCacheCtrl_->persistTime_ = std::fabs(audioCacheCtrl_->resumeTime_ - audioCacheCtrl_->pausedTime_);
282 if (audioCacheCtrl_->persistTime_ >= bufferDurationNs_) {
283 audioCacheCtrl_->persistTime_ -= bufferDurationNs_;
284 }
285 audioCacheCtrl_->pausedTime_ = 1; // reset the pause time
286 audioCacheCtrl_->totalPauseTime_ += audioCacheCtrl_->persistTime_;
287 MEDIA_LOGD("audio has %{public}d times pause, total PauseTime: %{public}" PRIu64 "",
288 audioCacheCtrl_->pausedCount_, audioCacheCtrl_->totalPauseTime_);
289 }
290 audioCacheCtrl_->lastTimeStamp_ = bufferOut->timestamp;
291 bufferOut->timestamp -= audioCacheCtrl_->totalPauseTime_;
292 return bufferOut;
293 }
294
StartAudioCapture()295 int32_t AudioCaptureAsImpl::StartAudioCapture()
296 {
297 MEDIA_LOGD("StartAudioCapture");
298
299 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
300 CHECK_AND_RETURN_RET(audioCapturer_->Start(), MSERR_UNKNOWN);
301
302 curState_.store(RECORDER_RUNNING);
303 captureLoop_ = std::make_unique<std::thread>(&AudioCaptureAsImpl::GetAudioCaptureBuffer, this);
304 CHECK_AND_RETURN_RET_LOG(captureLoop_ != nullptr, MSERR_INVALID_OPERATION, "create audio cache thread failed");
305
306 return MSERR_OK;
307 }
308
StopAudioCapture()309 int32_t AudioCaptureAsImpl::StopAudioCapture()
310 {
311 MEDIA_LOGD("StopAudioCapture");
312
313 curState_.store(RECORDER_STOP);
314
315 if (captureLoop_ != nullptr && captureLoop_->joinable()) {
316 audioCacheCtrl_->pauseCond_.notify_all();
317 captureLoop_->join();
318 captureLoop_.reset();
319 }
320
321 audioCacheCtrl_->captureCond_.notify_all();
322
323 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
324 if (audioCapturer_->GetStatus() == AudioStandard::CapturerState::CAPTURER_RUNNING) {
325 CHECK_AND_RETURN_RET(audioCapturer_->Stop(), MSERR_UNKNOWN);
326 }
327 if (audioCapturer_->GetStatus() != AudioStandard::CapturerState::CAPTURER_RELEASED) {
328 CHECK_AND_RETURN_RET(audioCapturer_->Release(), MSERR_UNKNOWN);
329 }
330
331 {
332 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
333 EmptyCaptureQueue();
334 }
335
336 audioCapturer_ = nullptr;
337 audioCacheCtrl_ = nullptr;
338 curState_.store(RECORDER_INITIALIZED);
339 MEDIA_LOGD("exit StopAudioCapture");
340 return MSERR_OK;
341 }
342
PauseAudioCapture()343 int32_t AudioCaptureAsImpl::PauseAudioCapture()
344 {
345 MEDIA_LOGD("PauseAudioCapture");
346
347 curState_.store(RECORDER_PAUSED);
348 audioCacheCtrl_->pausedCount_++;
349
350 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
351 if (audioCapturer_->GetStatus() == AudioStandard::CapturerState::CAPTURER_RUNNING) {
352 CHECK_AND_RETURN_RET(audioCapturer_->Stop(), MSERR_UNKNOWN);
353 }
354
355 MEDIA_LOGD("exit PauseAudioCapture");
356 return MSERR_OK;
357 }
358
ResumeAudioCapture()359 int32_t AudioCaptureAsImpl::ResumeAudioCapture()
360 {
361 MEDIA_LOGD("ResumeAudioCapture");
362
363 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
364 CHECK_AND_RETURN_RET(audioCapturer_->Start(), MSERR_UNKNOWN);
365
366 curState_.store(RECORDER_RESUME);
367 audioCacheCtrl_->pauseCond_.notify_all();
368
369 MEDIA_LOGD("exit ResumeAudioCapture");
370 return MSERR_OK;
371 }
372
WakeUpAudioThreads()373 int32_t AudioCaptureAsImpl::WakeUpAudioThreads()
374 {
375 MEDIA_LOGD("wake up threads when paused state");
376
377 CHECK_AND_RETURN_RET(audioCapturer_ != nullptr, MSERR_INVALID_OPERATION);
378
379 curState_.store(RECORDER_STOP);
380 audioCacheCtrl_->pauseCond_.notify_all();
381
382 {
383 std::unique_lock<std::mutex> loopLock(audioCacheCtrl_->captureMutex_);
384 audioCacheCtrl_->captureCond_.notify_all();
385 }
386
387 return MSERR_OK;
388 }
389
EmptyCaptureQueue()390 void AudioCaptureAsImpl::EmptyCaptureQueue()
391 {
392 MEDIA_LOGD("%{public}zu audio buffer has been dropped", audioCacheCtrl_->captureQueue_.size());
393 while (!audioCacheCtrl_->captureQueue_.empty()) {
394 auto iter = audioCacheCtrl_->captureQueue_.front();
395 if (iter != nullptr) {
396 gst_buffer_unref(iter->gstBuffer);
397 }
398 audioCacheCtrl_->captureQueue_.pop();
399 }
400 }
401 } // namespace Media
402 } // namespace OHOS
403