1 /*
2 * Copyright 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <cassert>
18
19 #include <SLES/OpenSLES.h>
20 #include <SLES/OpenSLES_Android.h>
21
22 #include "common/OboeDebug.h"
23 #include "oboe/AudioStreamBuilder.h"
24 #include "AudioInputStreamOpenSLES.h"
25 #include "AudioStreamOpenSLES.h"
26 #include "OpenSLESUtilities.h"
27
28 using namespace oboe;
29
OpenSLES_convertInputPreset(InputPreset oboePreset)30 static SLuint32 OpenSLES_convertInputPreset(InputPreset oboePreset) {
31 SLuint32 openslPreset = SL_ANDROID_RECORDING_PRESET_NONE;
32 switch(oboePreset) {
33 case InputPreset::Generic:
34 openslPreset = SL_ANDROID_RECORDING_PRESET_GENERIC;
35 break;
36 case InputPreset::Camcorder:
37 openslPreset = SL_ANDROID_RECORDING_PRESET_CAMCORDER;
38 break;
39 case InputPreset::VoiceRecognition:
40 case InputPreset::VoicePerformance:
41 openslPreset = SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION;
42 break;
43 case InputPreset::VoiceCommunication:
44 openslPreset = SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION;
45 break;
46 case InputPreset::Unprocessed:
47 openslPreset = SL_ANDROID_RECORDING_PRESET_UNPROCESSED;
48 break;
49 default:
50 break;
51 }
52 return openslPreset;
53 }
54
AudioInputStreamOpenSLES(const AudioStreamBuilder & builder)55 AudioInputStreamOpenSLES::AudioInputStreamOpenSLES(const AudioStreamBuilder &builder)
56 : AudioStreamOpenSLES(builder) {
57 }
58
~AudioInputStreamOpenSLES()59 AudioInputStreamOpenSLES::~AudioInputStreamOpenSLES() {
60 }
61
62 // Calculate masks specific to INPUT streams.
channelCountToChannelMask(int channelCount) const63 SLuint32 AudioInputStreamOpenSLES::channelCountToChannelMask(int channelCount) const {
64 // Derived from internal sles_channel_in_mask_from_count(chanCount);
65 // in "frameworks/wilhelm/src/android/channels.cpp".
66 // Yes, it seems strange to use SPEAKER constants to describe inputs.
67 // But that is how OpenSL ES does it internally.
68 switch (channelCount) {
69 case 1:
70 return SL_SPEAKER_FRONT_LEFT;
71 case 2:
72 return SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT;
73 default:
74 return channelCountToChannelMaskDefault(channelCount);
75 }
76 }
77
open()78 Result AudioInputStreamOpenSLES::open() {
79 logUnsupportedAttributes();
80
81 SLAndroidConfigurationItf configItf = nullptr;
82
83 if (getSdkVersion() < __ANDROID_API_M__ && mFormat == AudioFormat::Float){
84 // TODO: Allow floating point format on API <23 using float->int16 converter
85 return Result::ErrorInvalidFormat;
86 }
87
88 // If audio format is unspecified then choose a suitable default.
89 // API 23+: FLOAT
90 // API <23: INT16
91 if (mFormat == AudioFormat::Unspecified){
92 mFormat = (getSdkVersion() < __ANDROID_API_M__) ?
93 AudioFormat::I16 : AudioFormat::Float;
94 }
95
96 Result oboeResult = AudioStreamOpenSLES::open();
97 if (Result::OK != oboeResult) return oboeResult;
98
99 SLuint32 bitsPerSample = static_cast<SLuint32>(getBytesPerSample() * kBitsPerByte);
100
101 // configure audio sink
102 mBufferQueueLength = calculateOptimalBufferQueueLength();
103 SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {
104 SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, // locatorType
105 static_cast<SLuint32>(mBufferQueueLength)}; // numBuffers
106
107 // Define the audio data format.
108 SLDataFormat_PCM format_pcm = {
109 SL_DATAFORMAT_PCM, // formatType
110 static_cast<SLuint32>(mChannelCount), // numChannels
111 static_cast<SLuint32>(mSampleRate * kMillisPerSecond), // milliSamplesPerSec
112 bitsPerSample, // bitsPerSample
113 bitsPerSample, // containerSize;
114 channelCountToChannelMask(mChannelCount), // channelMask
115 getDefaultByteOrder(),
116 };
117
118 SLDataSink audioSink = {&loc_bufq, &format_pcm};
119
120 /**
121 * API 23 (Marshmallow) introduced support for floating-point data representation and an
122 * extended data format type: SLAndroidDataFormat_PCM_EX for recording streams (playback streams
123 * got this in API 21). If running on API 23+ use this newer format type, creating it from our
124 * original format.
125 */
126 SLAndroidDataFormat_PCM_EX format_pcm_ex;
127 if (getSdkVersion() >= __ANDROID_API_M__) {
128 SLuint32 representation = OpenSLES_ConvertFormatToRepresentation(getFormat());
129 // Fill in the format structure.
130 format_pcm_ex = OpenSLES_createExtendedFormat(format_pcm, representation);
131 // Use in place of the previous format.
132 audioSink.pFormat = &format_pcm_ex;
133 }
134
135
136 // configure audio source
137 SLDataLocator_IODevice loc_dev = {SL_DATALOCATOR_IODEVICE,
138 SL_IODEVICE_AUDIOINPUT,
139 SL_DEFAULTDEVICEID_AUDIOINPUT,
140 NULL};
141 SLDataSource audioSrc = {&loc_dev, NULL};
142
143 SLresult result = EngineOpenSLES::getInstance().createAudioRecorder(&mObjectInterface,
144 &audioSrc,
145 &audioSink);
146
147 if (SL_RESULT_SUCCESS != result) {
148 LOGE("createAudioRecorder() result:%s", getSLErrStr(result));
149 goto error;
150 }
151
152 // Configure the stream.
153 result = (*mObjectInterface)->GetInterface(mObjectInterface,
154 SL_IID_ANDROIDCONFIGURATION,
155 &configItf);
156
157 if (SL_RESULT_SUCCESS != result) {
158 LOGW("%s() GetInterface(SL_IID_ANDROIDCONFIGURATION) failed with %s",
159 __func__, getSLErrStr(result));
160 } else {
161 if (getInputPreset() == InputPreset::VoicePerformance) {
162 LOGD("OpenSL ES does not support InputPreset::VoicePerformance. Use VoiceRecognition.");
163 mInputPreset = InputPreset::VoiceRecognition;
164 }
165 SLuint32 presetValue = OpenSLES_convertInputPreset(getInputPreset());
166 result = (*configItf)->SetConfiguration(configItf,
167 SL_ANDROID_KEY_RECORDING_PRESET,
168 &presetValue,
169 sizeof(SLuint32));
170 if (SL_RESULT_SUCCESS != result
171 && presetValue != SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION) {
172 presetValue = SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION;
173 LOGD("Setting InputPreset %d failed. Using VoiceRecognition instead.", getInputPreset());
174 mInputPreset = InputPreset::VoiceRecognition;
175 (*configItf)->SetConfiguration(configItf,
176 SL_ANDROID_KEY_RECORDING_PRESET,
177 &presetValue,
178 sizeof(SLuint32));
179 }
180
181 result = configurePerformanceMode(configItf);
182 if (SL_RESULT_SUCCESS != result) {
183 goto error;
184 }
185 }
186
187 result = (*mObjectInterface)->Realize(mObjectInterface, SL_BOOLEAN_FALSE);
188 if (SL_RESULT_SUCCESS != result) {
189 LOGE("Realize recorder object result:%s", getSLErrStr(result));
190 goto error;
191 }
192
193 result = (*mObjectInterface)->GetInterface(mObjectInterface, SL_IID_RECORD, &mRecordInterface);
194 if (SL_RESULT_SUCCESS != result) {
195 LOGE("GetInterface RECORD result:%s", getSLErrStr(result));
196 goto error;
197 }
198
199 result = finishCommonOpen(configItf);
200 if (SL_RESULT_SUCCESS != result) {
201 goto error;
202 }
203
204 setState(StreamState::Open);
205 return Result::OK;
206
207 error:
208 close(); // Clean up various OpenSL objects and prevent resource leaks.
209 return Result::ErrorInternal; // TODO convert error from SLES to OBOE
210 }
211
close()212 Result AudioInputStreamOpenSLES::close() {
213 LOGD("AudioInputStreamOpenSLES::%s()", __func__);
214 std::lock_guard<std::mutex> lock(mLock);
215 Result result = Result::OK;
216 if (getState() == StreamState::Closed){
217 result = Result::ErrorClosed;
218 } else {
219 (void) requestStop_l();
220 if (OboeGlobals::areWorkaroundsEnabled()) {
221 sleepBeforeClose();
222 }
223 // invalidate any interfaces
224 mRecordInterface = nullptr;
225 result = AudioStreamOpenSLES::close_l();
226 }
227 return result;
228 }
229
setRecordState_l(SLuint32 newState)230 Result AudioInputStreamOpenSLES::setRecordState_l(SLuint32 newState) {
231 LOGD("AudioInputStreamOpenSLES::%s(%u)", __func__, newState);
232 Result result = Result::OK;
233
234 if (mRecordInterface == nullptr) {
235 LOGW("AudioInputStreamOpenSLES::%s() mRecordInterface is null", __func__);
236 return Result::ErrorInvalidState;
237 }
238 SLresult slResult = (*mRecordInterface)->SetRecordState(mRecordInterface, newState);
239 //LOGD("AudioInputStreamOpenSLES::%s(%u) returned %u", __func__, newState, slResult);
240 if (SL_RESULT_SUCCESS != slResult) {
241 LOGE("AudioInputStreamOpenSLES::%s(%u) returned error %s",
242 __func__, newState, getSLErrStr(slResult));
243 result = Result::ErrorInternal; // TODO review
244 }
245 return result;
246 }
247
requestStart()248 Result AudioInputStreamOpenSLES::requestStart() {
249 LOGD("AudioInputStreamOpenSLES(): %s() called", __func__);
250 std::lock_guard<std::mutex> lock(mLock);
251 StreamState initialState = getState();
252 switch (initialState) {
253 case StreamState::Starting:
254 case StreamState::Started:
255 return Result::OK;
256 case StreamState::Closed:
257 return Result::ErrorClosed;
258 default:
259 break;
260 }
261
262 // We use a callback if the user requests one
263 // OR if we have an internal callback to fill the blocking IO buffer.
264 setDataCallbackEnabled(true);
265
266 setState(StreamState::Starting);
267
268 if (getBufferDepth(mSimpleBufferQueueInterface) == 0) {
269 // Enqueue the first buffer to start the streaming.
270 // This does not call the callback function.
271 enqueueCallbackBuffer(mSimpleBufferQueueInterface);
272 }
273
274 Result result = setRecordState_l(SL_RECORDSTATE_RECORDING);
275 if (result == Result::OK) {
276 setState(StreamState::Started);
277 } else {
278 setState(initialState);
279 }
280 return result;
281 }
282
283
requestPause()284 Result AudioInputStreamOpenSLES::requestPause() {
285 LOGW("AudioInputStreamOpenSLES::%s() is intentionally not implemented for input "
286 "streams", __func__);
287 return Result::ErrorUnimplemented; // Matches AAudio behavior.
288 }
289
requestFlush()290 Result AudioInputStreamOpenSLES::requestFlush() {
291 LOGW("AudioInputStreamOpenSLES::%s() is intentionally not implemented for input "
292 "streams", __func__);
293 return Result::ErrorUnimplemented; // Matches AAudio behavior.
294 }
295
requestStop()296 Result AudioInputStreamOpenSLES::requestStop() {
297 LOGD("AudioInputStreamOpenSLES(): %s() called", __func__);
298 std::lock_guard<std::mutex> lock(mLock);
299 return requestStop_l();
300 }
301
302 // Call under mLock
requestStop_l()303 Result AudioInputStreamOpenSLES::requestStop_l() {
304 StreamState initialState = getState();
305 switch (initialState) {
306 case StreamState::Stopping:
307 case StreamState::Stopped:
308 return Result::OK;
309 case StreamState::Uninitialized:
310 case StreamState::Closed:
311 return Result::ErrorClosed;
312 default:
313 break;
314 }
315
316 setState(StreamState::Stopping);
317
318 Result result = setRecordState_l(SL_RECORDSTATE_STOPPED);
319 if (result == Result::OK) {
320 mPositionMillis.reset32(); // OpenSL ES resets its millisecond position when stopped.
321 setState(StreamState::Stopped);
322 } else {
323 setState(initialState);
324 }
325 return result;
326 }
327
updateFramesWritten()328 void AudioInputStreamOpenSLES::updateFramesWritten() {
329 if (usingFIFO()) {
330 AudioStreamBuffered::updateFramesWritten();
331 } else {
332 mFramesWritten = getFramesProcessedByServer();
333 }
334 }
335
updateServiceFrameCounter()336 Result AudioInputStreamOpenSLES::updateServiceFrameCounter() {
337 Result result = Result::OK;
338 // Avoid deadlock if another thread is trying to stop or close this stream
339 // and this is being called from a callback.
340 if (mLock.try_lock()) {
341
342 if (mRecordInterface == nullptr) {
343 mLock.unlock();
344 return Result::ErrorNull;
345 }
346 SLmillisecond msec = 0;
347 SLresult slResult = (*mRecordInterface)->GetPosition(mRecordInterface, &msec);
348 if (SL_RESULT_SUCCESS != slResult) {
349 LOGW("%s(): GetPosition() returned %s", __func__, getSLErrStr(slResult));
350 // set result based on SLresult
351 result = Result::ErrorInternal;
352 } else {
353 mPositionMillis.update32(msec);
354 }
355 mLock.unlock();
356 }
357 return result;
358 }
359