1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/audio_device/android/opensles_player.h"
12
13 #include <android/log.h>
14
15 #include "webrtc/base/arraysize.h"
16 #include "webrtc/base/checks.h"
17 #include "webrtc/base/format_macros.h"
18 #include "webrtc/base/timeutils.h"
19 #include "webrtc/modules/audio_device/android/audio_manager.h"
20 #include "webrtc/modules/audio_device/fine_audio_buffer.h"
21
22 #define TAG "OpenSLESPlayer"
23 #define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
24 #define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__)
25 #define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)
26 #define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
27 #define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
28
29 #define RETURN_ON_ERROR(op, ...) \
30 do { \
31 SLresult err = (op); \
32 if (err != SL_RESULT_SUCCESS) { \
33 ALOGE("%s failed: %d", #op, err); \
34 return __VA_ARGS__; \
35 } \
36 } while (0)
37
38 namespace webrtc {
39
OpenSLESPlayer(AudioManager * audio_manager)40 OpenSLESPlayer::OpenSLESPlayer(AudioManager* audio_manager)
41 : audio_parameters_(audio_manager->GetPlayoutAudioParameters()),
42 audio_device_buffer_(NULL),
43 initialized_(false),
44 playing_(false),
45 bytes_per_buffer_(0),
46 buffer_index_(0),
47 engine_(nullptr),
48 player_(nullptr),
49 simple_buffer_queue_(nullptr),
50 volume_(nullptr),
51 last_play_time_(0) {
52 ALOGD("ctor%s", GetThreadInfo().c_str());
53 // Use native audio output parameters provided by the audio manager and
54 // define the PCM format structure.
55 pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(),
56 audio_parameters_.sample_rate(),
57 audio_parameters_.bits_per_sample());
58 // Detach from this thread since we want to use the checker to verify calls
59 // from the internal audio thread.
60 thread_checker_opensles_.DetachFromThread();
61 }
62
~OpenSLESPlayer()63 OpenSLESPlayer::~OpenSLESPlayer() {
64 ALOGD("dtor%s", GetThreadInfo().c_str());
65 RTC_DCHECK(thread_checker_.CalledOnValidThread());
66 Terminate();
67 DestroyAudioPlayer();
68 DestroyMix();
69 DestroyEngine();
70 RTC_DCHECK(!engine_object_.Get());
71 RTC_DCHECK(!engine_);
72 RTC_DCHECK(!output_mix_.Get());
73 RTC_DCHECK(!player_);
74 RTC_DCHECK(!simple_buffer_queue_);
75 RTC_DCHECK(!volume_);
76 }
77
Init()78 int OpenSLESPlayer::Init() {
79 ALOGD("Init%s", GetThreadInfo().c_str());
80 RTC_DCHECK(thread_checker_.CalledOnValidThread());
81 return 0;
82 }
83
Terminate()84 int OpenSLESPlayer::Terminate() {
85 ALOGD("Terminate%s", GetThreadInfo().c_str());
86 RTC_DCHECK(thread_checker_.CalledOnValidThread());
87 StopPlayout();
88 return 0;
89 }
90
InitPlayout()91 int OpenSLESPlayer::InitPlayout() {
92 ALOGD("InitPlayout%s", GetThreadInfo().c_str());
93 RTC_DCHECK(thread_checker_.CalledOnValidThread());
94 RTC_DCHECK(!initialized_);
95 RTC_DCHECK(!playing_);
96 CreateEngine();
97 CreateMix();
98 initialized_ = true;
99 buffer_index_ = 0;
100 last_play_time_ = rtc::Time();
101 return 0;
102 }
103
StartPlayout()104 int OpenSLESPlayer::StartPlayout() {
105 ALOGD("StartPlayout%s", GetThreadInfo().c_str());
106 RTC_DCHECK(thread_checker_.CalledOnValidThread());
107 RTC_DCHECK(initialized_);
108 RTC_DCHECK(!playing_);
109 // The number of lower latency audio players is limited, hence we create the
110 // audio player in Start() and destroy it in Stop().
111 CreateAudioPlayer();
112 // Fill up audio buffers to avoid initial glitch and to ensure that playback
113 // starts when mode is later changed to SL_PLAYSTATE_PLAYING.
114 // TODO(henrika): we can save some delay by only making one call to
115 // EnqueuePlayoutData. Most likely not worth the risk of adding a glitch.
116 for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
117 EnqueuePlayoutData();
118 }
119 // Start streaming data by setting the play state to SL_PLAYSTATE_PLAYING.
120 // For a player object, when the object is in the SL_PLAYSTATE_PLAYING
121 // state, adding buffers will implicitly start playback.
122 RETURN_ON_ERROR((*player_)->SetPlayState(player_, SL_PLAYSTATE_PLAYING), -1);
123 playing_ = (GetPlayState() == SL_PLAYSTATE_PLAYING);
124 RTC_DCHECK(playing_);
125 return 0;
126 }
127
StopPlayout()128 int OpenSLESPlayer::StopPlayout() {
129 ALOGD("StopPlayout%s", GetThreadInfo().c_str());
130 RTC_DCHECK(thread_checker_.CalledOnValidThread());
131 if (!initialized_ || !playing_) {
132 return 0;
133 }
134 // Stop playing by setting the play state to SL_PLAYSTATE_STOPPED.
135 RETURN_ON_ERROR((*player_)->SetPlayState(player_, SL_PLAYSTATE_STOPPED), -1);
136 // Clear the buffer queue to flush out any remaining data.
137 RETURN_ON_ERROR((*simple_buffer_queue_)->Clear(simple_buffer_queue_), -1);
138 #ifndef NDEBUG
139 // Verify that the buffer queue is in fact cleared as it should.
140 SLAndroidSimpleBufferQueueState buffer_queue_state;
141 (*simple_buffer_queue_)->GetState(simple_buffer_queue_, &buffer_queue_state);
142 RTC_DCHECK_EQ(0u, buffer_queue_state.count);
143 RTC_DCHECK_EQ(0u, buffer_queue_state.index);
144 #endif
145 // The number of lower latency audio players is limited, hence we create the
146 // audio player in Start() and destroy it in Stop().
147 DestroyAudioPlayer();
148 thread_checker_opensles_.DetachFromThread();
149 initialized_ = false;
150 playing_ = false;
151 return 0;
152 }
153
SpeakerVolumeIsAvailable(bool & available)154 int OpenSLESPlayer::SpeakerVolumeIsAvailable(bool& available) {
155 available = false;
156 return 0;
157 }
158
MaxSpeakerVolume(uint32_t & maxVolume) const159 int OpenSLESPlayer::MaxSpeakerVolume(uint32_t& maxVolume) const {
160 return -1;
161 }
162
MinSpeakerVolume(uint32_t & minVolume) const163 int OpenSLESPlayer::MinSpeakerVolume(uint32_t& minVolume) const {
164 return -1;
165 }
166
SetSpeakerVolume(uint32_t volume)167 int OpenSLESPlayer::SetSpeakerVolume(uint32_t volume) {
168 return -1;
169 }
170
SpeakerVolume(uint32_t & volume) const171 int OpenSLESPlayer::SpeakerVolume(uint32_t& volume) const {
172 return -1;
173 }
174
AttachAudioBuffer(AudioDeviceBuffer * audioBuffer)175 void OpenSLESPlayer::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
176 ALOGD("AttachAudioBuffer");
177 RTC_DCHECK(thread_checker_.CalledOnValidThread());
178 audio_device_buffer_ = audioBuffer;
179 const int sample_rate_hz = audio_parameters_.sample_rate();
180 ALOGD("SetPlayoutSampleRate(%d)", sample_rate_hz);
181 audio_device_buffer_->SetPlayoutSampleRate(sample_rate_hz);
182 const size_t channels = audio_parameters_.channels();
183 ALOGD("SetPlayoutChannels(%" PRIuS ")", channels);
184 audio_device_buffer_->SetPlayoutChannels(channels);
185 RTC_CHECK(audio_device_buffer_);
186 AllocateDataBuffers();
187 }
188
CreatePCMConfiguration(size_t channels,int sample_rate,size_t bits_per_sample)189 SLDataFormat_PCM OpenSLESPlayer::CreatePCMConfiguration(
190 size_t channels,
191 int sample_rate,
192 size_t bits_per_sample) {
193 ALOGD("CreatePCMConfiguration");
194 RTC_CHECK_EQ(bits_per_sample, SL_PCMSAMPLEFORMAT_FIXED_16);
195 SLDataFormat_PCM format;
196 format.formatType = SL_DATAFORMAT_PCM;
197 format.numChannels = static_cast<SLuint32>(channels);
198 // Note that, the unit of sample rate is actually in milliHertz and not Hertz.
199 switch (sample_rate) {
200 case 8000:
201 format.samplesPerSec = SL_SAMPLINGRATE_8;
202 break;
203 case 16000:
204 format.samplesPerSec = SL_SAMPLINGRATE_16;
205 break;
206 case 22050:
207 format.samplesPerSec = SL_SAMPLINGRATE_22_05;
208 break;
209 case 32000:
210 format.samplesPerSec = SL_SAMPLINGRATE_32;
211 break;
212 case 44100:
213 format.samplesPerSec = SL_SAMPLINGRATE_44_1;
214 break;
215 case 48000:
216 format.samplesPerSec = SL_SAMPLINGRATE_48;
217 break;
218 default:
219 RTC_CHECK(false) << "Unsupported sample rate: " << sample_rate;
220 }
221 format.bitsPerSample = SL_PCMSAMPLEFORMAT_FIXED_16;
222 format.containerSize = SL_PCMSAMPLEFORMAT_FIXED_16;
223 format.endianness = SL_BYTEORDER_LITTLEENDIAN;
224 if (format.numChannels == 1)
225 format.channelMask = SL_SPEAKER_FRONT_CENTER;
226 else if (format.numChannels == 2)
227 format.channelMask = SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT;
228 else
229 RTC_CHECK(false) << "Unsupported number of channels: "
230 << format.numChannels;
231 return format;
232 }
233
AllocateDataBuffers()234 void OpenSLESPlayer::AllocateDataBuffers() {
235 ALOGD("AllocateDataBuffers");
236 RTC_DCHECK(thread_checker_.CalledOnValidThread());
237 RTC_DCHECK(!simple_buffer_queue_);
238 RTC_CHECK(audio_device_buffer_);
239 // Don't use the lowest possible size as native buffer size. Instead,
240 // use 10ms to better match the frame size that WebRTC uses. It will result
241 // in a reduced risk for audio glitches and also in a more "clean" sequence
242 // of callbacks from the OpenSL ES thread in to WebRTC when asking for audio
243 // to render.
244 ALOGD("lowest possible buffer size: %" PRIuS,
245 audio_parameters_.GetBytesPerBuffer());
246 bytes_per_buffer_ = audio_parameters_.GetBytesPerFrame() *
247 audio_parameters_.frames_per_10ms_buffer();
248 RTC_DCHECK_GE(bytes_per_buffer_, audio_parameters_.GetBytesPerBuffer());
249 ALOGD("native buffer size: %" PRIuS, bytes_per_buffer_);
250 // Create a modified audio buffer class which allows us to ask for any number
251 // of samples (and not only multiple of 10ms) to match the native OpenSL ES
252 // buffer size.
253 fine_buffer_.reset(new FineAudioBuffer(audio_device_buffer_,
254 bytes_per_buffer_,
255 audio_parameters_.sample_rate()));
256 // Each buffer must be of this size to avoid unnecessary memcpy while caching
257 // data between successive callbacks.
258 const size_t required_buffer_size =
259 fine_buffer_->RequiredPlayoutBufferSizeBytes();
260 ALOGD("required buffer size: %" PRIuS, required_buffer_size);
261 for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
262 audio_buffers_[i].reset(new SLint8[required_buffer_size]);
263 }
264 }
265
CreateEngine()266 bool OpenSLESPlayer::CreateEngine() {
267 ALOGD("CreateEngine");
268 RTC_DCHECK(thread_checker_.CalledOnValidThread());
269 if (engine_object_.Get())
270 return true;
271 RTC_DCHECK(!engine_);
272 const SLEngineOption option[] = {
273 {SL_ENGINEOPTION_THREADSAFE, static_cast<SLuint32>(SL_BOOLEAN_TRUE)}};
274 RETURN_ON_ERROR(
275 slCreateEngine(engine_object_.Receive(), 1, option, 0, NULL, NULL),
276 false);
277 RETURN_ON_ERROR(
278 engine_object_->Realize(engine_object_.Get(), SL_BOOLEAN_FALSE), false);
279 RETURN_ON_ERROR(engine_object_->GetInterface(engine_object_.Get(),
280 SL_IID_ENGINE, &engine_),
281 false);
282 return true;
283 }
284
DestroyEngine()285 void OpenSLESPlayer::DestroyEngine() {
286 ALOGD("DestroyEngine");
287 RTC_DCHECK(thread_checker_.CalledOnValidThread());
288 if (!engine_object_.Get())
289 return;
290 engine_ = nullptr;
291 engine_object_.Reset();
292 }
293
CreateMix()294 bool OpenSLESPlayer::CreateMix() {
295 ALOGD("CreateMix");
296 RTC_DCHECK(thread_checker_.CalledOnValidThread());
297 RTC_DCHECK(engine_);
298 if (output_mix_.Get())
299 return true;
300
301 // Create the ouput mix on the engine object. No interfaces will be used.
302 RETURN_ON_ERROR((*engine_)->CreateOutputMix(engine_, output_mix_.Receive(), 0,
303 NULL, NULL),
304 false);
305 RETURN_ON_ERROR(output_mix_->Realize(output_mix_.Get(), SL_BOOLEAN_FALSE),
306 false);
307 return true;
308 }
309
DestroyMix()310 void OpenSLESPlayer::DestroyMix() {
311 ALOGD("DestroyMix");
312 RTC_DCHECK(thread_checker_.CalledOnValidThread());
313 if (!output_mix_.Get())
314 return;
315 output_mix_.Reset();
316 }
317
CreateAudioPlayer()318 bool OpenSLESPlayer::CreateAudioPlayer() {
319 ALOGD("CreateAudioPlayer");
320 RTC_DCHECK(thread_checker_.CalledOnValidThread());
321 RTC_DCHECK(engine_object_.Get());
322 RTC_DCHECK(output_mix_.Get());
323 if (player_object_.Get())
324 return true;
325 RTC_DCHECK(!player_);
326 RTC_DCHECK(!simple_buffer_queue_);
327 RTC_DCHECK(!volume_);
328
329 // source: Android Simple Buffer Queue Data Locator is source.
330 SLDataLocator_AndroidSimpleBufferQueue simple_buffer_queue = {
331 SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
332 static_cast<SLuint32>(kNumOfOpenSLESBuffers)};
333 SLDataSource audio_source = {&simple_buffer_queue, &pcm_format_};
334
335 // sink: OutputMix-based data is sink.
336 SLDataLocator_OutputMix locator_output_mix = {SL_DATALOCATOR_OUTPUTMIX,
337 output_mix_.Get()};
338 SLDataSink audio_sink = {&locator_output_mix, NULL};
339
340 // Define interfaces that we indend to use and realize.
341 const SLInterfaceID interface_ids[] = {
342 SL_IID_ANDROIDCONFIGURATION, SL_IID_BUFFERQUEUE, SL_IID_VOLUME};
343 const SLboolean interface_required[] = {
344 SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE};
345
346 // Create the audio player on the engine interface.
347 RETURN_ON_ERROR(
348 (*engine_)->CreateAudioPlayer(
349 engine_, player_object_.Receive(), &audio_source, &audio_sink,
350 arraysize(interface_ids), interface_ids, interface_required),
351 false);
352
353 // Use the Android configuration interface to set platform-specific
354 // parameters. Should be done before player is realized.
355 SLAndroidConfigurationItf player_config;
356 RETURN_ON_ERROR(
357 player_object_->GetInterface(player_object_.Get(),
358 SL_IID_ANDROIDCONFIGURATION, &player_config),
359 false);
360 // Set audio player configuration to SL_ANDROID_STREAM_VOICE which
361 // corresponds to android.media.AudioManager.STREAM_VOICE_CALL.
362 SLint32 stream_type = SL_ANDROID_STREAM_VOICE;
363 RETURN_ON_ERROR(
364 (*player_config)
365 ->SetConfiguration(player_config, SL_ANDROID_KEY_STREAM_TYPE,
366 &stream_type, sizeof(SLint32)),
367 false);
368
369 // Realize the audio player object after configuration has been set.
370 RETURN_ON_ERROR(
371 player_object_->Realize(player_object_.Get(), SL_BOOLEAN_FALSE), false);
372
373 // Get the SLPlayItf interface on the audio player.
374 RETURN_ON_ERROR(
375 player_object_->GetInterface(player_object_.Get(), SL_IID_PLAY, &player_),
376 false);
377
378 // Get the SLAndroidSimpleBufferQueueItf interface on the audio player.
379 RETURN_ON_ERROR(
380 player_object_->GetInterface(player_object_.Get(), SL_IID_BUFFERQUEUE,
381 &simple_buffer_queue_),
382 false);
383
384 // Register callback method for the Android Simple Buffer Queue interface.
385 // This method will be called when the native audio layer needs audio data.
386 RETURN_ON_ERROR((*simple_buffer_queue_)
387 ->RegisterCallback(simple_buffer_queue_,
388 SimpleBufferQueueCallback, this),
389 false);
390
391 // Get the SLVolumeItf interface on the audio player.
392 RETURN_ON_ERROR(player_object_->GetInterface(player_object_.Get(),
393 SL_IID_VOLUME, &volume_),
394 false);
395
396 // TODO(henrika): might not be required to set volume to max here since it
397 // seems to be default on most devices. Might be required for unit tests.
398 // RETURN_ON_ERROR((*volume_)->SetVolumeLevel(volume_, 0), false);
399
400 return true;
401 }
402
DestroyAudioPlayer()403 void OpenSLESPlayer::DestroyAudioPlayer() {
404 ALOGD("DestroyAudioPlayer");
405 RTC_DCHECK(thread_checker_.CalledOnValidThread());
406 if (!player_object_.Get())
407 return;
408 player_object_.Reset();
409 player_ = nullptr;
410 simple_buffer_queue_ = nullptr;
411 volume_ = nullptr;
412 }
413
414 // static
SimpleBufferQueueCallback(SLAndroidSimpleBufferQueueItf caller,void * context)415 void OpenSLESPlayer::SimpleBufferQueueCallback(
416 SLAndroidSimpleBufferQueueItf caller,
417 void* context) {
418 OpenSLESPlayer* stream = reinterpret_cast<OpenSLESPlayer*>(context);
419 stream->FillBufferQueue();
420 }
421
FillBufferQueue()422 void OpenSLESPlayer::FillBufferQueue() {
423 RTC_DCHECK(thread_checker_opensles_.CalledOnValidThread());
424 SLuint32 state = GetPlayState();
425 if (state != SL_PLAYSTATE_PLAYING) {
426 ALOGW("Buffer callback in non-playing state!");
427 return;
428 }
429 EnqueuePlayoutData();
430 }
431
EnqueuePlayoutData()432 void OpenSLESPlayer::EnqueuePlayoutData() {
433 // Check delta time between two successive callbacks and provide a warning
434 // if it becomes very large.
435 // TODO(henrika): using 100ms as upper limit but this value is rather random.
436 const uint32_t current_time = rtc::Time();
437 const uint32_t diff = current_time - last_play_time_;
438 if (diff > 100) {
439 ALOGW("Bad OpenSL ES playout timing, dT=%u [ms]", diff);
440 }
441 last_play_time_ = current_time;
442 // Read audio data from the WebRTC source using the FineAudioBuffer object
443 // to adjust for differences in buffer size between WebRTC (10ms) and native
444 // OpenSL ES.
445 SLint8* audio_ptr = audio_buffers_[buffer_index_].get();
446 fine_buffer_->GetPlayoutData(audio_ptr);
447 // Enqueue the decoded audio buffer for playback.
448 SLresult err =
449 (*simple_buffer_queue_)
450 ->Enqueue(simple_buffer_queue_, audio_ptr, bytes_per_buffer_);
451 if (SL_RESULT_SUCCESS != err) {
452 ALOGE("Enqueue failed: %d", err);
453 }
454 buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers;
455 }
456
GetPlayState() const457 SLuint32 OpenSLESPlayer::GetPlayState() const {
458 RTC_DCHECK(player_);
459 SLuint32 state;
460 SLresult err = (*player_)->GetPlayState(player_, &state);
461 if (SL_RESULT_SUCCESS != err) {
462 ALOGE("GetPlayState failed: %d", err);
463 }
464 return state;
465 }
466
467 } // namespace webrtc
468