1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/audio_device/android/opensles_output.h"
12
13 #include <assert.h>
14
15 #include "webrtc/modules/audio_device/android/opensles_common.h"
16 #include "webrtc/modules/audio_device/android/fine_audio_buffer.h"
17 #include "webrtc/modules/audio_device/android/single_rw_fifo.h"
18 #include "webrtc/modules/audio_device/audio_device_buffer.h"
19 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
20 #include "webrtc/system_wrappers/interface/thread_wrapper.h"
21 #include "webrtc/system_wrappers/interface/trace.h"
22
23 #define VOID_RETURN
24 #define OPENSL_RETURN_ON_FAILURE(op, ret_val) \
25 do { \
26 SLresult err = (op); \
27 if (err != SL_RESULT_SUCCESS) { \
28 WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, \
29 "OpenSL error: %d", err); \
30 assert(false); \
31 return ret_val; \
32 } \
33 } while (0)
34
35 static const SLEngineOption kOption[] = {
36 { SL_ENGINEOPTION_THREADSAFE, static_cast<SLuint32>(SL_BOOLEAN_TRUE) },
37 };
38
39 enum {
40 kNoUnderrun,
41 kUnderrun,
42 };
43
44 namespace webrtc {
45
OpenSlesOutput(const int32_t id)46 OpenSlesOutput::OpenSlesOutput(const int32_t id)
47 : id_(id),
48 initialized_(false),
49 speaker_initialized_(false),
50 play_initialized_(false),
51 crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
52 playing_(false),
53 num_fifo_buffers_needed_(0),
54 number_underruns_(0),
55 sles_engine_(NULL),
56 sles_engine_itf_(NULL),
57 sles_player_(NULL),
58 sles_player_itf_(NULL),
59 sles_player_sbq_itf_(NULL),
60 sles_output_mixer_(NULL),
61 audio_buffer_(NULL),
62 active_queue_(0),
63 speaker_sampling_rate_(kDefaultSampleRate),
64 buffer_size_samples_(0),
65 buffer_size_bytes_(0),
66 playout_delay_(0) {
67 }
68
~OpenSlesOutput()69 OpenSlesOutput::~OpenSlesOutput() {
70 }
71
SetAndroidAudioDeviceObjects(void * javaVM,void * env,void * context)72 int32_t OpenSlesOutput::SetAndroidAudioDeviceObjects(void* javaVM,
73 void* env,
74 void* context) {
75 AudioManagerJni::SetAndroidAudioDeviceObjects(javaVM, env, context);
76 return 0;
77 }
78
ClearAndroidAudioDeviceObjects()79 void OpenSlesOutput::ClearAndroidAudioDeviceObjects() {
80 AudioManagerJni::ClearAndroidAudioDeviceObjects();
81 }
82
Init()83 int32_t OpenSlesOutput::Init() {
84 assert(!initialized_);
85
86 // Set up OpenSl engine.
87 OPENSL_RETURN_ON_FAILURE(slCreateEngine(&sles_engine_, 1, kOption, 0,
88 NULL, NULL),
89 -1);
90 OPENSL_RETURN_ON_FAILURE((*sles_engine_)->Realize(sles_engine_,
91 SL_BOOLEAN_FALSE),
92 -1);
93 OPENSL_RETURN_ON_FAILURE((*sles_engine_)->GetInterface(sles_engine_,
94 SL_IID_ENGINE,
95 &sles_engine_itf_),
96 -1);
97 // Set up OpenSl output mix.
98 OPENSL_RETURN_ON_FAILURE(
99 (*sles_engine_itf_)->CreateOutputMix(sles_engine_itf_,
100 &sles_output_mixer_,
101 0,
102 NULL,
103 NULL),
104 -1);
105 OPENSL_RETURN_ON_FAILURE(
106 (*sles_output_mixer_)->Realize(sles_output_mixer_,
107 SL_BOOLEAN_FALSE),
108 -1);
109
110 if (!InitSampleRate()) {
111 return -1;
112 }
113 AllocateBuffers();
114 initialized_ = true;
115 return 0;
116 }
117
Terminate()118 int32_t OpenSlesOutput::Terminate() {
119 // It is assumed that the caller has stopped recording before terminating.
120 assert(!playing_);
121 (*sles_output_mixer_)->Destroy(sles_output_mixer_);
122 (*sles_engine_)->Destroy(sles_engine_);
123 initialized_ = false;
124 speaker_initialized_ = false;
125 play_initialized_ = false;
126 return 0;
127 }
128
PlayoutDeviceName(uint16_t index,char name[kAdmMaxDeviceNameSize],char guid[kAdmMaxGuidSize])129 int32_t OpenSlesOutput::PlayoutDeviceName(uint16_t index,
130 char name[kAdmMaxDeviceNameSize],
131 char guid[kAdmMaxGuidSize]) {
132 assert(index == 0);
133 // Empty strings.
134 name[0] = '\0';
135 guid[0] = '\0';
136 return 0;
137 }
138
SetPlayoutDevice(uint16_t index)139 int32_t OpenSlesOutput::SetPlayoutDevice(uint16_t index) {
140 assert(index == 0);
141 return 0;
142 }
143
PlayoutIsAvailable(bool & available)144 int32_t OpenSlesOutput::PlayoutIsAvailable(bool& available) { // NOLINT
145 available = true;
146 return 0;
147 }
148
InitPlayout()149 int32_t OpenSlesOutput::InitPlayout() {
150 assert(initialized_);
151 play_initialized_ = true;
152 return 0;
153 }
154
StartPlayout()155 int32_t OpenSlesOutput::StartPlayout() {
156 assert(play_initialized_);
157 assert(!playing_);
158 if (!CreateAudioPlayer()) {
159 return -1;
160 }
161
162 // Register callback to receive enqueued buffers.
163 OPENSL_RETURN_ON_FAILURE(
164 (*sles_player_sbq_itf_)->RegisterCallback(sles_player_sbq_itf_,
165 PlayerSimpleBufferQueueCallback,
166 this),
167 -1);
168 if (!EnqueueAllBuffers()) {
169 return -1;
170 }
171
172 {
173 // To prevent the compiler from e.g. optimizing the code to
174 // playing_ = StartCbThreads() which wouldn't have been thread safe.
175 CriticalSectionScoped lock(crit_sect_.get());
176 playing_ = true;
177 }
178 if (!StartCbThreads()) {
179 playing_ = false;
180 }
181 return 0;
182 }
183
StopPlayout()184 int32_t OpenSlesOutput::StopPlayout() {
185 StopCbThreads();
186 DestroyAudioPlayer();
187 playing_ = false;
188 return 0;
189 }
190
InitSpeaker()191 int32_t OpenSlesOutput::InitSpeaker() {
192 assert(!playing_);
193 speaker_initialized_ = true;
194 return 0;
195 }
196
SpeakerVolumeIsAvailable(bool & available)197 int32_t OpenSlesOutput::SpeakerVolumeIsAvailable(bool& available) { // NOLINT
198 available = true;
199 return 0;
200 }
201
SetSpeakerVolume(uint32_t volume)202 int32_t OpenSlesOutput::SetSpeakerVolume(uint32_t volume) {
203 assert(speaker_initialized_);
204 assert(initialized_);
205 // TODO(hellner): implement.
206 return 0;
207 }
208
MaxSpeakerVolume(uint32_t & maxVolume) const209 int32_t OpenSlesOutput::MaxSpeakerVolume(uint32_t& maxVolume) const { // NOLINT
210 assert(speaker_initialized_);
211 assert(initialized_);
212 // TODO(hellner): implement.
213 maxVolume = 0;
214 return 0;
215 }
216
MinSpeakerVolume(uint32_t & minVolume) const217 int32_t OpenSlesOutput::MinSpeakerVolume(uint32_t& minVolume) const { // NOLINT
218 assert(speaker_initialized_);
219 assert(initialized_);
220 // TODO(hellner): implement.
221 minVolume = 0;
222 return 0;
223 }
224
SpeakerVolumeStepSize(uint16_t & stepSize) const225 int32_t OpenSlesOutput::SpeakerVolumeStepSize(
226 uint16_t& stepSize) const { // NOLINT
227 assert(speaker_initialized_);
228 stepSize = 1;
229 return 0;
230 }
231
SpeakerMuteIsAvailable(bool & available)232 int32_t OpenSlesOutput::SpeakerMuteIsAvailable(bool& available) { // NOLINT
233 available = false;
234 return 0;
235 }
236
StereoPlayoutIsAvailable(bool & available)237 int32_t OpenSlesOutput::StereoPlayoutIsAvailable(bool& available) { // NOLINT
238 available = false;
239 return 0;
240 }
241
SetStereoPlayout(bool enable)242 int32_t OpenSlesOutput::SetStereoPlayout(bool enable) {
243 if (enable) {
244 assert(false);
245 return -1;
246 }
247 return 0;
248 }
249
StereoPlayout(bool & enabled) const250 int32_t OpenSlesOutput::StereoPlayout(bool& enabled) const { // NOLINT
251 enabled = kNumChannels == 2;
252 return 0;
253 }
254
PlayoutBuffer(AudioDeviceModule::BufferType & type,uint16_t & sizeMS) const255 int32_t OpenSlesOutput::PlayoutBuffer(
256 AudioDeviceModule::BufferType& type, // NOLINT
257 uint16_t& sizeMS) const { // NOLINT
258 type = AudioDeviceModule::kAdaptiveBufferSize;
259 sizeMS = playout_delay_;
260 return 0;
261 }
262
PlayoutDelay(uint16_t & delayMS) const263 int32_t OpenSlesOutput::PlayoutDelay(uint16_t& delayMS) const { // NOLINT
264 delayMS = playout_delay_;
265 return 0;
266 }
267
AttachAudioBuffer(AudioDeviceBuffer * audioBuffer)268 void OpenSlesOutput::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
269 audio_buffer_ = audioBuffer;
270 }
271
SetLoudspeakerStatus(bool enable)272 int32_t OpenSlesOutput::SetLoudspeakerStatus(bool enable) {
273 return 0;
274 }
275
GetLoudspeakerStatus(bool & enabled) const276 int32_t OpenSlesOutput::GetLoudspeakerStatus(bool& enabled) const { // NOLINT
277 enabled = true;
278 return 0;
279 }
280
PlayoutDelayMs()281 int OpenSlesOutput::PlayoutDelayMs() {
282 return playout_delay_;
283 }
284
InitSampleRate()285 bool OpenSlesOutput::InitSampleRate() {
286 if (!SetLowLatency()) {
287 speaker_sampling_rate_ = kDefaultSampleRate;
288 // Default is to use 10ms buffers.
289 buffer_size_samples_ = speaker_sampling_rate_ * 10 / 1000;
290 }
291 if (audio_buffer_->SetPlayoutSampleRate(speaker_sampling_rate_) < 0) {
292 return false;
293 }
294 if (audio_buffer_->SetPlayoutChannels(kNumChannels) < 0) {
295 return false;
296 }
297 UpdatePlayoutDelay();
298 return true;
299 }
300
UpdatePlayoutDelay()301 void OpenSlesOutput::UpdatePlayoutDelay() {
302 // TODO(hellner): Add accurate delay estimate.
303 // On average half the current buffer will have been played out.
304 int outstanding_samples = (TotalBuffersUsed() - 0.5) * buffer_size_samples_;
305 playout_delay_ = outstanding_samples / (speaker_sampling_rate_ / 1000);
306 }
307
SetLowLatency()308 bool OpenSlesOutput::SetLowLatency() {
309 if (!audio_manager_.low_latency_supported()) {
310 return false;
311 }
312 buffer_size_samples_ = audio_manager_.native_buffer_size();
313 assert(buffer_size_samples_ > 0);
314 speaker_sampling_rate_ = audio_manager_.native_output_sample_rate();
315 assert(speaker_sampling_rate_ > 0);
316 return true;
317 }
318
CalculateNumFifoBuffersNeeded()319 void OpenSlesOutput::CalculateNumFifoBuffersNeeded() {
320 int number_of_bytes_needed =
321 (speaker_sampling_rate_ * kNumChannels * sizeof(int16_t)) * 10 / 1000;
322
323 // Ceiling of integer division: 1 + ((x - 1) / y)
324 int buffers_per_10_ms =
325 1 + ((number_of_bytes_needed - 1) / buffer_size_bytes_);
326 // |num_fifo_buffers_needed_| is a multiple of 10ms of buffered up audio.
327 num_fifo_buffers_needed_ = kNum10MsToBuffer * buffers_per_10_ms;
328 }
329
AllocateBuffers()330 void OpenSlesOutput::AllocateBuffers() {
331 // Allocate fine buffer to provide frames of the desired size.
332 buffer_size_bytes_ = buffer_size_samples_ * kNumChannels * sizeof(int16_t);
333 fine_buffer_.reset(new FineAudioBuffer(audio_buffer_, buffer_size_bytes_,
334 speaker_sampling_rate_));
335
336 // Allocate FIFO to handle passing buffers between processing and OpenSl
337 // threads.
338 CalculateNumFifoBuffersNeeded(); // Needs |buffer_size_bytes_| to be known
339 assert(num_fifo_buffers_needed_ > 0);
340 fifo_.reset(new SingleRwFifo(num_fifo_buffers_needed_));
341
342 // Allocate the memory area to be used.
343 play_buf_.reset(new scoped_ptr<int8_t[]>[TotalBuffersUsed()]);
344 int required_buffer_size = fine_buffer_->RequiredBufferSizeBytes();
345 for (int i = 0; i < TotalBuffersUsed(); ++i) {
346 play_buf_[i].reset(new int8_t[required_buffer_size]);
347 }
348 }
349
TotalBuffersUsed() const350 int OpenSlesOutput::TotalBuffersUsed() const {
351 return num_fifo_buffers_needed_ + kNumOpenSlBuffers;
352 }
353
EnqueueAllBuffers()354 bool OpenSlesOutput::EnqueueAllBuffers() {
355 active_queue_ = 0;
356 number_underruns_ = 0;
357 for (int i = 0; i < kNumOpenSlBuffers; ++i) {
358 memset(play_buf_[i].get(), 0, buffer_size_bytes_);
359 OPENSL_RETURN_ON_FAILURE(
360 (*sles_player_sbq_itf_)->Enqueue(
361 sles_player_sbq_itf_,
362 reinterpret_cast<void*>(play_buf_[i].get()),
363 buffer_size_bytes_),
364 false);
365 }
366 // OpenSL playing has been stopped. I.e. only this thread is touching
367 // |fifo_|.
368 while (fifo_->size() != 0) {
369 // Underrun might have happened when pushing new buffers to the FIFO.
370 fifo_->Pop();
371 }
372 for (int i = kNumOpenSlBuffers; i < TotalBuffersUsed(); ++i) {
373 memset(play_buf_[i].get(), 0, buffer_size_bytes_);
374 fifo_->Push(play_buf_[i].get());
375 }
376 return true;
377 }
378
CreateAudioPlayer()379 bool OpenSlesOutput::CreateAudioPlayer() {
380 if (!event_.Start()) {
381 assert(false);
382 return false;
383 }
384 SLDataLocator_AndroidSimpleBufferQueue simple_buf_queue = {
385 SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
386 static_cast<SLuint32>(kNumOpenSlBuffers)
387 };
388 SLDataFormat_PCM configuration =
389 webrtc_opensl::CreatePcmConfiguration(speaker_sampling_rate_);
390 SLDataSource audio_source = { &simple_buf_queue, &configuration };
391
392 SLDataLocator_OutputMix locator_outputmix;
393 // Setup the data sink structure.
394 locator_outputmix.locatorType = SL_DATALOCATOR_OUTPUTMIX;
395 locator_outputmix.outputMix = sles_output_mixer_;
396 SLDataSink audio_sink = { &locator_outputmix, NULL };
397
398 // Interfaces for streaming audio data, setting volume and Android are needed.
399 // Note the interfaces still need to be initialized. This only tells OpenSl
400 // that the interfaces will be needed at some point.
401 SLInterfaceID ids[kNumInterfaces] = {
402 SL_IID_BUFFERQUEUE, SL_IID_VOLUME, SL_IID_ANDROIDCONFIGURATION };
403 SLboolean req[kNumInterfaces] = {
404 SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE };
405 OPENSL_RETURN_ON_FAILURE(
406 (*sles_engine_itf_)->CreateAudioPlayer(sles_engine_itf_, &sles_player_,
407 &audio_source, &audio_sink,
408 kNumInterfaces, ids, req),
409 false);
410 // Realize the player in synchronous mode.
411 OPENSL_RETURN_ON_FAILURE((*sles_player_)->Realize(sles_player_,
412 SL_BOOLEAN_FALSE),
413 false);
414 OPENSL_RETURN_ON_FAILURE(
415 (*sles_player_)->GetInterface(sles_player_, SL_IID_PLAY,
416 &sles_player_itf_),
417 false);
418 OPENSL_RETURN_ON_FAILURE(
419 (*sles_player_)->GetInterface(sles_player_, SL_IID_BUFFERQUEUE,
420 &sles_player_sbq_itf_),
421 false);
422 return true;
423 }
424
DestroyAudioPlayer()425 void OpenSlesOutput::DestroyAudioPlayer() {
426 SLAndroidSimpleBufferQueueItf sles_player_sbq_itf = sles_player_sbq_itf_;
427 {
428 CriticalSectionScoped lock(crit_sect_.get());
429 sles_player_sbq_itf_ = NULL;
430 sles_player_itf_ = NULL;
431 }
432 event_.Stop();
433 if (sles_player_sbq_itf) {
434 // Release all buffers currently queued up.
435 OPENSL_RETURN_ON_FAILURE(
436 (*sles_player_sbq_itf)->Clear(sles_player_sbq_itf),
437 VOID_RETURN);
438 }
439
440 if (sles_player_) {
441 (*sles_player_)->Destroy(sles_player_);
442 sles_player_ = NULL;
443 }
444 }
445
HandleUnderrun(int event_id,int event_msg)446 bool OpenSlesOutput::HandleUnderrun(int event_id, int event_msg) {
447 if (!playing_) {
448 return false;
449 }
450 if (event_id == kNoUnderrun) {
451 return false;
452 }
453 WEBRTC_TRACE(kTraceWarning, kTraceAudioDevice, id_, "Audio underrun");
454 assert(event_id == kUnderrun);
455 assert(event_msg > 0);
456 // Wait for all enqueued buffers to be flushed.
457 if (event_msg != kNumOpenSlBuffers) {
458 return true;
459 }
460 // All buffers have been flushed. Restart the audio from scratch.
461 // No need to check sles_player_itf_ as playing_ would be false before it is
462 // set to NULL.
463 OPENSL_RETURN_ON_FAILURE(
464 (*sles_player_itf_)->SetPlayState(sles_player_itf_,
465 SL_PLAYSTATE_STOPPED),
466 true);
467 EnqueueAllBuffers();
468 OPENSL_RETURN_ON_FAILURE(
469 (*sles_player_itf_)->SetPlayState(sles_player_itf_,
470 SL_PLAYSTATE_PLAYING),
471 true);
472 return true;
473 }
474
PlayerSimpleBufferQueueCallback(SLAndroidSimpleBufferQueueItf sles_player_sbq_itf,void * p_context)475 void OpenSlesOutput::PlayerSimpleBufferQueueCallback(
476 SLAndroidSimpleBufferQueueItf sles_player_sbq_itf,
477 void* p_context) {
478 OpenSlesOutput* audio_device = reinterpret_cast<OpenSlesOutput*>(p_context);
479 audio_device->PlayerSimpleBufferQueueCallbackHandler(sles_player_sbq_itf);
480 }
481
PlayerSimpleBufferQueueCallbackHandler(SLAndroidSimpleBufferQueueItf sles_player_sbq_itf)482 void OpenSlesOutput::PlayerSimpleBufferQueueCallbackHandler(
483 SLAndroidSimpleBufferQueueItf sles_player_sbq_itf) {
484 if (fifo_->size() <= 0 || number_underruns_ > 0) {
485 ++number_underruns_;
486 event_.SignalEvent(kUnderrun, number_underruns_);
487 return;
488 }
489 int8_t* audio = fifo_->Pop();
490 if (audio)
491 OPENSL_RETURN_ON_FAILURE(
492 (*sles_player_sbq_itf)->Enqueue(sles_player_sbq_itf,
493 audio,
494 buffer_size_bytes_),
495 VOID_RETURN);
496 event_.SignalEvent(kNoUnderrun, 0);
497 }
498
StartCbThreads()499 bool OpenSlesOutput::StartCbThreads() {
500 play_thread_.reset(ThreadWrapper::CreateThread(CbThread,
501 this,
502 kRealtimePriority,
503 "opensl_play_thread"));
504 assert(play_thread_.get());
505 OPENSL_RETURN_ON_FAILURE(
506 (*sles_player_itf_)->SetPlayState(sles_player_itf_,
507 SL_PLAYSTATE_PLAYING),
508 false);
509
510 unsigned int thread_id = 0;
511 if (!play_thread_->Start(thread_id)) {
512 assert(false);
513 return false;
514 }
515 return true;
516 }
517
StopCbThreads()518 void OpenSlesOutput::StopCbThreads() {
519 {
520 CriticalSectionScoped lock(crit_sect_.get());
521 playing_ = false;
522 }
523 if (sles_player_itf_) {
524 OPENSL_RETURN_ON_FAILURE(
525 (*sles_player_itf_)->SetPlayState(sles_player_itf_,
526 SL_PLAYSTATE_STOPPED),
527 VOID_RETURN);
528 }
529 if (play_thread_.get() == NULL) {
530 return;
531 }
532 event_.Stop();
533 if (play_thread_->Stop()) {
534 play_thread_.reset();
535 } else {
536 assert(false);
537 }
538 }
539
CbThread(void * context)540 bool OpenSlesOutput::CbThread(void* context) {
541 return reinterpret_cast<OpenSlesOutput*>(context)->CbThreadImpl();
542 }
543
CbThreadImpl()544 bool OpenSlesOutput::CbThreadImpl() {
545 assert(fine_buffer_.get() != NULL);
546 int event_id;
547 int event_msg;
548 // event_ must not be waited on while a lock has been taken.
549 event_.WaitOnEvent(&event_id, &event_msg);
550
551 CriticalSectionScoped lock(crit_sect_.get());
552 if (HandleUnderrun(event_id, event_msg)) {
553 return playing_;
554 }
555 // if fifo_ is not full it means next item in memory must be free.
556 while (fifo_->size() < num_fifo_buffers_needed_ && playing_) {
557 int8_t* audio = play_buf_[active_queue_].get();
558 fine_buffer_->GetBufferData(audio);
559 fifo_->Push(audio);
560 active_queue_ = (active_queue_ + 1) % TotalBuffersUsed();
561 }
562 return playing_;
563 }
564
565 } // namespace webrtc
566