• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* GStreamer
2  * Copyright (C) 2012 Fluendo S.A. <support@fluendo.com>
3  *
4  * This library is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Library General Public
6  * License as published by the Free Software Foundation; either
7  * version 2 of the License, or (at your option) any later version.
8  *
9  * This library is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Library General Public License for more details.
13  *
14  * You should have received a copy of the GNU Library General Public
15  * License along with this library; if not, write to the
16  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17  * Boston, MA 02110-1301, USA.
18  */
19 
20 #ifdef HAVE_CONFIG_H
21 #  include <config.h>
22 #endif
23 
24 #include <string.h>
25 
26 #include "opensles.h"
27 #include "openslesringbuffer.h"
28 
29 GST_DEBUG_CATEGORY_STATIC (opensles_ringbuffer_debug);
30 #define GST_CAT_DEFAULT opensles_ringbuffer_debug
31 
32 #define _do_init \
33   GST_DEBUG_CATEGORY_INIT (opensles_ringbuffer_debug, \
34       "opensles_ringbuffer", 0, "OpenSL ES ringbuffer");
35 
36 #define parent_class gst_opensles_ringbuffer_parent_class
37 G_DEFINE_TYPE_WITH_CODE (GstOpenSLESRingBuffer, gst_opensles_ringbuffer,
38     GST_TYPE_AUDIO_RING_BUFFER, _do_init);
39 
40 /*
41  * Some generic helper functions
42  */
43 
44 static inline SLuint32
_opensles_sample_rate(guint rate)45 _opensles_sample_rate (guint rate)
46 {
47   switch (rate) {
48     case 8000:
49       return SL_SAMPLINGRATE_8;
50     case 11025:
51       return SL_SAMPLINGRATE_11_025;
52     case 12000:
53       return SL_SAMPLINGRATE_12;
54     case 16000:
55       return SL_SAMPLINGRATE_16;
56     case 22050:
57       return SL_SAMPLINGRATE_22_05;
58     case 24000:
59       return SL_SAMPLINGRATE_24;
60     case 32000:
61       return SL_SAMPLINGRATE_32;
62     case 44100:
63       return SL_SAMPLINGRATE_44_1;
64     case 48000:
65       return SL_SAMPLINGRATE_48;
66     case 64000:
67       return SL_SAMPLINGRATE_64;
68     case 88200:
69       return SL_SAMPLINGRATE_88_2;
70     case 96000:
71       return SL_SAMPLINGRATE_96;
72     case 192000:
73       return SL_SAMPLINGRATE_192;
74     default:
75       return 0;
76   }
77 }
78 
79 static inline SLuint32
_opensles_channel_mask(GstAudioRingBufferSpec * spec)80 _opensles_channel_mask (GstAudioRingBufferSpec * spec)
81 {
82   switch (spec->info.channels) {
83     case 1:
84       return (SL_SPEAKER_FRONT_CENTER);
85     case 2:
86       return (SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT);
87     default:
88       return 0;
89   }
90 }
91 
92 static inline void
_opensles_format(GstAudioRingBufferSpec * spec,SLDataFormat_PCM * format)93 _opensles_format (GstAudioRingBufferSpec * spec, SLDataFormat_PCM * format)
94 {
95   format->formatType = SL_DATAFORMAT_PCM;
96   format->numChannels = spec->info.channels;
97   format->samplesPerSec = _opensles_sample_rate (spec->info.rate);
98   format->bitsPerSample = spec->info.finfo->depth;
99   format->containerSize = spec->info.finfo->width;
100   format->channelMask = _opensles_channel_mask (spec);
101   format->endianness =
102       ((spec->info.finfo->endianness ==
103           G_BIG_ENDIAN) ? SL_BYTEORDER_BIGENDIAN : SL_BYTEORDER_LITTLEENDIAN);
104 }
105 
106 /*
107  * Recorder related functions
108  */
109 
110 static gboolean
_opensles_recorder_acquire(GstAudioRingBuffer * rb,GstAudioRingBufferSpec * spec)111 _opensles_recorder_acquire (GstAudioRingBuffer * rb,
112     GstAudioRingBufferSpec * spec)
113 {
114   GstOpenSLESRingBuffer *thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
115   SLresult result;
116   SLDataFormat_PCM format;
117   SLAndroidConfigurationItf config;
118 
119   /* Configure audio source */
120   SLDataLocator_IODevice loc_dev = {
121     SL_DATALOCATOR_IODEVICE, SL_IODEVICE_AUDIOINPUT,
122     SL_DEFAULTDEVICEID_AUDIOINPUT, NULL
123   };
124   SLDataSource audioSrc = { &loc_dev, NULL };
125 
126   /* Configure audio sink */
127   SLDataLocator_AndroidSimpleBufferQueue loc_bq = {
128     SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 2
129   };
130   SLDataSink audioSink = { &loc_bq, &format };
131 
132   /* Required optional interfaces */
133   const SLInterfaceID ids[2] = { SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
134     SL_IID_ANDROIDCONFIGURATION
135   };
136   const SLboolean req[2] = { SL_BOOLEAN_TRUE, SL_BOOLEAN_FALSE };
137 
138   /* Define the audio format in OpenSL ES terminology */
139   _opensles_format (spec, &format);
140 
141   /* Create the audio recorder object (requires the RECORD_AUDIO permission) */
142   result = (*thiz->engineEngine)->CreateAudioRecorder (thiz->engineEngine,
143       &thiz->recorderObject, &audioSrc, &audioSink, 2, ids, req);
144   if (result != SL_RESULT_SUCCESS) {
145     GST_ERROR_OBJECT (thiz, "engine.CreateAudioRecorder failed(0x%08x)",
146         (guint32) result);
147     goto failed;
148   }
149 
150   /* Set the recording preset if we have one */
151   if (thiz->preset != GST_OPENSLES_RECORDING_PRESET_NONE) {
152     SLint32 preset = gst_to_opensles_recording_preset (thiz->preset);
153 
154     result = (*thiz->recorderObject)->GetInterface (thiz->recorderObject,
155         SL_IID_ANDROIDCONFIGURATION, &config);
156 
157     if (result == SL_RESULT_SUCCESS) {
158       result = (*config)->SetConfiguration (config,
159           SL_ANDROID_KEY_RECORDING_PRESET, &preset, sizeof (preset));
160 
161       if (result != SL_RESULT_SUCCESS) {
162         GST_WARNING_OBJECT (thiz, "Failed to set playback stream type (0x%08x)",
163             (guint32) result);
164       }
165     } else {
166       GST_WARNING_OBJECT (thiz,
167           "Could not get configuration interface 0x%08x", (guint32) result);
168     }
169   }
170 
171   /* Realize the audio recorder object */
172   result =
173       (*thiz->recorderObject)->Realize (thiz->recorderObject, SL_BOOLEAN_FALSE);
174   if (result != SL_RESULT_SUCCESS) {
175     GST_ERROR_OBJECT (thiz, "recorder.Realize failed(0x%08x)",
176         (guint32) result);
177     goto failed;
178   }
179 
180   /* Get the record interface */
181   result = (*thiz->recorderObject)->GetInterface (thiz->recorderObject,
182       SL_IID_RECORD, &thiz->recorderRecord);
183   if (result != SL_RESULT_SUCCESS) {
184     GST_ERROR_OBJECT (thiz, "recorder.GetInterface(Record) failed(0x%08x)",
185         (guint32) result);
186     goto failed;
187   }
188 
189   /* Get the buffer queue interface */
190   result =
191       (*thiz->recorderObject)->GetInterface (thiz->recorderObject,
192       SL_IID_ANDROIDSIMPLEBUFFERQUEUE, &thiz->bufferQueue);
193   if (result != SL_RESULT_SUCCESS) {
194     GST_ERROR_OBJECT (thiz, "recorder.GetInterface(BufferQueue) failed(0x%08x)",
195         (guint32) result);
196     goto failed;
197   }
198 
199   return TRUE;
200 
201 failed:
202   return FALSE;
203 }
204 
205 /* This callback function is executed when the ringbuffer is started to preroll
206  * the output buffer queue with empty buffers, from app thread, and each time
207  * there's a filled buffer, from audio device processing thread,
208  * the callback behaviour.
209  */
210 static void
_opensles_recorder_cb(SLAndroidSimpleBufferQueueItf bufferQueue,void * context)211 _opensles_recorder_cb (SLAndroidSimpleBufferQueueItf bufferQueue, void *context)
212 {
213   GstAudioRingBuffer *rb = GST_AUDIO_RING_BUFFER_CAST (context);
214   GstOpenSLESRingBuffer *thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
215   SLresult result;
216   guint8 *ptr;
217   gint seg;
218   gint len;
219 
220   /* Advance only when we are called by the callback function */
221   if (bufferQueue) {
222     gst_audio_ring_buffer_advance (rb, 1);
223   }
224 
225   /* Get a segment form the GStreamer ringbuffer to write in */
226   if (!gst_audio_ring_buffer_prepare_read (rb, &seg, &ptr, &len)) {
227     GST_WARNING_OBJECT (rb, "No segment available");
228     return;
229   }
230 
231   GST_LOG_OBJECT (thiz, "enqueue: %p size %d segment: %d", ptr, len, seg);
232 
233   /* Enqueue the sefment as buffer to be written */
234   result = (*thiz->bufferQueue)->Enqueue (thiz->bufferQueue, ptr, len);
235   if (result != SL_RESULT_SUCCESS) {
236     GST_ERROR_OBJECT (thiz, "bufferQueue.Enqueue failed(0x%08x)",
237         (guint32) result);
238     return;
239   }
240 }
241 
242 static gboolean
_opensles_recorder_start(GstAudioRingBuffer * rb)243 _opensles_recorder_start (GstAudioRingBuffer * rb)
244 {
245   GstOpenSLESRingBuffer *thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
246   SLresult result;
247 
248   /* Register callback on the buffer queue */
249   if (!thiz->is_queue_callback_registered) {
250     result = (*thiz->bufferQueue)->RegisterCallback (thiz->bufferQueue,
251         _opensles_recorder_cb, rb);
252     if (result != SL_RESULT_SUCCESS) {
253       GST_ERROR_OBJECT (thiz, "bufferQueue.RegisterCallback failed(0x%08x)",
254           (guint32) result);
255       return FALSE;
256     }
257     thiz->is_queue_callback_registered = TRUE;
258   }
259 
260   /* Preroll one buffer */
261   _opensles_recorder_cb (NULL, rb);
262 
263   /* Start recording */
264   result =
265       (*thiz->recorderRecord)->SetRecordState (thiz->recorderRecord,
266       SL_RECORDSTATE_RECORDING);
267   if (result != SL_RESULT_SUCCESS) {
268     GST_ERROR_OBJECT (thiz, "recorder.SetRecordState failed(0x%08x)",
269         (guint32) result);
270     return FALSE;
271   }
272 
273   return TRUE;
274 }
275 
276 static gboolean
_opensles_recorder_stop(GstAudioRingBuffer * rb)277 _opensles_recorder_stop (GstAudioRingBuffer * rb)
278 {
279   GstOpenSLESRingBuffer *thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
280   SLresult result;
281 
282   /* Stop recording */
283   result =
284       (*thiz->recorderRecord)->SetRecordState (thiz->recorderRecord,
285       SL_RECORDSTATE_STOPPED);
286   if (result != SL_RESULT_SUCCESS) {
287     GST_ERROR_OBJECT (thiz, "recorder.SetRecordState failed(0x%08x)",
288         (guint32) result);
289     return FALSE;
290   }
291 
292   /* Unregister callback on the buffer queue */
293   result = (*thiz->bufferQueue)->RegisterCallback (thiz->bufferQueue,
294       NULL, NULL);
295   if (result != SL_RESULT_SUCCESS) {
296     GST_ERROR_OBJECT (thiz, "bufferQueue.RegisterCallback failed(0x%08x)",
297         (guint32) result);
298     return FALSE;
299   }
300   thiz->is_queue_callback_registered = FALSE;
301 
302   /* Reset the queue */
303   result = (*thiz->bufferQueue)->Clear (thiz->bufferQueue);
304   if (result != SL_RESULT_SUCCESS) {
305     GST_ERROR_OBJECT (thiz, "bufferQueue.Clear failed(0x%08x)",
306         (guint32) result);
307     return FALSE;
308   }
309 
310   return TRUE;
311 }
312 
313 /*
314  * Player related functions
315  */
316 
317 static gboolean
_opensles_player_change_volume(GstAudioRingBuffer * rb)318 _opensles_player_change_volume (GstAudioRingBuffer * rb)
319 {
320   GstOpenSLESRingBuffer *thiz;
321   SLresult result;
322 
323   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
324 
325   if (thiz->playerVolume) {
326     gint millibel = (1.0 - thiz->volume) * -5000.0;
327     result =
328         (*thiz->playerVolume)->SetVolumeLevel (thiz->playerVolume, millibel);
329     if (result != SL_RESULT_SUCCESS) {
330       GST_ERROR_OBJECT (thiz, "player.SetVolumeLevel failed(0x%08x)",
331           (guint32) result);
332       return FALSE;
333     }
334     GST_DEBUG_OBJECT (thiz, "changed volume to %d", millibel);
335   }
336 
337   return TRUE;
338 }
339 
340 static gboolean
_opensles_player_change_mute(GstAudioRingBuffer * rb)341 _opensles_player_change_mute (GstAudioRingBuffer * rb)
342 {
343   GstOpenSLESRingBuffer *thiz;
344   SLresult result;
345 
346   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
347 
348   if (thiz->playerVolume) {
349     result = (*thiz->playerVolume)->SetMute (thiz->playerVolume, thiz->mute);
350     if (result != SL_RESULT_SUCCESS) {
351       GST_ERROR_OBJECT (thiz, "player.SetMute failed(0x%08x)",
352           (guint32) result);
353       return FALSE;
354     }
355     GST_DEBUG_OBJECT (thiz, "changed mute to %d", thiz->mute);
356   }
357 
358   return TRUE;
359 }
360 
361 /* This is a callback function invoked by the playback device thread and
362  * it's used to monitor position changes */
363 static void
_opensles_player_event_cb(SLPlayItf caller,void * context,SLuint32 event)364 _opensles_player_event_cb (SLPlayItf caller, void *context, SLuint32 event)
365 {
366   if (event & SL_PLAYEVENT_HEADATNEWPOS) {
367     SLmillisecond position;
368 
369     (*caller)->GetPosition (caller, &position);
370     GST_LOG_OBJECT (context, "at position=%u ms", (guint) position);
371   }
372 }
373 
374 static gboolean
_opensles_player_acquire(GstAudioRingBuffer * rb,GstAudioRingBufferSpec * spec)375 _opensles_player_acquire (GstAudioRingBuffer * rb,
376     GstAudioRingBufferSpec * spec)
377 {
378   GstOpenSLESRingBuffer *thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
379   SLresult result;
380   SLDataFormat_PCM format;
381   SLAndroidConfigurationItf config;
382 
383   /* Configure audio source
384    * 4 buffers is the "typical" size as optimized inside Android's
385    * OpenSL ES, see frameworks/wilhelm/src/itfstruct.h BUFFER_HEADER_TYPICAL
386    *
387    * Also only use half of our segment size to make sure that there's always
388    * some more queued up in our ringbuffer and we don't start to read silence.
389    */
390   SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {
391     SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, MIN (4, MAX (spec->segtotal >> 1,
392             1))
393   };
394   SLDataSource audioSrc = { &loc_bufq, &format };
395 
396   /* Configure audio sink */
397   SLDataLocator_OutputMix loc_outmix = {
398     SL_DATALOCATOR_OUTPUTMIX, thiz->outputMixObject
399   };
400   SLDataSink audioSink = { &loc_outmix, NULL };
401 
402   /* Define the required interfaces */
403   const SLInterfaceID ids[3] = { SL_IID_BUFFERQUEUE, SL_IID_VOLUME,
404     SL_IID_ANDROIDCONFIGURATION
405   };
406   const SLboolean req[3] = { SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE,
407     SL_BOOLEAN_FALSE
408   };
409 
410   /* Define the format in OpenSL ES terminology */
411   _opensles_format (spec, &format);
412 
413   /* Create the player object */
414   result = (*thiz->engineEngine)->CreateAudioPlayer (thiz->engineEngine,
415       &thiz->playerObject, &audioSrc, &audioSink, 3, ids, req);
416   if (result != SL_RESULT_SUCCESS) {
417     GST_ERROR_OBJECT (thiz, "engine.CreateAudioPlayer failed(0x%08x)",
418         (guint32) result);
419     goto failed;
420   }
421 
422   /* Set the stream type if we have one */
423   if (thiz->stream_type != GST_OPENSLES_STREAM_TYPE_NONE) {
424     SLint32 stream_type = gst_to_opensles_stream_type (thiz->stream_type);
425 
426     result = (*thiz->playerObject)->GetInterface (thiz->playerObject,
427         SL_IID_ANDROIDCONFIGURATION, &config);
428 
429     if (result == SL_RESULT_SUCCESS) {
430       result = (*config)->SetConfiguration (config,
431           SL_ANDROID_KEY_STREAM_TYPE, &stream_type, sizeof (stream_type));
432 
433       if (result != SL_RESULT_SUCCESS) {
434         GST_WARNING_OBJECT (thiz, "Failed to set playback stream type (0x%08x)",
435             (guint32) result);
436       }
437     } else {
438       GST_WARNING_OBJECT (thiz,
439           "Could not get configuration interface 0x%08x", (guint32) result);
440     }
441   }
442 
443   /* Realize the player object */
444   result =
445       (*thiz->playerObject)->Realize (thiz->playerObject, SL_BOOLEAN_FALSE);
446   if (result != SL_RESULT_SUCCESS) {
447     GST_ERROR_OBJECT (thiz, "player.Realize failed(0x%08x)", (guint32) result);
448     goto failed;
449   }
450 
451   /* Get the play interface */
452   result = (*thiz->playerObject)->GetInterface (thiz->playerObject,
453       SL_IID_PLAY, &thiz->playerPlay);
454   if (result != SL_RESULT_SUCCESS) {
455     GST_ERROR_OBJECT (thiz, "player.GetInterface(Play) failed(0x%08x)",
456         (guint32) result);
457     goto failed;
458   }
459 
460   /* Get the buffer queue interface */
461   result = (*thiz->playerObject)->GetInterface (thiz->playerObject,
462       SL_IID_BUFFERQUEUE, &thiz->bufferQueue);
463   if (result != SL_RESULT_SUCCESS) {
464     GST_ERROR_OBJECT (thiz, "player.GetInterface(BufferQueue) failed(0x%08x)",
465         (guint32) result);
466     goto failed;
467   }
468 
469   /* Get the volume interface */
470   result = (*thiz->playerObject)->GetInterface (thiz->playerObject,
471       SL_IID_VOLUME, &thiz->playerVolume);
472   if (result != SL_RESULT_SUCCESS) {
473     GST_ERROR_OBJECT (thiz, "player.GetInterface(Volume) failed(0x%08x)",
474         (guint32) result);
475     goto failed;
476   }
477 
478   /* Request position update events at each 20 ms */
479   result = (*thiz->playerPlay)->SetPositionUpdatePeriod (thiz->playerPlay, 20);
480   if (result != SL_RESULT_SUCCESS) {
481     GST_ERROR_OBJECT (thiz, "player.SetPositionUpdatePeriod failed(0x%08x)",
482         (guint32) result);
483     goto failed;
484   }
485 
486   /* Define the event mask to be monitorized */
487   result = (*thiz->playerPlay)->SetCallbackEventsMask (thiz->playerPlay,
488       SL_PLAYEVENT_HEADATNEWPOS);
489   if (result != SL_RESULT_SUCCESS) {
490     GST_ERROR_OBJECT (thiz, "player.SetCallbackEventsMask failed(0x%08x)",
491         (guint32) result);
492     goto failed;
493   }
494 
495   /* Register a callback to process the events */
496   result = (*thiz->playerPlay)->RegisterCallback (thiz->playerPlay,
497       _opensles_player_event_cb, thiz);
498   if (result != SL_RESULT_SUCCESS) {
499     GST_ERROR_OBJECT (thiz, "player.RegisterCallback(event_cb) failed(0x%08x)",
500         (guint32) result);
501     goto failed;
502   }
503 
504   /* Configure the volume and mute state */
505   _opensles_player_change_volume (rb);
506   _opensles_player_change_mute (rb);
507 
508   /* Allocate the queue associated ringbuffer memory */
509   thiz->data_segtotal = loc_bufq.numBuffers;
510   thiz->data_size = spec->segsize * thiz->data_segtotal;
511   thiz->data = g_malloc0 (thiz->data_size);
512   g_atomic_int_set (&thiz->segqueued, 0);
513   g_atomic_int_set (&thiz->is_prerolled, 0);
514   thiz->cursor = 0;
515 
516   return TRUE;
517 
518 failed:
519   return FALSE;
520 }
521 
522 /* This callback function is executed when the ringbuffer is started to preroll
523  * the input buffer queue with few buffers, from app thread, and each time
524  * that rendering of one buffer finishes, from audio device processing thread,
525  * the callback behaviour.
526  *
527  * We wrap the queue behaviour with an appropriate chunk of memory (queue len *
528  * ringbuffer segment size) which is used to hold the audio data while it's
529  * being processed in the queue. The memory region is used whit a ringbuffer
530  * behaviour.
531  */
532 static void
_opensles_player_cb(SLAndroidSimpleBufferQueueItf bufferQueue,void * context)533 _opensles_player_cb (SLAndroidSimpleBufferQueueItf bufferQueue, void *context)
534 {
535   GstAudioRingBuffer *rb = GST_AUDIO_RING_BUFFER_CAST (context);
536   GstOpenSLESRingBuffer *thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
537   SLresult result;
538   guint8 *ptr, *cur;
539   gint seg;
540   gint len;
541 
542   /* Get a segment form the GStreamer ringbuffer to read some samples */
543   if (!gst_audio_ring_buffer_prepare_read (rb, &seg, &ptr, &len)) {
544     GST_WARNING_OBJECT (rb, "No segment available");
545     return;
546   }
547 
548   /* copy the segment data to our queue associated ringbuffer memory */
549   cur = thiz->data + (thiz->cursor * rb->spec.segsize);
550   memcpy (cur, ptr, len);
551   g_atomic_int_inc (&thiz->segqueued);
552 
553   GST_LOG_OBJECT (thiz, "enqueue: %p size %d segment: %d in queue[%d]",
554       cur, len, seg, thiz->cursor);
555   /* advance the cursor in our queue associated ringbuffer */
556   thiz->cursor = (thiz->cursor + 1) % thiz->data_segtotal;
557 
558   /* Enqueue the buffer to be rendered */
559   result = (*thiz->bufferQueue)->Enqueue (thiz->bufferQueue, cur, len);
560   if (result != SL_RESULT_SUCCESS) {
561     GST_ERROR_OBJECT (thiz, "bufferQueue.Enqueue failed(0x%08x)",
562         (guint32) result);
563     return;
564   }
565 
566   /* Fill with silence samples the segment of the GStreamer ringbuffer */
567   gst_audio_ring_buffer_clear (rb, seg);
568   /* Make the segment reusable */
569   gst_audio_ring_buffer_advance (rb, 1);
570 }
571 
572 static gboolean
_opensles_player_start(GstAudioRingBuffer * rb)573 _opensles_player_start (GstAudioRingBuffer * rb)
574 {
575   GstOpenSLESRingBuffer *thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
576   SLresult result;
577 
578   /* Register callback on the buffer queue */
579   if (!thiz->is_queue_callback_registered) {
580     result = (*thiz->bufferQueue)->RegisterCallback (thiz->bufferQueue,
581         _opensles_player_cb, rb);
582     if (result != SL_RESULT_SUCCESS) {
583       GST_ERROR_OBJECT (thiz, "bufferQueue.RegisterCallback failed(0x%08x)",
584           (guint32) result);
585       return FALSE;
586     }
587     thiz->is_queue_callback_registered = TRUE;
588   }
589 
590   /* Fill the queue by enqueing a buffer */
591   if (!g_atomic_int_get (&thiz->is_prerolled)) {
592     _opensles_player_cb (NULL, rb);
593     g_atomic_int_set (&thiz->is_prerolled, 1);
594   }
595 
596   /* Change player state into PLAYING */
597   result =
598       (*thiz->playerPlay)->SetPlayState (thiz->playerPlay,
599       SL_PLAYSTATE_PLAYING);
600   if (result != SL_RESULT_SUCCESS) {
601     GST_ERROR_OBJECT (thiz, "player.SetPlayState failed(0x%08x)",
602         (guint32) result);
603     return FALSE;
604   }
605 
606   return TRUE;
607 }
608 
609 static gboolean
_opensles_player_pause(GstAudioRingBuffer * rb)610 _opensles_player_pause (GstAudioRingBuffer * rb)
611 {
612   GstOpenSLESRingBuffer *thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
613   SLresult result;
614 
615   result =
616       (*thiz->playerPlay)->SetPlayState (thiz->playerPlay, SL_PLAYSTATE_PAUSED);
617   if (result != SL_RESULT_SUCCESS) {
618     GST_ERROR_OBJECT (thiz, "player.SetPlayState failed(0x%08x)",
619         (guint32) result);
620     return FALSE;
621   }
622 
623   return TRUE;
624 }
625 
626 static gboolean
_opensles_player_stop(GstAudioRingBuffer * rb)627 _opensles_player_stop (GstAudioRingBuffer * rb)
628 {
629   GstOpenSLESRingBuffer *thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
630   SLresult result;
631 
632   /* Change player state into STOPPED */
633   result =
634       (*thiz->playerPlay)->SetPlayState (thiz->playerPlay,
635       SL_PLAYSTATE_STOPPED);
636   if (result != SL_RESULT_SUCCESS) {
637     GST_ERROR_OBJECT (thiz, "player.SetPlayState failed(0x%08x)",
638         (guint32) result);
639     return FALSE;
640   }
641 
642   /* Unregister callback on the buffer queue */
643   result = (*thiz->bufferQueue)->RegisterCallback (thiz->bufferQueue,
644       NULL, NULL);
645   if (result != SL_RESULT_SUCCESS) {
646     GST_ERROR_OBJECT (thiz, "bufferQueue.RegisterCallback failed(0x%08x)",
647         (guint32) result);
648     return FALSE;
649   }
650   thiz->is_queue_callback_registered = FALSE;
651 
652   /* Reset the queue */
653   result = (*thiz->bufferQueue)->Clear (thiz->bufferQueue);
654   if (result != SL_RESULT_SUCCESS) {
655     GST_ERROR_OBJECT (thiz, "bufferQueue.Clear failed(0x%08x)",
656         (guint32) result);
657     return FALSE;
658   }
659 
660   /* Reset our state */
661   g_atomic_int_set (&thiz->segqueued, 0);
662   thiz->cursor = 0;
663 
664   return TRUE;
665 }
666 
667 /*
668  * OpenSL ES ringbuffer wrapper
669  */
670 
671 GstAudioRingBuffer *
gst_opensles_ringbuffer_new(RingBufferMode mode)672 gst_opensles_ringbuffer_new (RingBufferMode mode)
673 {
674   GstOpenSLESRingBuffer *thiz;
675 
676   g_return_val_if_fail (mode > RB_MODE_NONE && mode < RB_MODE_LAST, NULL);
677 
678   thiz = g_object_new (GST_TYPE_OPENSLES_RING_BUFFER, NULL);
679 
680   if (thiz) {
681     thiz->mode = mode;
682     if (mode == RB_MODE_SRC) {
683       thiz->acquire = _opensles_recorder_acquire;
684       thiz->start = _opensles_recorder_start;
685       thiz->pause = _opensles_recorder_stop;
686       thiz->stop = _opensles_recorder_stop;
687       thiz->change_volume = NULL;
688     } else if (mode == RB_MODE_SINK_PCM) {
689       thiz->acquire = _opensles_player_acquire;
690       thiz->start = _opensles_player_start;
691       thiz->pause = _opensles_player_pause;
692       thiz->stop = _opensles_player_stop;
693       thiz->change_volume = _opensles_player_change_volume;
694     }
695   }
696 
697   GST_DEBUG_OBJECT (thiz, "ringbuffer created");
698 
699   return GST_AUDIO_RING_BUFFER (thiz);
700 }
701 
702 void
gst_opensles_ringbuffer_set_volume(GstAudioRingBuffer * rb,gfloat volume)703 gst_opensles_ringbuffer_set_volume (GstAudioRingBuffer * rb, gfloat volume)
704 {
705   GstOpenSLESRingBuffer *thiz;
706 
707   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
708 
709   thiz->volume = volume;
710 
711   if (thiz->change_volume) {
712     thiz->change_volume (rb);
713   }
714 }
715 
716 void
gst_opensles_ringbuffer_set_mute(GstAudioRingBuffer * rb,gboolean mute)717 gst_opensles_ringbuffer_set_mute (GstAudioRingBuffer * rb, gboolean mute)
718 {
719   GstOpenSLESRingBuffer *thiz;
720 
721   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
722 
723   thiz->mute = mute;
724 
725   if (thiz->change_mute) {
726     thiz->change_mute (rb);
727   }
728 }
729 
730 static gboolean
gst_opensles_ringbuffer_open_device(GstAudioRingBuffer * rb)731 gst_opensles_ringbuffer_open_device (GstAudioRingBuffer * rb)
732 {
733   GstOpenSLESRingBuffer *thiz;
734   SLresult result;
735 
736   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
737 
738   /* Create and realize the engine object */
739   thiz->engineObject = gst_opensles_get_engine ();
740   if (!thiz->engineObject) {
741     GST_ERROR_OBJECT (thiz, "Failed to get engine object");
742     goto failed;
743   }
744 
745   /* Get the engine interface, which is needed in order to create other objects */
746   result = (*thiz->engineObject)->GetInterface (thiz->engineObject,
747       SL_IID_ENGINE, &thiz->engineEngine);
748   if (result != SL_RESULT_SUCCESS) {
749     GST_ERROR_OBJECT (thiz, "engine.GetInterface(Engine) failed(0x%08x)",
750         (guint32) result);
751     goto failed;
752   }
753 
754   if (thiz->mode == RB_MODE_SINK_PCM) {
755     SLOutputMixItf outputMix;
756 
757     /* Create an output mixer object */
758     result = (*thiz->engineEngine)->CreateOutputMix (thiz->engineEngine,
759         &thiz->outputMixObject, 0, NULL, NULL);
760     if (result != SL_RESULT_SUCCESS) {
761       GST_ERROR_OBJECT (thiz, "engine.CreateOutputMix failed(0x%08x)",
762           (guint32) result);
763       goto failed;
764     }
765 
766     /* Realize the output mixer object */
767     result = (*thiz->outputMixObject)->Realize (thiz->outputMixObject,
768         SL_BOOLEAN_FALSE);
769     if (result != SL_RESULT_SUCCESS) {
770       GST_ERROR_OBJECT (thiz, "outputMix.Realize failed(0x%08x)",
771           (guint32) result);
772       goto failed;
773     }
774 
775     /* Get the mixer interface */
776     result = (*thiz->outputMixObject)->GetInterface (thiz->outputMixObject,
777         SL_IID_OUTPUTMIX, &outputMix);
778     if (result != SL_RESULT_SUCCESS) {
779       GST_WARNING_OBJECT (thiz, "outputMix.GetInterface failed(0x%08x)",
780           (guint32) result);
781     } else {
782       SLint32 numDevices = MAX_NUMBER_OUTPUT_DEVICES;
783       SLuint32 deviceIDs[MAX_NUMBER_OUTPUT_DEVICES];
784       gint i;
785 
786       /* Query the list of output devices */
787       (*outputMix)->GetDestinationOutputDeviceIDs (outputMix, &numDevices,
788           deviceIDs);
789       GST_DEBUG_OBJECT (thiz, "Found %d output devices", (gint) numDevices);
790       for (i = 0; i < numDevices; i++) {
791         GST_DEBUG_OBJECT (thiz, "  DeviceID: %08x", (guint) deviceIDs[i]);
792       }
793     }
794   }
795 
796   GST_DEBUG_OBJECT (thiz, "device opened");
797   return TRUE;
798 
799 failed:
800   return FALSE;
801 }
802 
803 static gboolean
gst_opensles_ringbuffer_close_device(GstAudioRingBuffer * rb)804 gst_opensles_ringbuffer_close_device (GstAudioRingBuffer * rb)
805 {
806   GstOpenSLESRingBuffer *thiz;
807 
808   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
809 
810   /* Destroy the output mix object */
811   if (thiz->outputMixObject) {
812     (*thiz->outputMixObject)->Destroy (thiz->outputMixObject);
813     thiz->outputMixObject = NULL;
814   }
815 
816   /* Destroy the engine object and invalidate all associated interfaces */
817   if (thiz->engineObject) {
818     gst_opensles_release_engine (thiz->engineObject);
819     thiz->engineObject = NULL;
820     thiz->engineEngine = NULL;
821   }
822 
823   thiz->bufferQueue = NULL;
824 
825   GST_DEBUG_OBJECT (thiz, "device closed");
826   return TRUE;
827 }
828 
829 static gboolean
gst_opensles_ringbuffer_acquire(GstAudioRingBuffer * rb,GstAudioRingBufferSpec * spec)830 gst_opensles_ringbuffer_acquire (GstAudioRingBuffer * rb,
831     GstAudioRingBufferSpec * spec)
832 {
833   GstOpenSLESRingBuffer *thiz;
834 
835   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
836 
837   /* Instantiate and configure the OpenSL ES interfaces */
838   if (!thiz->acquire (rb, spec)) {
839     return FALSE;
840   }
841 
842   /* Initialize our ringbuffer memory region */
843   rb->size = spec->segtotal * spec->segsize;
844   rb->memory = g_malloc0 (rb->size);
845 
846   GST_DEBUG_OBJECT (thiz, "ringbuffer acquired");
847   return TRUE;
848 }
849 
850 static gboolean
gst_opensles_ringbuffer_release(GstAudioRingBuffer * rb)851 gst_opensles_ringbuffer_release (GstAudioRingBuffer * rb)
852 {
853   GstOpenSLESRingBuffer *thiz;
854 
855   thiz = GST_OPENSLES_RING_BUFFER (rb);
856 
857   /* XXX: We need to sleep a bit before destroying the player object
858    * because of a bug in Android in versions < 4.2.
859    *
860    * OpenSLES is using AudioTrack for rendering the sound. AudioTrack
861    * has a thread that pulls raw audio from the buffer queue and then
862    * passes it forward to AudioFlinger (AudioTrack::processAudioBuffer()).
863    * This thread is calling various callbacks on events, e.g. when
864    * an underrun happens or to request data. OpenSLES sets this callback
865    * on AudioTrack (audioTrack_callBack_pullFromBuffQueue() from
866    * android_AudioPlayer.cpp). Among other things this is taking a lock
867    * on the player interface.
868    *
869    * Now if we destroy the player interface object, it will first of all
870    * take the player interface lock (IObject_Destroy()). Then it destroys
871    * the audio player instance (android_audioPlayer_destroy()) which then
872    * calls stop() on the AudioTrack and deletes it. Now the destructor of
873    * AudioTrack will wait until the rendering thread (AudioTrack::processAudioBuffer())
874    * has finished.
875    *
876    * If all this happens with bad timing it can happen that the rendering
877    * thread is currently e.g. handling underrun but did not lock the player
878    * interface object yet. Then destroying happens and takes the lock and waits
879    * for the thread to finish. Then the thread tries to take the lock and waits
880    * forever.
881    *
882    * We wait a bit before destroying the player object to make sure that
883    * the rendering thread finished whatever it was doing, and then stops
884    * (note: we called gst_opensles_ringbuffer_stop() before this already).
885    */
886 
887   /* Destroy audio player object, and invalidate all associated interfaces */
888   if (thiz->playerObject) {
889     g_usleep (50000);
890     (*thiz->playerObject)->Destroy (thiz->playerObject);
891     thiz->playerObject = NULL;
892     thiz->playerPlay = NULL;
893     thiz->playerVolume = NULL;
894   }
895 
896   /* Destroy audio recorder object, and invalidate all associated interfaces */
897   if (thiz->recorderObject) {
898     g_usleep (50000);
899     (*thiz->recorderObject)->Destroy (thiz->recorderObject);
900     thiz->recorderObject = NULL;
901     thiz->recorderRecord = NULL;
902   }
903 
904   if (thiz->data) {
905     g_free (thiz->data);
906     thiz->data = NULL;
907   }
908 
909   if (rb->memory) {
910     g_free (rb->memory);
911     rb->memory = NULL;
912     rb->size = 0;
913   }
914 
915   GST_DEBUG_OBJECT (thiz, "ringbuffer released");
916   return TRUE;
917 }
918 
919 static gboolean
gst_opensles_ringbuffer_start(GstAudioRingBuffer * rb)920 gst_opensles_ringbuffer_start (GstAudioRingBuffer * rb)
921 {
922   GstOpenSLESRingBuffer *thiz;
923   gboolean res;
924 
925   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
926   res = thiz->start (rb);
927 
928   GST_DEBUG_OBJECT (thiz, "ringbuffer %s started", (res ? "" : "not"));
929   return res;
930 }
931 
932 static gboolean
gst_opensles_ringbuffer_pause(GstAudioRingBuffer * rb)933 gst_opensles_ringbuffer_pause (GstAudioRingBuffer * rb)
934 {
935   GstOpenSLESRingBuffer *thiz;
936   gboolean res;
937 
938   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
939   res = thiz->pause (rb);
940 
941   GST_DEBUG_OBJECT (thiz, "ringbuffer %s paused", (res ? "" : "not"));
942   return res;
943 }
944 
945 static gboolean
gst_opensles_ringbuffer_stop(GstAudioRingBuffer * rb)946 gst_opensles_ringbuffer_stop (GstAudioRingBuffer * rb)
947 {
948   GstOpenSLESRingBuffer *thiz;
949   gboolean res;
950 
951   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
952   res = thiz->stop (rb);
953 
954   GST_DEBUG_OBJECT (thiz, "ringbuffer %s stopped", (res ? " " : "not"));
955   return res;
956 }
957 
958 static guint
gst_opensles_ringbuffer_delay(GstAudioRingBuffer * rb)959 gst_opensles_ringbuffer_delay (GstAudioRingBuffer * rb)
960 {
961   GstOpenSLESRingBuffer *thiz;
962   guint res = 0;
963 
964   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
965 
966   if (thiz->playerPlay) {
967     SLuint32 state;
968     SLmillisecond position;
969     guint64 playedpos = 0, queuedpos = 0;
970     (*thiz->playerPlay)->GetPlayState (thiz->playerPlay, &state);
971     if (state == SL_PLAYSTATE_PLAYING) {
972       (*thiz->playerPlay)->GetPosition (thiz->playerPlay, &position);
973       playedpos =
974           gst_util_uint64_scale_round (position, rb->spec.info.rate, 1000);
975       queuedpos = g_atomic_int_get (&thiz->segqueued) * rb->samples_per_seg;
976       if (queuedpos < playedpos) {
977         res = 0;
978         GST_ERROR_OBJECT (thiz,
979             "Queued position smaller than playback position (%" G_GUINT64_FORMAT
980             " < %" G_GUINT64_FORMAT ")", queuedpos, playedpos);
981       } else {
982         res = queuedpos - playedpos;
983       }
984     }
985 
986     GST_LOG_OBJECT (thiz, "queued samples %" G_GUINT64_FORMAT " position %u ms "
987         "(%" G_GUINT64_FORMAT " samples) delay %u samples",
988         queuedpos, (guint) position, playedpos, res);
989   }
990 
991   return res;
992 }
993 
994 static void
gst_opensles_ringbuffer_clear_all(GstAudioRingBuffer * rb)995 gst_opensles_ringbuffer_clear_all (GstAudioRingBuffer * rb)
996 {
997   GstOpenSLESRingBuffer *thiz;
998 
999   thiz = GST_OPENSLES_RING_BUFFER_CAST (rb);
1000 
1001   if (thiz->data) {
1002     SLresult result;
1003 
1004     memset (thiz->data, 0, thiz->data_size);
1005     g_atomic_int_set (&thiz->segqueued, 0);
1006     thiz->cursor = 0;
1007     /* Reset the queue */
1008     result = (*thiz->bufferQueue)->Clear (thiz->bufferQueue);
1009     if (result != SL_RESULT_SUCCESS) {
1010       GST_WARNING_OBJECT (thiz, "bufferQueue.Clear failed(0x%08x)",
1011           (guint32) result);
1012     }
1013     g_atomic_int_set (&thiz->is_prerolled, 0);
1014   }
1015 
1016   GST_CALL_PARENT (GST_AUDIO_RING_BUFFER_CLASS, clear_all, (rb));
1017 }
1018 
1019 static void
gst_opensles_ringbuffer_dispose(GObject * object)1020 gst_opensles_ringbuffer_dispose (GObject * object)
1021 {
1022   G_OBJECT_CLASS (parent_class)->dispose (object);
1023 }
1024 
1025 static void
gst_opensles_ringbuffer_finalize(GObject * object)1026 gst_opensles_ringbuffer_finalize (GObject * object)
1027 {
1028   G_OBJECT_CLASS (parent_class)->finalize (object);
1029 }
1030 
1031 static void
gst_opensles_ringbuffer_class_init(GstOpenSLESRingBufferClass * klass)1032 gst_opensles_ringbuffer_class_init (GstOpenSLESRingBufferClass * klass)
1033 {
1034   GObjectClass *gobject_class;
1035   GstAudioRingBufferClass *gstringbuffer_class;
1036 
1037   gobject_class = (GObjectClass *) klass;
1038   gstringbuffer_class = (GstAudioRingBufferClass *) klass;
1039 
1040   gobject_class->dispose = gst_opensles_ringbuffer_dispose;
1041   gobject_class->finalize = gst_opensles_ringbuffer_finalize;
1042 
1043   gstringbuffer_class->open_device =
1044       GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_open_device);
1045   gstringbuffer_class->close_device =
1046       GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_close_device);
1047   gstringbuffer_class->acquire =
1048       GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_acquire);
1049   gstringbuffer_class->release =
1050       GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_release);
1051   gstringbuffer_class->start =
1052       GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_start);
1053   gstringbuffer_class->pause =
1054       GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_pause);
1055   gstringbuffer_class->resume =
1056       GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_start);
1057   gstringbuffer_class->stop = GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_stop);
1058   gstringbuffer_class->delay =
1059       GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_delay);
1060   gstringbuffer_class->clear_all =
1061       GST_DEBUG_FUNCPTR (gst_opensles_ringbuffer_clear_all);
1062 }
1063 
1064 static void
gst_opensles_ringbuffer_init(GstOpenSLESRingBuffer * thiz)1065 gst_opensles_ringbuffer_init (GstOpenSLESRingBuffer * thiz)
1066 {
1067   thiz->mode = RB_MODE_NONE;
1068   thiz->engineObject = NULL;
1069   thiz->outputMixObject = NULL;
1070   thiz->playerObject = NULL;
1071   thiz->recorderObject = NULL;
1072   thiz->is_queue_callback_registered = FALSE;
1073 }
1074