• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2019 Collabora Ltd.
3  *   Author: Xavier Claessens <xavier.claessens@collabora.com>
4  *
5  * This library is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU Lesser General Public
7  * License as published by the Free Software Foundation
8  * version 2.1 of the License.
9  *
10  * This library is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  * Lesser General Public License for more details.
14  *
15  * You should have received a copy of the GNU Lesser General Public
16  * License along with this library; if not, write to the Free Software
17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
18  *
19  */
20 
21 /**
22  * SECTION:mlaudiosink
23  * @short_description: Audio sink for Magic Leap platform
24  * @see_also: #GstAudioSink
25  *
26  * An audio sink element for LuminOS, the Magic Leap platform. There are 2 modes
27  * supported: normal and spatial. By default the audio is output directly to the
28  * stereo speakers, but in spatial mode the audio will be localised in the 3D
29  * environment. The user ears the sound as coming from a point in space, from a
30  * given distance and direction.
31  *
32  * To enable the spatial mode, the application needs to set a sync bus
33  * handler, using gst_bus_set_sync_handler(), to catch messages of type
34  * %GST_MESSAGE_ELEMENT named "gst.mlaudiosink.need-app" and
35  * "gst.mlaudiosink.need-audio-node". The need-app message will be posted first,
36  * application must then set the #GstMLAudioSink::app property with the pointer
37  * to application's lumin::BaseApp C++ object. That property can also be set on
38  * element creation in which case the need-app message won't be posted. After
39  * that, and if #GstMLAudioSink::app has been set, the need-audio-node message
40  * is posted from lumin::BaseApp's main thread. The application must then create
41  * a lumin::AudioNode C++ object, using lumin::Prism::createAudioNode(), and set
42  * the #GstMLAudioSink::audio-node property. Note that it is important that the
43  * lumin::AudioNode object must be created from within that message handler,
44  * and in the caller's thread, this is a limitation/bug of the platform
45  * (atleast until version 0.97).
46  *
47  * Here is an example of bus message handler to enable spatial sound:
48  * ```C
49  * static GstBusSyncReply
50  * bus_sync_handler_cb (GstBus * bus, GstMessage * msg, gpointer user_data)
51  * {
52  *   MyApplication * self = user_data;
53  *
54  *   if (GST_MESSAGE_TYPE (msg) == GST_MESSAGE_ELEMENT) {
55  *     if (gst_message_has_name (msg, "gst.mlaudiosink.need-app")) {
56  *       g_object_set (G_OBJECT (msg->src), "app", &self->app, NULL);
57  *     } else if (gst_message_has_name (msg, "gst.mlaudiosink.need-audio-node")) {
58  *       self->audio_node = self->prism->createAudioNode ();
59  *       self->audio_node->setSpatialSoundEnable (true);
60  *       self->ui_node->addChild(self->audio_node);
61  *       g_object_set (G_OBJECT (msg->src), "audio-node", self->audio_node, NULL);
62  *     }
63  *   }
64  *   return GST_BUS_PASS;
65  * }
66  * ```
67  *
68  * Since: 1.18
69  */
70 
71 #ifdef HAVE_CONFIG_H
72 #include "config.h"
73 #endif
74 
75 #include "mlaudiosink.h"
76 #include "mlaudiowrapper.h"
77 
78 GST_DEBUG_CATEGORY_EXTERN (mgl_debug);
79 #define GST_CAT_DEFAULT mgl_debug
80 
81 static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
82     GST_PAD_SINK,
83     GST_PAD_ALWAYS,
84     GST_STATIC_CAPS ("audio/x-raw, "
85         "format = (string) { S16LE }, "
86         "channels = (int) [ 1, 2 ], "
87         "rate = (int) [ 16000, 48000 ], " "layout = (string) interleaved"));
88 
89 /* HACK: After calling MLAudioStopSound() there is no way to know when it will
90  * actually stop calling buffer_cb(). If the sink is disposed first, it would
91  * crash. Keep here a set of active sinks. */
92 static GHashTable *active_sinks;
93 static GMutex active_sinks_mutex;
94 
95 struct _GstMLAudioSink
96 {
97   GstAudioSink parent;
98 
99   gpointer audio_node;
100   gpointer app;
101 
102   GstMLAudioWrapper *wrapper;
103   MLAudioBufferFormat format;
104   uint32_t recommended_buffer_size;
105   MLAudioBuffer buffer;
106   guint buffer_offset;
107   gboolean has_buffer;
108   gboolean paused;
109   gboolean stopped;
110 
111   GMutex mutex;
112   GCond cond;
113 };
114 
115 G_DEFINE_TYPE (GstMLAudioSink, gst_ml_audio_sink, GST_TYPE_AUDIO_SINK);
116 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (mlaudiosink, "mlaudiosink",
117     GST_RANK_PRIMARY + 10, GST_TYPE_ML_AUDIO_SINK,
118     GST_DEBUG_CATEGORY_INIT (mgl_debug, "magicleap", 0, "Magic Leap elements"));
119 
120 enum
121 {
122   PROP_0,
123   PROP_AUDIO_NODE,
124   PROP_APP,
125 };
126 
127 static void
gst_ml_audio_sink_init(GstMLAudioSink * self)128 gst_ml_audio_sink_init (GstMLAudioSink * self)
129 {
130   g_mutex_init (&self->mutex);
131   g_cond_init (&self->cond);
132 }
133 
134 static void
gst_ml_audio_sink_dispose(GObject * object)135 gst_ml_audio_sink_dispose (GObject * object)
136 {
137   GstMLAudioSink *self = GST_ML_AUDIO_SINK (object);
138 
139   g_mutex_clear (&self->mutex);
140   g_cond_clear (&self->cond);
141 
142   G_OBJECT_CLASS (gst_ml_audio_sink_parent_class)->dispose (object);
143 }
144 
145 static void
gst_ml_audio_sink_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)146 gst_ml_audio_sink_set_property (GObject * object, guint prop_id,
147     const GValue * value, GParamSpec * pspec)
148 {
149   GstMLAudioSink *self = GST_ML_AUDIO_SINK (object);
150 
151   switch (prop_id) {
152     case PROP_AUDIO_NODE:
153       self->audio_node = g_value_get_pointer (value);
154       break;
155     case PROP_APP:
156       self->app = g_value_get_pointer (value);
157       break;
158     default:
159       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
160       break;
161   }
162 }
163 
164 static void
gst_ml_audio_sink_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)165 gst_ml_audio_sink_get_property (GObject * object, guint prop_id, GValue * value,
166     GParamSpec * pspec)
167 {
168   GstMLAudioSink *self = GST_ML_AUDIO_SINK (object);
169 
170   switch (prop_id) {
171     case PROP_AUDIO_NODE:
172       g_value_set_pointer (value, self->audio_node);
173       break;
174     case PROP_APP:
175       g_value_set_pointer (value, self->app);
176       break;
177     default:
178       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
179       break;
180   }
181 }
182 
183 static GstCaps *
gst_ml_audio_sink_getcaps(GstBaseSink * bsink,GstCaps * filter)184 gst_ml_audio_sink_getcaps (GstBaseSink * bsink, GstCaps * filter)
185 {
186   GstCaps *caps;
187 
188   caps = gst_static_caps_get (&sink_template.static_caps);
189 
190   if (filter) {
191     gst_caps_replace (&caps,
192         gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST));
193   }
194 
195   return caps;
196 }
197 
198 static gboolean
gst_ml_audio_sink_open(GstAudioSink * sink)199 gst_ml_audio_sink_open (GstAudioSink * sink)
200 {
201   /* Nothing to do in open/close */
202   return TRUE;
203 }
204 
205 static void
buffer_cb(MLHandle handle,gpointer user_data)206 buffer_cb (MLHandle handle, gpointer user_data)
207 {
208   GstMLAudioSink *self = user_data;
209 
210   g_mutex_lock (&active_sinks_mutex);
211   if (!g_hash_table_contains (active_sinks, self))
212     goto out;
213 
214   gst_ml_audio_wrapper_set_handle (self->wrapper, handle);
215 
216   g_mutex_lock (&self->mutex);
217   g_cond_signal (&self->cond);
218   g_mutex_unlock (&self->mutex);
219 
220 out:
221   g_mutex_unlock (&active_sinks_mutex);
222 }
223 
224 /* Must be called with self->mutex locked */
225 static gboolean
wait_for_buffer(GstMLAudioSink * self)226 wait_for_buffer (GstMLAudioSink * self)
227 {
228   gboolean ret = TRUE;
229 
230   while (!self->has_buffer && !self->stopped) {
231     MLResult result;
232 
233     result = gst_ml_audio_wrapper_get_buffer (self->wrapper, &self->buffer);
234     if (result == MLResult_Ok) {
235       self->has_buffer = TRUE;
236       self->buffer_offset = 0;
237     } else if (result == MLAudioResult_BufferNotReady) {
238       g_cond_wait (&self->cond, &self->mutex);
239     } else {
240       GST_ERROR_OBJECT (self, "Failed to get output buffer: %d", result);
241       ret = FALSE;
242       break;
243     }
244   }
245 
246   return ret;
247 }
248 
249 static gboolean
create_sound_cb(GstMLAudioWrapper * wrapper,gpointer user_data)250 create_sound_cb (GstMLAudioWrapper * wrapper, gpointer user_data)
251 {
252   GstMLAudioSink *self = user_data;
253   MLResult result;
254 
255   if (self->app) {
256     gst_element_post_message (GST_ELEMENT (self),
257         gst_message_new_element (GST_OBJECT (self),
258             gst_structure_new_empty ("gst.mlaudiosink.need-audio-node")));
259   }
260 
261   gst_ml_audio_wrapper_set_node (self->wrapper, self->audio_node);
262 
263   result = gst_ml_audio_wrapper_create_sound (self->wrapper, &self->format,
264       self->recommended_buffer_size, buffer_cb, self);
265   if (result != MLResult_Ok) {
266     GST_ERROR_OBJECT (self, "Failed to create output stream: %d", result);
267     return FALSE;
268   }
269 
270   return TRUE;
271 }
272 
273 static gboolean
gst_ml_audio_sink_prepare(GstAudioSink * sink,GstAudioRingBufferSpec * spec)274 gst_ml_audio_sink_prepare (GstAudioSink * sink, GstAudioRingBufferSpec * spec)
275 {
276   GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
277   float max_pitch = 1.0f;
278   uint32_t min_size;
279   MLResult result;
280 
281   result =
282       MLAudioGetOutputStreamDefaults (GST_AUDIO_INFO_CHANNELS (&spec->info),
283       GST_AUDIO_INFO_RATE (&spec->info), max_pitch, &self->format,
284       &self->recommended_buffer_size, &min_size);
285   if (result != MLResult_Ok) {
286     GST_ERROR_OBJECT (self, "Failed to get output stream defaults: %d", result);
287     return FALSE;
288   }
289 
290   if (!self->app) {
291     gst_element_post_message (GST_ELEMENT (self),
292         gst_message_new_element (GST_OBJECT (self),
293             gst_structure_new_empty ("gst.mlaudiosink.need-app")));
294   }
295 
296   self->wrapper = gst_ml_audio_wrapper_new (self->app);
297   self->has_buffer = FALSE;
298   self->stopped = FALSE;
299   self->paused = FALSE;
300 
301   g_mutex_lock (&active_sinks_mutex);
302   g_hash_table_add (active_sinks, self);
303   g_mutex_unlock (&active_sinks_mutex);
304 
305   /* createAudioNode() and createSoundWithOutputStream() must both be called in
306    * application's main thread, and in a single main loop iteration. */
307   if (!gst_ml_audio_wrapper_invoke_sync (self->wrapper, create_sound_cb, self))
308     return FALSE;
309 
310   return TRUE;
311 }
312 
313 static void
release_current_buffer(GstMLAudioSink * self)314 release_current_buffer (GstMLAudioSink * self)
315 {
316   if (self->has_buffer) {
317     memset (self->buffer.ptr + self->buffer_offset, 0,
318         self->buffer.size - self->buffer_offset);
319     gst_ml_audio_wrapper_release_buffer (self->wrapper);
320     self->has_buffer = false;
321   }
322 }
323 
324 static gboolean
gst_ml_audio_sink_unprepare(GstAudioSink * sink)325 gst_ml_audio_sink_unprepare (GstAudioSink * sink)
326 {
327   GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
328 
329   g_mutex_lock (&active_sinks_mutex);
330   g_hash_table_remove (active_sinks, self);
331   release_current_buffer (self);
332   g_clear_pointer (&self->wrapper, gst_ml_audio_wrapper_free);
333   g_mutex_unlock (&active_sinks_mutex);
334 
335   return TRUE;
336 }
337 
338 static gboolean
gst_ml_audio_sink_close(GstAudioSink * sink)339 gst_ml_audio_sink_close (GstAudioSink * sink)
340 {
341   /* Nothing to do in open/close */
342   return TRUE;
343 }
344 
345 static gint
gst_ml_audio_sink_write(GstAudioSink * sink,gpointer data,guint length)346 gst_ml_audio_sink_write (GstAudioSink * sink, gpointer data, guint length)
347 {
348   GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
349   guint8 *input = data;
350   gint written = 0;
351 
352   g_mutex_lock (&self->mutex);
353 
354   while (length > 0) {
355     MLResult result;
356     guint to_write;
357 
358     if (!wait_for_buffer (self)) {
359       written = -1;
360       break;
361     }
362 
363     if (self->stopped) {
364       /* Pretend we have written the full buffer (drop data) and return
365        * immediately. */
366       release_current_buffer (self);
367       gst_ml_audio_wrapper_stop_sound (self->wrapper);
368       written = length;
369       break;
370     }
371 
372     to_write = MIN (length, self->buffer.size - self->buffer_offset);
373     memcpy (self->buffer.ptr + self->buffer_offset, input + written, to_write);
374     self->buffer_offset += to_write;
375     if (self->buffer_offset == self->buffer.size) {
376       result = gst_ml_audio_wrapper_release_buffer (self->wrapper);
377       if (result != MLResult_Ok) {
378         GST_ERROR_OBJECT (self, "Failed to release buffer: %d", result);
379         written = -1;
380         break;
381       }
382       self->has_buffer = FALSE;
383     }
384 
385     length -= to_write;
386     written += to_write;
387   }
388 
389   if (self->paused) {
390     /* Pause was requested and we finished writing current buffer.
391      * See https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/665
392      */
393     gst_ml_audio_wrapper_pause_sound (self->wrapper);
394   }
395 
396   g_mutex_unlock (&self->mutex);
397 
398   return written;
399 }
400 
401 static guint
gst_ml_audio_sink_delay(GstAudioSink * sink)402 gst_ml_audio_sink_delay (GstAudioSink * sink)
403 {
404   GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
405   MLResult result;
406   float latency_ms;
407 
408   result = gst_ml_audio_wrapper_get_latency (self->wrapper, &latency_ms);
409   if (result != MLResult_Ok) {
410     GST_ERROR_OBJECT (self, "Failed to get latency: %d", result);
411     return 0;
412   }
413 
414   return latency_ms * self->format.samples_per_second / 1000;
415 }
416 
417 static void
gst_ml_audio_sink_pause(GstAudioSink * sink)418 gst_ml_audio_sink_pause (GstAudioSink * sink)
419 {
420   GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
421 
422   g_mutex_lock (&self->mutex);
423   self->paused = TRUE;
424   g_cond_signal (&self->cond);
425   g_mutex_unlock (&self->mutex);
426 }
427 
428 static void
gst_ml_audio_sink_resume(GstAudioSink * sink)429 gst_ml_audio_sink_resume (GstAudioSink * sink)
430 {
431   GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
432 
433   g_mutex_lock (&self->mutex);
434   self->paused = FALSE;
435   gst_ml_audio_wrapper_resume_sound (self->wrapper);
436   g_mutex_unlock (&self->mutex);
437 }
438 
439 static void
gst_ml_audio_sink_stop(GstAudioSink * sink)440 gst_ml_audio_sink_stop (GstAudioSink * sink)
441 {
442   GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
443 
444   g_mutex_lock (&self->mutex);
445   self->stopped = TRUE;
446   g_cond_signal (&self->cond);
447   g_mutex_unlock (&self->mutex);
448 }
449 
450 static void
gst_ml_audio_sink_class_init(GstMLAudioSinkClass * klass)451 gst_ml_audio_sink_class_init (GstMLAudioSinkClass * klass)
452 {
453   GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
454   GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
455   GstBaseSinkClass *basesink_class = GST_BASE_SINK_CLASS (klass);
456   GstAudioSinkClass *audiosink_class = GST_AUDIO_SINK_CLASS (klass);
457 
458   active_sinks = g_hash_table_new (NULL, NULL);
459   g_mutex_init (&active_sinks_mutex);
460 
461   gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_dispose);
462   gobject_class->set_property =
463       GST_DEBUG_FUNCPTR (gst_ml_audio_sink_set_property);
464   gobject_class->get_property =
465       GST_DEBUG_FUNCPTR (gst_ml_audio_sink_get_property);
466 
467   g_object_class_install_property (gobject_class,
468       PROP_AUDIO_NODE, g_param_spec_pointer ("audio-node",
469           "A pointer to a lumin::AudioNode object",
470           "Enable spatial sound", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
471 
472   g_object_class_install_property (gobject_class,
473       PROP_APP, g_param_spec_pointer ("app",
474           "A pointer to a lumin::BaseApp object",
475           "Enable spatial sound", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
476 
477   gst_element_class_set_static_metadata (element_class,
478       "Magic Leap Audio Sink",
479       "Sink/Audio", "Plays audio on a Magic Leap device",
480       "Xavier Claessens <xavier.claessens@collabora.com>");
481 
482   gst_element_class_add_static_pad_template (element_class, &sink_template);
483 
484   basesink_class->get_caps = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_getcaps);
485 
486   audiosink_class->open = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_open);
487   audiosink_class->prepare = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_prepare);
488   audiosink_class->unprepare = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_unprepare);
489   audiosink_class->close = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_close);
490   audiosink_class->write = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_write);
491   audiosink_class->delay = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_delay);
492   audiosink_class->pause = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_pause);
493   audiosink_class->resume = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_resume);
494   audiosink_class->stop = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_stop);
495 }
496