1 /*
2 * Copyright (C) 2019 Collabora Ltd.
3 * Author: Xavier Claessens <xavier.claessens@collabora.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation
8 * version 2.1 of the License.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 *
19 */
20
21 /**
22 * SECTION:mlaudiosink
23 * @short_description: Audio sink for Magic Leap platform
24 * @see_also: #GstAudioSink
25 *
26 * An audio sink element for LuminOS, the Magic Leap platform. There are 2 modes
27 * supported: normal and spatial. By default the audio is output directly to the
28 * stereo speakers, but in spatial mode the audio will be localised in the 3D
29 * environment. The user ears the sound as coming from a point in space, from a
30 * given distance and direction.
31 *
32 * To enable the spatial mode, the application needs to set a sync bus
33 * handler, using gst_bus_set_sync_handler(), to catch messages of type
34 * %GST_MESSAGE_ELEMENT named "gst.mlaudiosink.need-app" and
35 * "gst.mlaudiosink.need-audio-node". The need-app message will be posted first,
36 * application must then set the #GstMLAudioSink::app property with the pointer
37 * to application's lumin::BaseApp C++ object. That property can also be set on
38 * element creation in which case the need-app message won't be posted. After
39 * that, and if #GstMLAudioSink::app has been set, the need-audio-node message
40 * is posted from lumin::BaseApp's main thread. The application must then create
41 * a lumin::AudioNode C++ object, using lumin::Prism::createAudioNode(), and set
42 * the #GstMLAudioSink::audio-node property. Note that it is important that the
43 * lumin::AudioNode object must be created from within that message handler,
44 * and in the caller's thread, this is a limitation/bug of the platform
45 * (atleast until version 0.97).
46 *
47 * Here is an example of bus message handler to enable spatial sound:
48 * ```C
49 * static GstBusSyncReply
50 * bus_sync_handler_cb (GstBus * bus, GstMessage * msg, gpointer user_data)
51 * {
52 * MyApplication * self = user_data;
53 *
54 * if (GST_MESSAGE_TYPE (msg) == GST_MESSAGE_ELEMENT) {
55 * if (gst_message_has_name (msg, "gst.mlaudiosink.need-app")) {
56 * g_object_set (G_OBJECT (msg->src), "app", &self->app, NULL);
57 * } else if (gst_message_has_name (msg, "gst.mlaudiosink.need-audio-node")) {
58 * self->audio_node = self->prism->createAudioNode ();
59 * self->audio_node->setSpatialSoundEnable (true);
60 * self->ui_node->addChild(self->audio_node);
61 * g_object_set (G_OBJECT (msg->src), "audio-node", self->audio_node, NULL);
62 * }
63 * }
64 * return GST_BUS_PASS;
65 * }
66 * ```
67 *
68 * Since: 1.18
69 */
70
71 #ifdef HAVE_CONFIG_H
72 #include "config.h"
73 #endif
74
75 #include "mlaudiosink.h"
76 #include "mlaudiowrapper.h"
77
78 GST_DEBUG_CATEGORY_EXTERN (mgl_debug);
79 #define GST_CAT_DEFAULT mgl_debug
80
81 static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
82 GST_PAD_SINK,
83 GST_PAD_ALWAYS,
84 GST_STATIC_CAPS ("audio/x-raw, "
85 "format = (string) { S16LE }, "
86 "channels = (int) [ 1, 2 ], "
87 "rate = (int) [ 16000, 48000 ], " "layout = (string) interleaved"));
88
89 /* HACK: After calling MLAudioStopSound() there is no way to know when it will
90 * actually stop calling buffer_cb(). If the sink is disposed first, it would
91 * crash. Keep here a set of active sinks. */
92 static GHashTable *active_sinks;
93 static GMutex active_sinks_mutex;
94
95 struct _GstMLAudioSink
96 {
97 GstAudioSink parent;
98
99 gpointer audio_node;
100 gpointer app;
101
102 GstMLAudioWrapper *wrapper;
103 MLAudioBufferFormat format;
104 uint32_t recommended_buffer_size;
105 MLAudioBuffer buffer;
106 guint buffer_offset;
107 gboolean has_buffer;
108 gboolean paused;
109 gboolean stopped;
110
111 GMutex mutex;
112 GCond cond;
113 };
114
115 G_DEFINE_TYPE (GstMLAudioSink, gst_ml_audio_sink, GST_TYPE_AUDIO_SINK);
116 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (mlaudiosink, "mlaudiosink",
117 GST_RANK_PRIMARY + 10, GST_TYPE_ML_AUDIO_SINK,
118 GST_DEBUG_CATEGORY_INIT (mgl_debug, "magicleap", 0, "Magic Leap elements"));
119
120 enum
121 {
122 PROP_0,
123 PROP_AUDIO_NODE,
124 PROP_APP,
125 };
126
127 static void
gst_ml_audio_sink_init(GstMLAudioSink * self)128 gst_ml_audio_sink_init (GstMLAudioSink * self)
129 {
130 g_mutex_init (&self->mutex);
131 g_cond_init (&self->cond);
132 }
133
134 static void
gst_ml_audio_sink_dispose(GObject * object)135 gst_ml_audio_sink_dispose (GObject * object)
136 {
137 GstMLAudioSink *self = GST_ML_AUDIO_SINK (object);
138
139 g_mutex_clear (&self->mutex);
140 g_cond_clear (&self->cond);
141
142 G_OBJECT_CLASS (gst_ml_audio_sink_parent_class)->dispose (object);
143 }
144
145 static void
gst_ml_audio_sink_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)146 gst_ml_audio_sink_set_property (GObject * object, guint prop_id,
147 const GValue * value, GParamSpec * pspec)
148 {
149 GstMLAudioSink *self = GST_ML_AUDIO_SINK (object);
150
151 switch (prop_id) {
152 case PROP_AUDIO_NODE:
153 self->audio_node = g_value_get_pointer (value);
154 break;
155 case PROP_APP:
156 self->app = g_value_get_pointer (value);
157 break;
158 default:
159 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
160 break;
161 }
162 }
163
164 static void
gst_ml_audio_sink_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)165 gst_ml_audio_sink_get_property (GObject * object, guint prop_id, GValue * value,
166 GParamSpec * pspec)
167 {
168 GstMLAudioSink *self = GST_ML_AUDIO_SINK (object);
169
170 switch (prop_id) {
171 case PROP_AUDIO_NODE:
172 g_value_set_pointer (value, self->audio_node);
173 break;
174 case PROP_APP:
175 g_value_set_pointer (value, self->app);
176 break;
177 default:
178 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
179 break;
180 }
181 }
182
183 static GstCaps *
gst_ml_audio_sink_getcaps(GstBaseSink * bsink,GstCaps * filter)184 gst_ml_audio_sink_getcaps (GstBaseSink * bsink, GstCaps * filter)
185 {
186 GstCaps *caps;
187
188 caps = gst_static_caps_get (&sink_template.static_caps);
189
190 if (filter) {
191 gst_caps_replace (&caps,
192 gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST));
193 }
194
195 return caps;
196 }
197
198 static gboolean
gst_ml_audio_sink_open(GstAudioSink * sink)199 gst_ml_audio_sink_open (GstAudioSink * sink)
200 {
201 /* Nothing to do in open/close */
202 return TRUE;
203 }
204
205 static void
buffer_cb(MLHandle handle,gpointer user_data)206 buffer_cb (MLHandle handle, gpointer user_data)
207 {
208 GstMLAudioSink *self = user_data;
209
210 g_mutex_lock (&active_sinks_mutex);
211 if (!g_hash_table_contains (active_sinks, self))
212 goto out;
213
214 gst_ml_audio_wrapper_set_handle (self->wrapper, handle);
215
216 g_mutex_lock (&self->mutex);
217 g_cond_signal (&self->cond);
218 g_mutex_unlock (&self->mutex);
219
220 out:
221 g_mutex_unlock (&active_sinks_mutex);
222 }
223
224 /* Must be called with self->mutex locked */
225 static gboolean
wait_for_buffer(GstMLAudioSink * self)226 wait_for_buffer (GstMLAudioSink * self)
227 {
228 gboolean ret = TRUE;
229
230 while (!self->has_buffer && !self->stopped) {
231 MLResult result;
232
233 result = gst_ml_audio_wrapper_get_buffer (self->wrapper, &self->buffer);
234 if (result == MLResult_Ok) {
235 self->has_buffer = TRUE;
236 self->buffer_offset = 0;
237 } else if (result == MLAudioResult_BufferNotReady) {
238 g_cond_wait (&self->cond, &self->mutex);
239 } else {
240 GST_ERROR_OBJECT (self, "Failed to get output buffer: %d", result);
241 ret = FALSE;
242 break;
243 }
244 }
245
246 return ret;
247 }
248
249 static gboolean
create_sound_cb(GstMLAudioWrapper * wrapper,gpointer user_data)250 create_sound_cb (GstMLAudioWrapper * wrapper, gpointer user_data)
251 {
252 GstMLAudioSink *self = user_data;
253 MLResult result;
254
255 if (self->app) {
256 gst_element_post_message (GST_ELEMENT (self),
257 gst_message_new_element (GST_OBJECT (self),
258 gst_structure_new_empty ("gst.mlaudiosink.need-audio-node")));
259 }
260
261 gst_ml_audio_wrapper_set_node (self->wrapper, self->audio_node);
262
263 result = gst_ml_audio_wrapper_create_sound (self->wrapper, &self->format,
264 self->recommended_buffer_size, buffer_cb, self);
265 if (result != MLResult_Ok) {
266 GST_ERROR_OBJECT (self, "Failed to create output stream: %d", result);
267 return FALSE;
268 }
269
270 return TRUE;
271 }
272
273 static gboolean
gst_ml_audio_sink_prepare(GstAudioSink * sink,GstAudioRingBufferSpec * spec)274 gst_ml_audio_sink_prepare (GstAudioSink * sink, GstAudioRingBufferSpec * spec)
275 {
276 GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
277 float max_pitch = 1.0f;
278 uint32_t min_size;
279 MLResult result;
280
281 result =
282 MLAudioGetOutputStreamDefaults (GST_AUDIO_INFO_CHANNELS (&spec->info),
283 GST_AUDIO_INFO_RATE (&spec->info), max_pitch, &self->format,
284 &self->recommended_buffer_size, &min_size);
285 if (result != MLResult_Ok) {
286 GST_ERROR_OBJECT (self, "Failed to get output stream defaults: %d", result);
287 return FALSE;
288 }
289
290 if (!self->app) {
291 gst_element_post_message (GST_ELEMENT (self),
292 gst_message_new_element (GST_OBJECT (self),
293 gst_structure_new_empty ("gst.mlaudiosink.need-app")));
294 }
295
296 self->wrapper = gst_ml_audio_wrapper_new (self->app);
297 self->has_buffer = FALSE;
298 self->stopped = FALSE;
299 self->paused = FALSE;
300
301 g_mutex_lock (&active_sinks_mutex);
302 g_hash_table_add (active_sinks, self);
303 g_mutex_unlock (&active_sinks_mutex);
304
305 /* createAudioNode() and createSoundWithOutputStream() must both be called in
306 * application's main thread, and in a single main loop iteration. */
307 if (!gst_ml_audio_wrapper_invoke_sync (self->wrapper, create_sound_cb, self))
308 return FALSE;
309
310 return TRUE;
311 }
312
313 static void
release_current_buffer(GstMLAudioSink * self)314 release_current_buffer (GstMLAudioSink * self)
315 {
316 if (self->has_buffer) {
317 memset (self->buffer.ptr + self->buffer_offset, 0,
318 self->buffer.size - self->buffer_offset);
319 gst_ml_audio_wrapper_release_buffer (self->wrapper);
320 self->has_buffer = false;
321 }
322 }
323
324 static gboolean
gst_ml_audio_sink_unprepare(GstAudioSink * sink)325 gst_ml_audio_sink_unprepare (GstAudioSink * sink)
326 {
327 GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
328
329 g_mutex_lock (&active_sinks_mutex);
330 g_hash_table_remove (active_sinks, self);
331 release_current_buffer (self);
332 g_clear_pointer (&self->wrapper, gst_ml_audio_wrapper_free);
333 g_mutex_unlock (&active_sinks_mutex);
334
335 return TRUE;
336 }
337
338 static gboolean
gst_ml_audio_sink_close(GstAudioSink * sink)339 gst_ml_audio_sink_close (GstAudioSink * sink)
340 {
341 /* Nothing to do in open/close */
342 return TRUE;
343 }
344
345 static gint
gst_ml_audio_sink_write(GstAudioSink * sink,gpointer data,guint length)346 gst_ml_audio_sink_write (GstAudioSink * sink, gpointer data, guint length)
347 {
348 GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
349 guint8 *input = data;
350 gint written = 0;
351
352 g_mutex_lock (&self->mutex);
353
354 while (length > 0) {
355 MLResult result;
356 guint to_write;
357
358 if (!wait_for_buffer (self)) {
359 written = -1;
360 break;
361 }
362
363 if (self->stopped) {
364 /* Pretend we have written the full buffer (drop data) and return
365 * immediately. */
366 release_current_buffer (self);
367 gst_ml_audio_wrapper_stop_sound (self->wrapper);
368 written = length;
369 break;
370 }
371
372 to_write = MIN (length, self->buffer.size - self->buffer_offset);
373 memcpy (self->buffer.ptr + self->buffer_offset, input + written, to_write);
374 self->buffer_offset += to_write;
375 if (self->buffer_offset == self->buffer.size) {
376 result = gst_ml_audio_wrapper_release_buffer (self->wrapper);
377 if (result != MLResult_Ok) {
378 GST_ERROR_OBJECT (self, "Failed to release buffer: %d", result);
379 written = -1;
380 break;
381 }
382 self->has_buffer = FALSE;
383 }
384
385 length -= to_write;
386 written += to_write;
387 }
388
389 if (self->paused) {
390 /* Pause was requested and we finished writing current buffer.
391 * See https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/665
392 */
393 gst_ml_audio_wrapper_pause_sound (self->wrapper);
394 }
395
396 g_mutex_unlock (&self->mutex);
397
398 return written;
399 }
400
401 static guint
gst_ml_audio_sink_delay(GstAudioSink * sink)402 gst_ml_audio_sink_delay (GstAudioSink * sink)
403 {
404 GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
405 MLResult result;
406 float latency_ms;
407
408 result = gst_ml_audio_wrapper_get_latency (self->wrapper, &latency_ms);
409 if (result != MLResult_Ok) {
410 GST_ERROR_OBJECT (self, "Failed to get latency: %d", result);
411 return 0;
412 }
413
414 return latency_ms * self->format.samples_per_second / 1000;
415 }
416
417 static void
gst_ml_audio_sink_pause(GstAudioSink * sink)418 gst_ml_audio_sink_pause (GstAudioSink * sink)
419 {
420 GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
421
422 g_mutex_lock (&self->mutex);
423 self->paused = TRUE;
424 g_cond_signal (&self->cond);
425 g_mutex_unlock (&self->mutex);
426 }
427
428 static void
gst_ml_audio_sink_resume(GstAudioSink * sink)429 gst_ml_audio_sink_resume (GstAudioSink * sink)
430 {
431 GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
432
433 g_mutex_lock (&self->mutex);
434 self->paused = FALSE;
435 gst_ml_audio_wrapper_resume_sound (self->wrapper);
436 g_mutex_unlock (&self->mutex);
437 }
438
439 static void
gst_ml_audio_sink_stop(GstAudioSink * sink)440 gst_ml_audio_sink_stop (GstAudioSink * sink)
441 {
442 GstMLAudioSink *self = GST_ML_AUDIO_SINK (sink);
443
444 g_mutex_lock (&self->mutex);
445 self->stopped = TRUE;
446 g_cond_signal (&self->cond);
447 g_mutex_unlock (&self->mutex);
448 }
449
450 static void
gst_ml_audio_sink_class_init(GstMLAudioSinkClass * klass)451 gst_ml_audio_sink_class_init (GstMLAudioSinkClass * klass)
452 {
453 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
454 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
455 GstBaseSinkClass *basesink_class = GST_BASE_SINK_CLASS (klass);
456 GstAudioSinkClass *audiosink_class = GST_AUDIO_SINK_CLASS (klass);
457
458 active_sinks = g_hash_table_new (NULL, NULL);
459 g_mutex_init (&active_sinks_mutex);
460
461 gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_dispose);
462 gobject_class->set_property =
463 GST_DEBUG_FUNCPTR (gst_ml_audio_sink_set_property);
464 gobject_class->get_property =
465 GST_DEBUG_FUNCPTR (gst_ml_audio_sink_get_property);
466
467 g_object_class_install_property (gobject_class,
468 PROP_AUDIO_NODE, g_param_spec_pointer ("audio-node",
469 "A pointer to a lumin::AudioNode object",
470 "Enable spatial sound", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
471
472 g_object_class_install_property (gobject_class,
473 PROP_APP, g_param_spec_pointer ("app",
474 "A pointer to a lumin::BaseApp object",
475 "Enable spatial sound", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
476
477 gst_element_class_set_static_metadata (element_class,
478 "Magic Leap Audio Sink",
479 "Sink/Audio", "Plays audio on a Magic Leap device",
480 "Xavier Claessens <xavier.claessens@collabora.com>");
481
482 gst_element_class_add_static_pad_template (element_class, &sink_template);
483
484 basesink_class->get_caps = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_getcaps);
485
486 audiosink_class->open = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_open);
487 audiosink_class->prepare = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_prepare);
488 audiosink_class->unprepare = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_unprepare);
489 audiosink_class->close = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_close);
490 audiosink_class->write = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_write);
491 audiosink_class->delay = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_delay);
492 audiosink_class->pause = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_pause);
493 audiosink_class->resume = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_resume);
494 audiosink_class->stop = GST_DEBUG_FUNCPTR (gst_ml_audio_sink_stop);
495 }
496