1 /*
2 * GStreamer
3 * Copyright (C) 2015 Vivia Nikolaidou <vivia@toolsonair.com>
4 *
5 * Based on gstlevel.c:
6 * Copyright (C) 2000,2001,2002,2003,2005
7 * Thomas Vander Stichele <thomas at apestaart dot org>
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Library General Public
11 * License as published by the Free Software Foundation; either
12 * version 2 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Library General Public License for more details.
18 *
19 * You should have received a copy of the GNU Library General Public
20 * License along with this library; if not, write to the
21 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
22 * Boston, MA 02110-1301, USA.
23 */
24
25 /**
26 * SECTION:element-videoframe-audiolevel
27 * @title: videoframe-audiolevel
28 *
29 * This element acts like a synchronized audio/video "level". It gathers
30 * all audio buffers sent between two video frames, and then sends a message
31 * that contains the RMS value of all samples for these buffers.
32 *
33 * ## Example launch line
34 * |[
35 * gst-launch-1.0 -m filesrc location="file.mkv" ! decodebin name=d ! "audio/x-raw" ! videoframe-audiolevel name=l ! autoaudiosink d. ! "video/x-raw" ! l. l. ! queue ! autovideosink ]|
36 *
37 */
38
39 #ifdef HAVE_CONFIG_H
40 #include "config.h"
41 #endif
42
43 /* FIXME 2.0: suppress warnings for deprecated API such as GValueArray
44 * with newer GLib versions (>= 2.31.0) */
45 #define GLIB_DISABLE_DEPRECATION_WARNINGS
46
47 #include "gstvideoframe-audiolevel.h"
48 #include <math.h>
49
50 #define GST_CAT_DEFAULT gst_videoframe_audiolevel_debug
51 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
52 # define FORMATS "{ S8, S16LE, S32LE, F32LE, F64LE }"
53 #else
54 # define FORMATS "{ S8, S16BE, S32BE, F32BE, F64BE }"
55 #endif
56 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
57
58 static GstStaticPadTemplate audio_sink_template =
59 GST_STATIC_PAD_TEMPLATE ("asink",
60 GST_PAD_SINK,
61 GST_PAD_ALWAYS,
62 GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (FORMATS))
63 );
64
65 static GstStaticPadTemplate audio_src_template =
66 GST_STATIC_PAD_TEMPLATE ("asrc",
67 GST_PAD_SRC,
68 GST_PAD_ALWAYS,
69 GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (FORMATS))
70 );
71
72 static GstStaticPadTemplate video_sink_template =
73 GST_STATIC_PAD_TEMPLATE ("vsink",
74 GST_PAD_SINK,
75 GST_PAD_ALWAYS,
76 GST_STATIC_CAPS ("video/x-raw")
77 );
78
79 static GstStaticPadTemplate video_src_template =
80 GST_STATIC_PAD_TEMPLATE ("vsrc",
81 GST_PAD_SRC,
82 GST_PAD_ALWAYS,
83 GST_STATIC_CAPS ("video/x-raw")
84 );
85
86 #define parent_class gst_videoframe_audiolevel_parent_class
87 G_DEFINE_TYPE (GstVideoFrameAudioLevel, gst_videoframe_audiolevel,
88 GST_TYPE_ELEMENT);
89 GST_ELEMENT_REGISTER_DEFINE (videoframe_audiolevel, "videoframe-audiolevel",
90 GST_RANK_NONE, GST_TYPE_VIDEOFRAME_AUDIOLEVEL);
91
92 static GstFlowReturn gst_videoframe_audiolevel_asink_chain (GstPad * pad,
93 GstObject * parent, GstBuffer * inbuf);
94 static GstFlowReturn gst_videoframe_audiolevel_vsink_chain (GstPad * pad,
95 GstObject * parent, GstBuffer * inbuf);
96 static gboolean gst_videoframe_audiolevel_asink_event (GstPad * pad,
97 GstObject * parent, GstEvent * event);
98 static gboolean gst_videoframe_audiolevel_vsink_event (GstPad * pad,
99 GstObject * parent, GstEvent * event);
100 static GstIterator *gst_videoframe_audiolevel_iterate_internal_links (GstPad *
101 pad, GstObject * parent);
102
103 static void gst_videoframe_audiolevel_finalize (GObject * gobject);
104
105 static GstStateChangeReturn gst_videoframe_audiolevel_change_state (GstElement *
106 element, GstStateChange transition);
107
108 static void
gst_videoframe_audiolevel_class_init(GstVideoFrameAudioLevelClass * klass)109 gst_videoframe_audiolevel_class_init (GstVideoFrameAudioLevelClass * klass)
110 {
111 GstElementClass *gstelement_class;
112 GObjectClass *gobject_class = (GObjectClass *) klass;
113
114 GST_DEBUG_CATEGORY_INIT (gst_videoframe_audiolevel_debug,
115 "videoframe-audiolevel", 0, "Synchronized audio/video level");
116
117 gstelement_class = (GstElementClass *) klass;
118
119 gst_element_class_set_static_metadata (gstelement_class,
120 "Video-frame audio level", "Filter/Analyzer/Audio",
121 "Synchronized audio/video RMS Level messenger for audio/raw",
122 "Vivia Nikolaidou <vivia@toolsonair.com>");
123
124 gobject_class->finalize = gst_videoframe_audiolevel_finalize;
125 gstelement_class->change_state = gst_videoframe_audiolevel_change_state;
126
127 gst_element_class_add_static_pad_template (gstelement_class,
128 &audio_src_template);
129 gst_element_class_add_static_pad_template (gstelement_class,
130 &audio_sink_template);
131
132 gst_element_class_add_static_pad_template (gstelement_class,
133 &video_src_template);
134 gst_element_class_add_static_pad_template (gstelement_class,
135 &video_sink_template);
136 }
137
138 static void
gst_videoframe_audiolevel_init(GstVideoFrameAudioLevel * self)139 gst_videoframe_audiolevel_init (GstVideoFrameAudioLevel * self)
140 {
141 self->asinkpad =
142 gst_pad_new_from_static_template (&audio_sink_template, "asink");
143 gst_pad_set_chain_function (self->asinkpad,
144 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_asink_chain));
145 gst_pad_set_event_function (self->asinkpad,
146 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_asink_event));
147 gst_pad_set_iterate_internal_links_function (self->asinkpad,
148 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
149 gst_element_add_pad (GST_ELEMENT (self), self->asinkpad);
150
151 self->vsinkpad =
152 gst_pad_new_from_static_template (&video_sink_template, "vsink");
153 gst_pad_set_chain_function (self->vsinkpad,
154 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_vsink_chain));
155 gst_pad_set_event_function (self->vsinkpad,
156 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_vsink_event));
157 gst_pad_set_iterate_internal_links_function (self->vsinkpad,
158 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
159 gst_element_add_pad (GST_ELEMENT (self), self->vsinkpad);
160
161 self->asrcpad =
162 gst_pad_new_from_static_template (&audio_src_template, "asrc");
163 gst_pad_set_iterate_internal_links_function (self->asrcpad,
164 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
165 gst_element_add_pad (GST_ELEMENT (self), self->asrcpad);
166
167 self->vsrcpad =
168 gst_pad_new_from_static_template (&video_src_template, "vsrc");
169 gst_pad_set_iterate_internal_links_function (self->vsrcpad,
170 GST_DEBUG_FUNCPTR (gst_videoframe_audiolevel_iterate_internal_links));
171 gst_element_add_pad (GST_ELEMENT (self), self->vsrcpad);
172
173 GST_PAD_SET_PROXY_CAPS (self->asinkpad);
174 GST_PAD_SET_PROXY_ALLOCATION (self->asinkpad);
175
176 GST_PAD_SET_PROXY_CAPS (self->asrcpad);
177 GST_PAD_SET_PROXY_SCHEDULING (self->asrcpad);
178
179 GST_PAD_SET_PROXY_CAPS (self->vsinkpad);
180 GST_PAD_SET_PROXY_ALLOCATION (self->vsinkpad);
181
182 GST_PAD_SET_PROXY_CAPS (self->vsrcpad);
183 GST_PAD_SET_PROXY_SCHEDULING (self->vsrcpad);
184
185 self->adapter = gst_adapter_new ();
186
187 g_queue_init (&self->vtimeq);
188 self->first_time = GST_CLOCK_TIME_NONE;
189 self->total_frames = 0;
190 /* alignment_threshold and discont_wait should become properties if needed */
191 self->alignment_threshold = 40 * GST_MSECOND;
192 self->discont_time = GST_CLOCK_TIME_NONE;
193 self->next_offset = -1;
194 self->discont_wait = 1 * GST_SECOND;
195
196 self->video_eos_flag = FALSE;
197 self->audio_flush_flag = FALSE;
198 self->shutdown_flag = FALSE;
199
200 g_mutex_init (&self->mutex);
201 g_cond_init (&self->cond);
202 }
203
204 static GstStateChangeReturn
gst_videoframe_audiolevel_change_state(GstElement * element,GstStateChange transition)205 gst_videoframe_audiolevel_change_state (GstElement * element,
206 GstStateChange transition)
207 {
208 GstStateChangeReturn ret;
209 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (element);
210
211 switch (transition) {
212 case GST_STATE_CHANGE_PAUSED_TO_READY:
213 g_mutex_lock (&self->mutex);
214 self->shutdown_flag = TRUE;
215 g_cond_signal (&self->cond);
216 g_mutex_unlock (&self->mutex);
217 break;
218 case GST_STATE_CHANGE_READY_TO_PAUSED:
219 g_mutex_lock (&self->mutex);
220 self->shutdown_flag = FALSE;
221 self->video_eos_flag = FALSE;
222 self->audio_flush_flag = FALSE;
223 g_mutex_unlock (&self->mutex);
224 default:
225 break;
226 }
227
228 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
229
230 switch (transition) {
231 case GST_STATE_CHANGE_PAUSED_TO_READY:
232 g_mutex_lock (&self->mutex);
233 self->first_time = GST_CLOCK_TIME_NONE;
234 self->total_frames = 0;
235 gst_segment_init (&self->asegment, GST_FORMAT_UNDEFINED);
236 gst_segment_init (&self->vsegment, GST_FORMAT_UNDEFINED);
237 self->vsegment.position = GST_CLOCK_TIME_NONE;
238 gst_adapter_clear (self->adapter);
239 g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
240 g_queue_clear (&self->vtimeq);
241 if (self->CS) {
242 g_free (self->CS);
243 self->CS = NULL;
244 }
245 g_mutex_unlock (&self->mutex);
246 break;
247 default:
248 break;
249 }
250
251 return ret;
252 }
253
254 static void
gst_videoframe_audiolevel_finalize(GObject * object)255 gst_videoframe_audiolevel_finalize (GObject * object)
256 {
257 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (object);
258
259 if (self->adapter) {
260 g_object_unref (self->adapter);
261 self->adapter = NULL;
262 }
263 g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
264 g_queue_clear (&self->vtimeq);
265 self->first_time = GST_CLOCK_TIME_NONE;
266 self->total_frames = 0;
267 if (self->CS) {
268 g_free (self->CS);
269 self->CS = NULL;
270 }
271
272 g_mutex_clear (&self->mutex);
273 g_cond_clear (&self->cond);
274
275 G_OBJECT_CLASS (parent_class)->finalize (object);
276 }
277
278 #define DEFINE_INT_LEVEL_CALCULATOR(TYPE, RESOLUTION) \
279 static void inline \
280 gst_videoframe_audiolevel_calculate_##TYPE (gpointer data, guint num, guint channels, \
281 gdouble *NCS) \
282 { \
283 TYPE * in = (TYPE *)data; \
284 register guint j; \
285 gdouble squaresum = 0.0; /* square sum of the input samples */ \
286 register gdouble square = 0.0; /* Square */ \
287 gdouble normalizer; /* divisor to get a [-1.0, 1.0] range */ \
288 \
289 /* *NCS = 0.0; Normalized Cumulative Square */ \
290 \
291 for (j = 0; j < num; j += channels) { \
292 square = ((gdouble) in[j]) * in[j]; \
293 squaresum += square; \
294 } \
295 \
296 normalizer = (gdouble) (G_GINT64_CONSTANT(1) << (RESOLUTION * 2)); \
297 *NCS = squaresum / normalizer; \
298 }
299
300 DEFINE_INT_LEVEL_CALCULATOR (gint32, 31);
301 DEFINE_INT_LEVEL_CALCULATOR (gint16, 15);
302 DEFINE_INT_LEVEL_CALCULATOR (gint8, 7);
303
304 #define DEFINE_FLOAT_LEVEL_CALCULATOR(TYPE) \
305 static void inline \
306 gst_videoframe_audiolevel_calculate_##TYPE (gpointer data, guint num, guint channels, \
307 gdouble *NCS) \
308 { \
309 TYPE * in = (TYPE *)data; \
310 register guint j; \
311 gdouble squaresum = 0.0; /* square sum of the input samples */ \
312 register gdouble square = 0.0; /* Square */ \
313 \
314 /* *NCS = 0.0; Normalized Cumulative Square */ \
315 \
316 for (j = 0; j < num; j += channels) { \
317 square = ((gdouble) in[j]) * in[j]; \
318 squaresum += square; \
319 } \
320 \
321 *NCS = squaresum; \
322 }
323
324 DEFINE_FLOAT_LEVEL_CALCULATOR (gfloat);
325 DEFINE_FLOAT_LEVEL_CALCULATOR (gdouble);
326
327 static gboolean
gst_videoframe_audiolevel_vsink_event(GstPad * pad,GstObject * parent,GstEvent * event)328 gst_videoframe_audiolevel_vsink_event (GstPad * pad, GstObject * parent,
329 GstEvent * event)
330 {
331 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
332 GST_LOG_OBJECT (pad, "Got %s event", GST_EVENT_TYPE_NAME (event));
333
334 switch (GST_EVENT_TYPE (event)) {
335 case GST_EVENT_SEGMENT:
336 g_mutex_lock (&self->mutex);
337 g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
338 g_queue_clear (&self->vtimeq);
339 g_mutex_unlock (&self->mutex);
340 gst_event_copy_segment (event, &self->vsegment);
341 if (self->vsegment.format != GST_FORMAT_TIME)
342 return FALSE;
343 self->vsegment.position = GST_CLOCK_TIME_NONE;
344 break;
345 case GST_EVENT_GAP:
346 return TRUE;
347 case GST_EVENT_EOS:
348 g_mutex_lock (&self->mutex);
349 self->video_eos_flag = TRUE;
350 g_cond_signal (&self->cond);
351 g_mutex_unlock (&self->mutex);
352 break;
353 case GST_EVENT_FLUSH_STOP:
354 g_mutex_lock (&self->mutex);
355 g_queue_foreach (&self->vtimeq, (GFunc) g_free, NULL);
356 g_queue_clear (&self->vtimeq);
357 gst_segment_init (&self->vsegment, GST_FORMAT_UNDEFINED);
358 g_cond_signal (&self->cond);
359 g_mutex_unlock (&self->mutex);
360 self->vsegment.position = GST_CLOCK_TIME_NONE;
361 break;
362 default:
363 break;
364 }
365 return gst_pad_event_default (pad, parent, event);
366 }
367
368 static gboolean
gst_videoframe_audiolevel_asink_event(GstPad * pad,GstObject * parent,GstEvent * event)369 gst_videoframe_audiolevel_asink_event (GstPad * pad, GstObject * parent,
370 GstEvent * event)
371 {
372 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
373 GST_LOG_OBJECT (pad, "Got %s event", GST_EVENT_TYPE_NAME (event));
374
375 switch (GST_EVENT_TYPE (event)) {
376 case GST_EVENT_SEGMENT:
377 self->first_time = GST_CLOCK_TIME_NONE;
378 self->total_frames = 0;
379 gst_adapter_clear (self->adapter);
380 gst_event_copy_segment (event, &self->asegment);
381 if (self->asegment.format != GST_FORMAT_TIME)
382 return FALSE;
383 break;
384 case GST_EVENT_FLUSH_START:
385 g_mutex_lock (&self->mutex);
386 self->audio_flush_flag = TRUE;
387 g_cond_signal (&self->cond);
388 g_mutex_unlock (&self->mutex);
389 break;
390 case GST_EVENT_FLUSH_STOP:
391 self->audio_flush_flag = FALSE;
392 self->total_frames = 0;
393 self->first_time = GST_CLOCK_TIME_NONE;
394 gst_adapter_clear (self->adapter);
395 gst_segment_init (&self->asegment, GST_FORMAT_UNDEFINED);
396 break;
397 case GST_EVENT_CAPS:{
398 GstCaps *caps;
399 gint channels;
400 gst_event_parse_caps (event, &caps);
401 GST_DEBUG_OBJECT (self, "Got caps %" GST_PTR_FORMAT, caps);
402 if (!gst_audio_info_from_caps (&self->ainfo, caps))
403 return FALSE;
404 switch (GST_AUDIO_INFO_FORMAT (&self->ainfo)) {
405 case GST_AUDIO_FORMAT_S8:
406 self->process = gst_videoframe_audiolevel_calculate_gint8;
407 break;
408 case GST_AUDIO_FORMAT_S16:
409 self->process = gst_videoframe_audiolevel_calculate_gint16;
410 break;
411 case GST_AUDIO_FORMAT_S32:
412 self->process = gst_videoframe_audiolevel_calculate_gint32;
413 break;
414 case GST_AUDIO_FORMAT_F32:
415 self->process = gst_videoframe_audiolevel_calculate_gfloat;
416 break;
417 case GST_AUDIO_FORMAT_F64:
418 self->process = gst_videoframe_audiolevel_calculate_gdouble;
419 break;
420 default:
421 self->process = NULL;
422 break;
423 }
424 gst_adapter_clear (self->adapter);
425 channels = GST_AUDIO_INFO_CHANNELS (&self->ainfo);
426 self->first_time = GST_CLOCK_TIME_NONE;
427 self->total_frames = 0;
428 if (self->CS)
429 g_free (self->CS);
430 self->CS = g_new0 (gdouble, channels);
431 break;
432 }
433 default:
434 break;
435 }
436
437 return gst_pad_event_default (pad, parent, event);
438 }
439
440 static GstMessage *
update_rms_from_buffer(GstVideoFrameAudioLevel * self,GstBuffer * inbuf)441 update_rms_from_buffer (GstVideoFrameAudioLevel * self, GstBuffer * inbuf)
442 {
443 GstMapInfo map;
444 guint8 *in_data;
445 gsize in_size;
446 gdouble CS;
447 guint i;
448 guint num_frames, frames;
449 guint num_int_samples = 0; /* number of interleaved samples
450 * ie. total count for all channels combined */
451 gint channels, rate, bps;
452 GValue v = G_VALUE_INIT;
453 GValue va = G_VALUE_INIT;
454 GValueArray *a;
455 GstStructure *s;
456 GstMessage *msg;
457 GstClockTime duration, running_time;
458
459 channels = GST_AUDIO_INFO_CHANNELS (&self->ainfo);
460 bps = GST_AUDIO_INFO_BPS (&self->ainfo);
461 rate = GST_AUDIO_INFO_RATE (&self->ainfo);
462
463 gst_buffer_map (inbuf, &map, GST_MAP_READ);
464 in_data = map.data;
465 in_size = map.size;
466
467 num_int_samples = in_size / bps;
468
469 GST_LOG_OBJECT (self, "analyzing %u sample frames at ts %" GST_TIME_FORMAT,
470 num_int_samples, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (inbuf)));
471
472 g_return_val_if_fail (num_int_samples % channels == 0, NULL);
473
474 num_frames = num_int_samples / channels;
475 frames = num_frames;
476 duration = GST_FRAMES_TO_CLOCK_TIME (frames, rate);
477 if (num_frames > 0) {
478 for (i = 0; i < channels; ++i) {
479 self->process (in_data + (bps * i), num_int_samples, channels, &CS);
480 GST_LOG_OBJECT (self,
481 "[%d]: cumulative squares %lf, over %d samples/%d channels",
482 i, CS, num_int_samples, channels);
483 self->CS[i] += CS;
484 }
485 in_data += num_frames * bps;
486
487 self->total_frames += num_frames;
488 }
489 running_time =
490 self->first_time + gst_util_uint64_scale (self->total_frames, GST_SECOND,
491 rate);
492
493 a = g_value_array_new (channels);
494 s = gst_structure_new ("videoframe-audiolevel", "running-time", G_TYPE_UINT64,
495 running_time, "duration", G_TYPE_UINT64, duration, NULL);
496
497 g_value_init (&v, G_TYPE_DOUBLE);
498 g_value_init (&va, G_TYPE_VALUE_ARRAY);
499 for (i = 0; i < channels; i++) {
500 gdouble rms;
501 if (frames == 0 || self->CS[i] == 0) {
502 rms = 0; /* empty buffer */
503 } else {
504 rms = sqrt (self->CS[i] / frames);
505 }
506 self->CS[i] = 0.0;
507 g_value_set_double (&v, rms);
508 g_value_array_append (a, &v);
509 }
510 g_value_take_boxed (&va, a);
511 gst_structure_take_value (s, "rms", &va);
512 msg = gst_message_new_element (GST_OBJECT (self), s);
513
514 gst_buffer_unmap (inbuf, &map);
515
516 return msg;
517 }
518
519 static GstFlowReturn
gst_videoframe_audiolevel_vsink_chain(GstPad * pad,GstObject * parent,GstBuffer * inbuf)520 gst_videoframe_audiolevel_vsink_chain (GstPad * pad, GstObject * parent,
521 GstBuffer * inbuf)
522 {
523 GstClockTime timestamp;
524 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
525 GstClockTime duration;
526 GstClockTime *ptrtime = g_new (GstClockTime, 1);
527
528 timestamp = GST_BUFFER_TIMESTAMP (inbuf);
529 *ptrtime =
530 gst_segment_to_running_time (&self->vsegment, GST_FORMAT_TIME, timestamp);
531 g_mutex_lock (&self->mutex);
532 self->vsegment.position = timestamp;
533 duration = GST_BUFFER_DURATION (inbuf);
534 if (duration != GST_CLOCK_TIME_NONE)
535 self->vsegment.position += duration;
536 g_queue_push_tail (&self->vtimeq, ptrtime);
537 g_cond_signal (&self->cond);
538 GST_DEBUG_OBJECT (pad, "Pushed a frame");
539 g_mutex_unlock (&self->mutex);
540 return gst_pad_push (self->vsrcpad, inbuf);
541 }
542
543 static GstFlowReturn
gst_videoframe_audiolevel_asink_chain(GstPad * pad,GstObject * parent,GstBuffer * inbuf)544 gst_videoframe_audiolevel_asink_chain (GstPad * pad, GstObject * parent,
545 GstBuffer * inbuf)
546 {
547 GstClockTime timestamp, cur_time;
548 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
549 GstBuffer *buf;
550 gsize inbuf_size;
551 guint64 start_offset, end_offset;
552 GstClockTime running_time;
553 gint rate, bpf;
554 gboolean discont = FALSE;
555
556 timestamp = GST_BUFFER_TIMESTAMP (inbuf);
557 running_time =
558 gst_segment_to_running_time (&self->asegment, GST_FORMAT_TIME, timestamp);
559
560 rate = GST_AUDIO_INFO_RATE (&self->ainfo);
561 bpf = GST_AUDIO_INFO_BPF (&self->ainfo);
562 start_offset = gst_util_uint64_scale (timestamp, rate, GST_SECOND);
563 inbuf_size = gst_buffer_get_size (inbuf);
564 end_offset = start_offset + inbuf_size / bpf;
565
566 g_mutex_lock (&self->mutex);
567
568 if (GST_BUFFER_IS_DISCONT (inbuf)
569 || GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_RESYNC)
570 || self->first_time == GST_CLOCK_TIME_NONE) {
571 discont = TRUE;
572 } else {
573 guint64 diff, max_sample_diff;
574
575 /* Check discont, based on audiobasesink */
576 if (start_offset <= self->next_offset)
577 diff = self->next_offset - start_offset;
578 else
579 diff = start_offset - self->next_offset;
580
581 max_sample_diff =
582 gst_util_uint64_scale_int (self->alignment_threshold, rate, GST_SECOND);
583
584 /* Discont! */
585 if (G_UNLIKELY (diff >= max_sample_diff)) {
586 if (self->discont_wait > 0) {
587 if (self->discont_time == GST_CLOCK_TIME_NONE) {
588 self->discont_time = timestamp;
589 } else if (timestamp - self->discont_time >= self->discont_wait) {
590 discont = TRUE;
591 self->discont_time = GST_CLOCK_TIME_NONE;
592 }
593 } else {
594 discont = TRUE;
595 }
596 } else if (G_UNLIKELY (self->discont_time != GST_CLOCK_TIME_NONE)) {
597 /* we have had a discont, but are now back on track! */
598 self->discont_time = GST_CLOCK_TIME_NONE;
599 }
600 }
601
602 if (discont) {
603 /* Have discont, need resync */
604 if (self->next_offset != -1)
605 GST_INFO_OBJECT (pad, "Have discont. Expected %"
606 G_GUINT64_FORMAT ", got %" G_GUINT64_FORMAT,
607 self->next_offset, start_offset);
608 self->total_frames = 0;
609 self->first_time = running_time;
610 self->next_offset = end_offset;
611 } else {
612 self->next_offset += inbuf_size / bpf;
613 }
614
615 gst_adapter_push (self->adapter, gst_buffer_ref (inbuf));
616
617 GST_DEBUG_OBJECT (self, "Queue length %i",
618 g_queue_get_length (&self->vtimeq));
619
620 while (TRUE) {
621 GstClockTime *vt0, *vt1;
622 GstClockTime vtemp;
623 GstMessage *msg;
624 gsize bytes, available_bytes;
625
626 vtemp = GST_CLOCK_TIME_NONE;
627
628 while (!(g_queue_get_length (&self->vtimeq) >= 2 || self->video_eos_flag
629 || self->audio_flush_flag || self->shutdown_flag))
630 g_cond_wait (&self->cond, &self->mutex);
631
632 if (self->audio_flush_flag || self->shutdown_flag) {
633 g_mutex_unlock (&self->mutex);
634 gst_buffer_unref (inbuf);
635 return GST_FLOW_FLUSHING;
636 } else if (self->video_eos_flag) {
637 GST_DEBUG_OBJECT (self, "Video EOS flag alert");
638 /* nothing to do here if queue is empty */
639 if (g_queue_get_length (&self->vtimeq) == 0)
640 break;
641
642 if (g_queue_get_length (&self->vtimeq) < 2) {
643 vtemp = self->vsegment.position;
644 } else if (self->vsegment.position == GST_CLOCK_TIME_NONE) {
645 /* g_queue_get_length is surely >= 2 at this point
646 * so the adapter isn't empty */
647 buf =
648 gst_adapter_take_buffer (self->adapter,
649 gst_adapter_available (self->adapter));
650 if (buf != NULL) {
651 GstMessage *msg;
652 msg = update_rms_from_buffer (self, buf);
653 g_mutex_unlock (&self->mutex);
654 gst_element_post_message (GST_ELEMENT (self), msg);
655 gst_buffer_unref (buf);
656 g_mutex_lock (&self->mutex); /* we unlock again later */
657 }
658 break;
659 }
660 } else if (g_queue_get_length (&self->vtimeq) < 2) {
661 continue;
662 }
663
664 vt0 = g_queue_pop_head (&self->vtimeq);
665 if (vtemp == GST_CLOCK_TIME_NONE)
666 vt1 = g_queue_peek_head (&self->vtimeq);
667 else
668 vt1 = &vtemp;
669
670 cur_time =
671 self->first_time + gst_util_uint64_scale (self->total_frames,
672 GST_SECOND, rate);
673 GST_DEBUG_OBJECT (self,
674 "Processing: current time is %" GST_TIME_FORMAT,
675 GST_TIME_ARGS (cur_time));
676 GST_DEBUG_OBJECT (self, "Total frames is %i with a rate of %d",
677 self->total_frames, rate);
678 GST_DEBUG_OBJECT (self, "Start time is %" GST_TIME_FORMAT,
679 GST_TIME_ARGS (self->first_time));
680 GST_DEBUG_OBJECT (self, "Time on top is %" GST_TIME_FORMAT,
681 GST_TIME_ARGS (*vt0));
682
683 if (cur_time < *vt0) {
684 guint num_frames =
685 gst_util_uint64_scale (*vt0 - cur_time, rate, GST_SECOND);
686 bytes = num_frames * GST_AUDIO_INFO_BPF (&self->ainfo);
687 available_bytes = gst_adapter_available (self->adapter);
688 if (available_bytes == 0) {
689 g_queue_push_head (&self->vtimeq, vt0);
690 break;
691 }
692 if (bytes == 0) {
693 cur_time = *vt0;
694 } else {
695 GST_DEBUG_OBJECT (self,
696 "Flushed %" G_GSIZE_FORMAT " out of %" G_GSIZE_FORMAT " bytes",
697 bytes, available_bytes);
698 gst_adapter_flush (self->adapter, MIN (bytes, available_bytes));
699 self->total_frames += num_frames;
700 if (available_bytes <= bytes) {
701 g_queue_push_head (&self->vtimeq, vt0);
702 break;
703 }
704 cur_time =
705 self->first_time + gst_util_uint64_scale (self->total_frames,
706 GST_SECOND, rate);
707 }
708 }
709 if (*vt1 > cur_time) {
710 bytes =
711 GST_AUDIO_INFO_BPF (&self->ainfo) * gst_util_uint64_scale (*vt1 -
712 cur_time, rate, GST_SECOND);
713 } else {
714 bytes = 0; /* We just need to discard vt0 */
715 }
716 available_bytes = gst_adapter_available (self->adapter);
717 GST_DEBUG_OBJECT (self,
718 "Adapter contains %" G_GSIZE_FORMAT " out of %" G_GSIZE_FORMAT " bytes",
719 available_bytes, bytes);
720
721 if (available_bytes < bytes) {
722 g_queue_push_head (&self->vtimeq, vt0);
723 goto done;
724 }
725
726 if (bytes > 0) {
727 buf = gst_adapter_take_buffer (self->adapter, bytes);
728 g_assert (buf != NULL);
729 } else {
730 /* Just an empty buffer */
731 buf = gst_buffer_new ();
732 }
733 msg = update_rms_from_buffer (self, buf);
734 g_mutex_unlock (&self->mutex);
735 gst_element_post_message (GST_ELEMENT (self), msg);
736 g_mutex_lock (&self->mutex);
737
738 gst_buffer_unref (buf);
739 g_free (vt0);
740 if (available_bytes == bytes)
741 break;
742 }
743 done:
744 g_mutex_unlock (&self->mutex);
745 return gst_pad_push (self->asrcpad, inbuf);
746 }
747
748 static GstIterator *
gst_videoframe_audiolevel_iterate_internal_links(GstPad * pad,GstObject * parent)749 gst_videoframe_audiolevel_iterate_internal_links (GstPad * pad,
750 GstObject * parent)
751 {
752 GstIterator *it = NULL;
753 GstPad *opad;
754 GValue val = { 0, };
755 GstVideoFrameAudioLevel *self = GST_VIDEOFRAME_AUDIOLEVEL (parent);
756
757 if (self->asinkpad == pad)
758 opad = gst_object_ref (self->asrcpad);
759 else if (self->asrcpad == pad)
760 opad = gst_object_ref (self->asinkpad);
761 else if (self->vsinkpad == pad)
762 opad = gst_object_ref (self->vsrcpad);
763 else if (self->vsrcpad == pad)
764 opad = gst_object_ref (self->vsinkpad);
765 else
766 goto out;
767
768 g_value_init (&val, GST_TYPE_PAD);
769 g_value_set_object (&val, opad);
770 it = gst_iterator_new_single (GST_TYPE_PAD, &val);
771 g_value_unset (&val);
772
773 gst_object_unref (opad);
774
775 out:
776 return it;
777 }
778
779 static gboolean
gst_videoframe_audiolevel_plugin_init(GstPlugin * plugin)780 gst_videoframe_audiolevel_plugin_init (GstPlugin * plugin)
781 {
782 return GST_ELEMENT_REGISTER (videoframe_audiolevel, plugin);
783 }
784
785 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
786 GST_VERSION_MINOR,
787 videoframe_audiolevel,
788 "Video frame-synchronized audio level",
789 gst_videoframe_audiolevel_plugin_init, VERSION, GST_LICENSE,
790 GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN);
791