1 /* GStreamer
2 * Copyright (C) <2011> Stefan Kost <ensonic@users.sf.net>
3 *
4 * gstsynaescope.c: frequency spectrum scope
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
15 *
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
20 */
21 /**
22 * SECTION:element-synaescope
23 * @title: synaescope
24 * @see_also: goom
25 *
26 * Synaescope is an audio visualisation element. It analyzes frequencies and
27 * out-of phase properties of audio and draws this as clouds of stars.
28 *
29 * ## Example launch line
30 * |[
31 * gst-launch-1.0 audiotestsrc ! audioconvert ! synaescope ! ximagesink
32 * ]|
33 *
34 */
35 #ifdef HAVE_CONFIG_H
36 #include "config.h"
37 #endif
38
39 #include "gstsynaescope.h"
40
41 #if G_BYTE_ORDER == G_BIG_ENDIAN
42 #define RGB_ORDER "xRGB"
43 #else
44 #define RGB_ORDER "BGRx"
45 #endif
46
47 static GstStaticPadTemplate gst_synae_scope_src_template =
48 GST_STATIC_PAD_TEMPLATE ("src",
49 GST_PAD_SRC,
50 GST_PAD_ALWAYS,
51 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (RGB_ORDER))
52 );
53
54 static GstStaticPadTemplate gst_synae_scope_sink_template =
55 GST_STATIC_PAD_TEMPLATE ("sink",
56 GST_PAD_SINK,
57 GST_PAD_ALWAYS,
58 GST_STATIC_CAPS ("audio/x-raw, "
59 "format = (string) " GST_AUDIO_NE (S16) ", "
60 "layout = (string) interleaved, "
61 "rate = (int) [ 8000, 96000 ], "
62 "channels = (int) 2, " "channel-mask = (bitmask) 0x3")
63 );
64
65
66 GST_DEBUG_CATEGORY_STATIC (synae_scope_debug);
67 #define GST_CAT_DEFAULT synae_scope_debug
68
69 static void gst_synae_scope_finalize (GObject * object);
70
71 static gboolean gst_synae_scope_setup (GstAudioVisualizer * scope);
72 static gboolean gst_synae_scope_render (GstAudioVisualizer * scope,
73 GstBuffer * audio, GstVideoFrame * video);
74
75
76 G_DEFINE_TYPE_WITH_CODE (GstSynaeScope, gst_synae_scope,
77 GST_TYPE_AUDIO_VISUALIZER, GST_DEBUG_CATEGORY_INIT (synae_scope_debug,
78 "synaescope", 0, "synaescope"););
79 GST_ELEMENT_REGISTER_DEFINE (synaescope, "synaescope", GST_RANK_NONE,
80 GST_TYPE_SYNAE_SCOPE);
81
82 static void
gst_synae_scope_class_init(GstSynaeScopeClass * g_class)83 gst_synae_scope_class_init (GstSynaeScopeClass * g_class)
84 {
85 GObjectClass *gobject_class = (GObjectClass *) g_class;
86 GstElementClass *element_class = (GstElementClass *) g_class;
87 GstAudioVisualizerClass *scope_class = (GstAudioVisualizerClass *) g_class;
88
89 gobject_class->finalize = gst_synae_scope_finalize;
90
91 gst_element_class_set_static_metadata (element_class, "Synaescope",
92 "Visualization",
93 "Creates video visualizations of audio input, using stereo and pitch information",
94 "Stefan Kost <ensonic@users.sf.net>");
95
96 gst_element_class_add_static_pad_template (element_class,
97 &gst_synae_scope_src_template);
98 gst_element_class_add_static_pad_template (element_class,
99 &gst_synae_scope_sink_template);
100
101 scope_class->setup = GST_DEBUG_FUNCPTR (gst_synae_scope_setup);
102 scope_class->render = GST_DEBUG_FUNCPTR (gst_synae_scope_render);
103 }
104
105 static void
gst_synae_scope_init(GstSynaeScope * scope)106 gst_synae_scope_init (GstSynaeScope * scope)
107 {
108 guint32 *colors = scope->colors;
109 guint *shade = scope->shade;
110 guint i, r, g, b;
111
112 #define BOUND(x) ((x) > 255 ? 255 : (x))
113 #define PEAKIFY(x) BOUND((x) - (x)*(255-(x))/255/2)
114
115 for (i = 0; i < 256; i++) {
116 r = PEAKIFY ((i & 15 * 16));
117 g = PEAKIFY ((i & 15) * 16 + (i & 15 * 16) / 4);
118 b = PEAKIFY ((i & 15) * 16);
119
120 colors[i] = (r << 16) | (g << 8) | b;
121 }
122 #undef BOUND
123 #undef PEAKIFY
124
125 for (i = 0; i < 256; i++)
126 shade[i] = i * 200 >> 8;
127 }
128
129 static void
gst_synae_scope_finalize(GObject * object)130 gst_synae_scope_finalize (GObject * object)
131 {
132 GstSynaeScope *scope = GST_SYNAE_SCOPE (object);
133
134 if (scope->fft_ctx) {
135 gst_fft_s16_free (scope->fft_ctx);
136 scope->fft_ctx = NULL;
137 }
138 if (scope->freq_data_l) {
139 g_free (scope->freq_data_l);
140 scope->freq_data_l = NULL;
141 }
142 if (scope->freq_data_r) {
143 g_free (scope->freq_data_r);
144 scope->freq_data_r = NULL;
145 }
146 if (scope->adata_l) {
147 g_free (scope->adata_l);
148 scope->adata_l = NULL;
149 }
150 if (scope->adata_r) {
151 g_free (scope->adata_r);
152 scope->adata_r = NULL;
153 }
154
155 G_OBJECT_CLASS (gst_synae_scope_parent_class)->finalize (object);
156 }
157
158 static gboolean
gst_synae_scope_setup(GstAudioVisualizer * bscope)159 gst_synae_scope_setup (GstAudioVisualizer * bscope)
160 {
161 GstSynaeScope *scope = GST_SYNAE_SCOPE (bscope);
162 guint num_freq = GST_VIDEO_INFO_HEIGHT (&bscope->vinfo) + 1;
163
164 if (scope->fft_ctx)
165 gst_fft_s16_free (scope->fft_ctx);
166 g_free (scope->freq_data_l);
167 g_free (scope->freq_data_r);
168 g_free (scope->adata_l);
169 g_free (scope->adata_r);
170
171 /* FIXME: we could have horizontal or vertical layout */
172
173 /* we'd need this amount of samples per render() call */
174 bscope->req_spf = num_freq * 2 - 2;
175 scope->fft_ctx = gst_fft_s16_new (bscope->req_spf, FALSE);
176 scope->freq_data_l = g_new (GstFFTS16Complex, num_freq);
177 scope->freq_data_r = g_new (GstFFTS16Complex, num_freq);
178
179 scope->adata_l = g_new (gint16, bscope->req_spf);
180 scope->adata_r = g_new (gint16, bscope->req_spf);
181
182 return TRUE;
183 }
184
185 static inline void
add_pixel(guint32 * _p,guint32 _c)186 add_pixel (guint32 * _p, guint32 _c)
187 {
188 guint8 *p = (guint8 *) _p;
189 guint8 *c = (guint8 *) & _c;
190
191 if (p[0] < 255 - c[0])
192 p[0] += c[0];
193 else
194 p[0] = 255;
195 if (p[1] < 255 - c[1])
196 p[1] += c[1];
197 else
198 p[1] = 255;
199 if (p[2] < 255 - c[2])
200 p[2] += c[2];
201 else
202 p[2] = 255;
203 if (p[3] < 255 - c[3])
204 p[3] += c[3];
205 else
206 p[3] = 255;
207 }
208
209 static gboolean
gst_synae_scope_render(GstAudioVisualizer * bscope,GstBuffer * audio,GstVideoFrame * video)210 gst_synae_scope_render (GstAudioVisualizer * bscope, GstBuffer * audio,
211 GstVideoFrame * video)
212 {
213 GstSynaeScope *scope = GST_SYNAE_SCOPE (bscope);
214 GstMapInfo amap;
215 guint32 *vdata;
216 gint16 *adata;
217 gint16 *adata_l = scope->adata_l;
218 gint16 *adata_r = scope->adata_r;
219 GstFFTS16Complex *fdata_l = scope->freq_data_l;
220 GstFFTS16Complex *fdata_r = scope->freq_data_r;
221 gint x, y;
222 guint off;
223 guint w = GST_VIDEO_INFO_WIDTH (&bscope->vinfo);
224 guint h = GST_VIDEO_INFO_HEIGHT (&bscope->vinfo);
225 guint32 *colors = scope->colors, c;
226 guint *shade = scope->shade;
227 //guint w2 = w /2;
228 guint ch = GST_AUDIO_INFO_CHANNELS (&bscope->ainfo);
229 guint num_samples;
230 gint i, j, b;
231 gint br, br1, br2;
232 gint clarity;
233 gdouble fc, r, l, rr, ll;
234 gdouble frl, fil, frr, fir;
235 const guint sl = 30;
236
237 gst_buffer_map (audio, &amap, GST_MAP_READ);
238
239 vdata = (guint32 *) GST_VIDEO_FRAME_PLANE_DATA (video, 0);
240 adata = (gint16 *) amap.data;
241
242 num_samples = amap.size / (ch * sizeof (gint16));
243
244 /* deinterleave */
245 for (i = 0, j = 0; i < num_samples; i++) {
246 adata_l[i] = adata[j++];
247 adata_r[i] = adata[j++];
248 }
249
250 /* run fft */
251 /*gst_fft_s16_window (scope->fft_ctx, adata_l, GST_FFT_WINDOW_HAMMING); */
252 gst_fft_s16_fft (scope->fft_ctx, adata_l, fdata_l);
253 /*gst_fft_s16_window (scope->fft_ctx, adata_r, GST_FFT_WINDOW_HAMMING); */
254 gst_fft_s16_fft (scope->fft_ctx, adata_r, fdata_r);
255
256 /* draw stars */
257 for (y = 0; y < h; y++) {
258 b = h - y;
259 frl = (gdouble) fdata_l[b].r;
260 fil = (gdouble) fdata_l[b].i;
261 frr = (gdouble) fdata_r[b].r;
262 fir = (gdouble) fdata_r[b].i;
263
264 ll = (frl + fil) * (frl + fil) + (frr - fir) * (frr - fir);
265 l = sqrt (ll);
266 rr = (frl - fil) * (frl - fil) + (frr + fir) * (frr + fir);
267 r = sqrt (rr);
268 /* out-of-phase'ness for this frequency component */
269 clarity = (gint) (
270 ((frl + fil) * (frl - fil) + (frr + fir) * (frr - fir)) /
271 (ll + rr) * 256);
272 fc = r + l;
273
274 x = (guint) (r * w / fc);
275 /* the brightness scaling factor was picked by experimenting */
276 br = b * fc * 0.01;
277
278 br1 = br * (clarity + 128) >> 8;
279 br2 = br * (128 - clarity) >> 8;
280 br1 = CLAMP (br1, 0, 255);
281 br2 = CLAMP (br2, 0, 255);
282
283 GST_DEBUG ("y %3d fc %10.6f clarity %d br %d br1 %d br2 %d", y, fc, clarity,
284 br, br1, br2);
285
286 /* draw a star */
287 off = (y * w) + x;
288 c = colors[(br1 >> 4) | (br2 & 0xf0)];
289 add_pixel (&vdata[off], c);
290 if ((x > (sl - 1)) && (x < (w - sl)) && (y > (sl - 1)) && (y < (h - sl))) {
291 for (i = 1; br1 || br2; i++, br1 = shade[br1], br2 = shade[br2]) {
292 c = colors[(br1 >> 4) + (br2 & 0xf0)];
293 add_pixel (&vdata[off - i], c);
294 add_pixel (&vdata[off + i], c);
295 add_pixel (&vdata[off - i * w], c);
296 add_pixel (&vdata[off + i * w], c);
297 }
298 } else {
299 for (i = 1; br1 || br2; i++, br1 = shade[br1], br2 = shade[br2]) {
300 c = colors[(br1 >> 4) | (br2 & 0xf0)];
301 if (x - i > 0)
302 add_pixel (&vdata[off - i], c);
303 if (x + i < (w - 1))
304 add_pixel (&vdata[off + i], c);
305 if (y - i > 0)
306 add_pixel (&vdata[off - i * w], c);
307 if (y + i < (h - 1))
308 add_pixel (&vdata[off + i * w], c);
309 }
310 }
311 }
312 gst_buffer_unmap (audio, &amap);
313
314 return TRUE;
315 }
316