1 /*
2 * Copyright (c) 2015 Paul B Mahol
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 /**
22 * @file
23 * audio to video multimedia aphasemeter filter
24 */
25
26 #include "libavutil/channel_layout.h"
27 #include "libavutil/intreadwrite.h"
28 #include "libavutil/opt.h"
29 #include "libavutil/parseutils.h"
30 #include "libavutil/timestamp.h"
31 #include "avfilter.h"
32 #include "formats.h"
33 #include "audio.h"
34 #include "video.h"
35 #include "internal.h"
36 #include "float.h"
37
38 typedef struct AudioPhaseMeterContext {
39 const AVClass *class;
40 AVFrame *out;
41 int do_video;
42 int do_phasing_detection;
43 int w, h;
44 AVRational frame_rate;
45 int contrast[4];
46 uint8_t *mpc_str;
47 uint8_t mpc[4];
48 int draw_median_phase;
49 int is_mono;
50 int is_out_phase;
51 int start_mono_presence;
52 int start_out_phase_presence;
53 float tolerance;
54 float angle;
55 float phase;
56 AVRational time_base;
57 int64_t duration;
58 int64_t frame_end;
59 int64_t mono_idx[2];
60 int64_t out_phase_idx[2];
61 } AudioPhaseMeterContext;
62
63 #define MAX_DURATION (24*60*60*1000000LL)
64 #define OFFSET(x) offsetof(AudioPhaseMeterContext, x)
65 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
66 #define get_duration(index) (index[1] - index[0])
67
68 static const AVOption aphasemeter_options[] = {
69 { "rate", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str="25"}, 0, INT_MAX, FLAGS },
70 { "r", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str="25"}, 0, INT_MAX, FLAGS },
71 { "size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str="800x400"}, 0, 0, FLAGS },
72 { "s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str="800x400"}, 0, 0, FLAGS },
73 { "rc", "set red contrast", OFFSET(contrast[0]), AV_OPT_TYPE_INT, {.i64=2}, 0, 255, FLAGS },
74 { "gc", "set green contrast", OFFSET(contrast[1]), AV_OPT_TYPE_INT, {.i64=7}, 0, 255, FLAGS },
75 { "bc", "set blue contrast", OFFSET(contrast[2]), AV_OPT_TYPE_INT, {.i64=1}, 0, 255, FLAGS },
76 { "mpc", "set median phase color", OFFSET(mpc_str), AV_OPT_TYPE_STRING, {.str = "none"}, 0, 0, FLAGS },
77 { "video", "set video output", OFFSET(do_video), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, FLAGS },
78 { "phasing", "set mono and out-of-phase detection output", OFFSET(do_phasing_detection), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, FLAGS },
79 { "tolerance", "set phase tolerance for mono detection", OFFSET(tolerance), AV_OPT_TYPE_FLOAT, {.dbl = 0.}, 0, 1, FLAGS },
80 { "t", "set phase tolerance for mono detection", OFFSET(tolerance), AV_OPT_TYPE_FLOAT, {.dbl = 0.}, 0, 1, FLAGS },
81 { "angle", "set angle threshold for out-of-phase detection", OFFSET(angle), AV_OPT_TYPE_FLOAT, {.dbl = 170.}, 90, 180, FLAGS },
82 { "a", "set angle threshold for out-of-phase detection", OFFSET(angle), AV_OPT_TYPE_FLOAT, {.dbl = 170.}, 90, 180, FLAGS },
83 { "duration", "set minimum mono or out-of-phase duration in seconds", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64=2000000}, 0, MAX_DURATION, FLAGS },
84 { "d", "set minimum mono or out-of-phase duration in seconds", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64=2000000}, 0, MAX_DURATION, FLAGS },
85 { NULL }
86 };
87
88 AVFILTER_DEFINE_CLASS(aphasemeter);
89
query_formats(AVFilterContext * ctx)90 static int query_formats(AVFilterContext *ctx)
91 {
92 AudioPhaseMeterContext *s = ctx->priv;
93 AVFilterFormats *formats = NULL;
94 AVFilterChannelLayouts *layout = NULL;
95 AVFilterLink *inlink = ctx->inputs[0];
96 AVFilterLink *outlink = ctx->outputs[0];
97 static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_NONE };
98 static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_RGBA, AV_PIX_FMT_NONE };
99 int ret;
100
101 formats = ff_make_format_list(sample_fmts);
102 if ((ret = ff_formats_ref (formats, &inlink->outcfg.formats )) < 0 ||
103 (ret = ff_formats_ref (formats, &outlink->incfg.formats )) < 0 ||
104 (ret = ff_add_channel_layout (&layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO )) < 0 ||
105 (ret = ff_channel_layouts_ref (layout , &inlink->outcfg.channel_layouts)) < 0 ||
106 (ret = ff_channel_layouts_ref (layout , &outlink->incfg.channel_layouts)) < 0)
107 return ret;
108
109 formats = ff_all_samplerates();
110 if ((ret = ff_formats_ref(formats, &inlink->outcfg.samplerates)) < 0 ||
111 (ret = ff_formats_ref(formats, &outlink->incfg.samplerates)) < 0)
112 return ret;
113
114 if (s->do_video) {
115 AVFilterLink *outlink = ctx->outputs[1];
116
117 formats = ff_make_format_list(pix_fmts);
118 if ((ret = ff_formats_ref(formats, &outlink->incfg.formats)) < 0)
119 return ret;
120 }
121
122 return 0;
123 }
124
config_input(AVFilterLink * inlink)125 static int config_input(AVFilterLink *inlink)
126 {
127 AVFilterContext *ctx = inlink->dst;
128 AudioPhaseMeterContext *s = ctx->priv;
129 int nb_samples;
130 s->duration = av_rescale(s->duration, inlink->sample_rate, AV_TIME_BASE);
131
132 if (s->do_video) {
133 nb_samples = FFMAX(1, av_rescale(inlink->sample_rate, s->frame_rate.den, s->frame_rate.num));
134 inlink->min_samples =
135 inlink->max_samples = nb_samples;
136 }
137
138 return 0;
139 }
140
config_video_output(AVFilterLink * outlink)141 static int config_video_output(AVFilterLink *outlink)
142 {
143 AVFilterContext *ctx = outlink->src;
144 AudioPhaseMeterContext *s = ctx->priv;
145
146 outlink->w = s->w;
147 outlink->h = s->h;
148 outlink->sample_aspect_ratio = (AVRational){1,1};
149 outlink->frame_rate = s->frame_rate;
150
151 if (!strcmp(s->mpc_str, "none"))
152 s->draw_median_phase = 0;
153 else if (av_parse_color(s->mpc, s->mpc_str, -1, ctx) >= 0)
154 s->draw_median_phase = 1;
155 else
156 return AVERROR(EINVAL);
157
158 return 0;
159 }
160
get_x(float phase,int w)161 static inline int get_x(float phase, int w)
162 {
163 return (phase + 1.) / 2. * (w - 1);
164 }
165
add_metadata(AVFrame * insamples,const char * key,char * value)166 static inline void add_metadata(AVFrame *insamples, const char *key, char *value)
167 {
168 char buf[128];
169
170 snprintf(buf, sizeof(buf), "lavfi.aphasemeter.%s", key);
171 av_dict_set(&insamples->metadata, buf, value, 0);
172 }
173
update_mono_detection(AudioPhaseMeterContext * s,AVFrame * insamples,int mono_measurement)174 static inline void update_mono_detection(AudioPhaseMeterContext *s, AVFrame *insamples, int mono_measurement)
175 {
176 int64_t mono_duration;
177 if (!s->is_mono && mono_measurement) {
178 s->is_mono = 1;
179 s->start_mono_presence = 1;
180 s->mono_idx[0] = insamples->pts;
181 }
182 if (s->is_mono && mono_measurement && s->start_mono_presence) {
183 s->mono_idx[1] = s->frame_end;
184 mono_duration = get_duration(s->mono_idx);
185 if (mono_duration >= s->duration) {
186 add_metadata(insamples, "mono_start", av_ts2timestr(s->mono_idx[0], &s->time_base));
187 av_log(s, AV_LOG_INFO, "mono_start: %s\n", av_ts2timestr(s->mono_idx[0], &s->time_base));
188 s->start_mono_presence = 0;
189 }
190 }
191 if (s->is_mono && !mono_measurement) {
192 s->mono_idx[1] = insamples ? insamples->pts : s->frame_end;
193 mono_duration = get_duration(s->mono_idx);
194 if (mono_duration >= s->duration) {
195 if (insamples) {
196 add_metadata(insamples, "mono_end", av_ts2timestr(s->mono_idx[1], &s->time_base));
197 add_metadata(insamples, "mono_duration", av_ts2timestr(mono_duration, &s->time_base));
198 }
199 av_log(s, AV_LOG_INFO, "mono_end: %s | mono_duration: %s\n", av_ts2timestr(s->mono_idx[1], &s->time_base), av_ts2timestr(mono_duration, &s->time_base));
200 }
201 s->is_mono = 0;
202 }
203 }
204
update_out_phase_detection(AudioPhaseMeterContext * s,AVFrame * insamples,int out_phase_measurement)205 static inline void update_out_phase_detection(AudioPhaseMeterContext *s, AVFrame *insamples, int out_phase_measurement)
206 {
207 int64_t out_phase_duration;
208 if (!s->is_out_phase && out_phase_measurement) {
209 s->is_out_phase = 1;
210 s->start_out_phase_presence = 1;
211 s->out_phase_idx[0] = insamples->pts;
212 }
213 if (s->is_out_phase && out_phase_measurement && s->start_out_phase_presence) {
214 s->out_phase_idx[1] = s->frame_end;
215 out_phase_duration = get_duration(s->out_phase_idx);
216 if (out_phase_duration >= s->duration) {
217 add_metadata(insamples, "out_phase_start", av_ts2timestr(s->out_phase_idx[0], &s->time_base));
218 av_log(s, AV_LOG_INFO, "out_phase_start: %s\n", av_ts2timestr(s->out_phase_idx[0], &s->time_base));
219 s->start_out_phase_presence = 0;
220 }
221 }
222 if (s->is_out_phase && !out_phase_measurement) {
223 s->out_phase_idx[1] = insamples ? insamples->pts : s->frame_end;
224 out_phase_duration = get_duration(s->out_phase_idx);
225 if (out_phase_duration >= s->duration) {
226 if (insamples) {
227 add_metadata(insamples, "out_phase_end", av_ts2timestr(s->out_phase_idx[1], &s->time_base));
228 add_metadata(insamples, "out_phase_duration", av_ts2timestr(out_phase_duration, &s->time_base));
229 }
230 av_log(s, AV_LOG_INFO, "out_phase_end: %s | out_phase_duration: %s\n", av_ts2timestr(s->out_phase_idx[1], &s->time_base), av_ts2timestr(out_phase_duration, &s->time_base));
231 }
232 s->is_out_phase = 0;
233 }
234 }
235
filter_frame(AVFilterLink * inlink,AVFrame * in)236 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
237 {
238 AVFilterContext *ctx = inlink->dst;
239 AudioPhaseMeterContext *s = ctx->priv;
240 AVFilterLink *outlink = s->do_video ? ctx->outputs[1] : NULL;
241 AVFilterLink *aoutlink = ctx->outputs[0];
242 AVDictionary **metadata;
243 const int rc = s->contrast[0];
244 const int gc = s->contrast[1];
245 const int bc = s->contrast[2];
246 float fphase = 0;
247 AVFrame *out;
248 uint8_t *dst;
249 int i;
250 int mono_measurement;
251 int out_phase_measurement;
252 float tolerance = 1.0f - s->tolerance;
253 float angle = cosf(s->angle/180.0f*M_PI);
254
255 if (s->do_video && (!s->out || s->out->width != outlink->w ||
256 s->out->height != outlink->h)) {
257 av_frame_free(&s->out);
258 s->out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
259 if (!s->out) {
260 av_frame_free(&in);
261 return AVERROR(ENOMEM);
262 }
263
264 out = s->out;
265 for (i = 0; i < outlink->h; i++)
266 memset(out->data[0] + i * out->linesize[0], 0, outlink->w * 4);
267 } else if (s->do_video) {
268 out = s->out;
269 av_frame_make_writable(s->out);
270 for (i = outlink->h - 1; i >= 10; i--)
271 memmove(out->data[0] + (i ) * out->linesize[0],
272 out->data[0] + (i-1) * out->linesize[0],
273 outlink->w * 4);
274 for (i = 0; i < outlink->w; i++)
275 AV_WL32(out->data[0] + i * 4, 0);
276 }
277
278 for (i = 0; i < in->nb_samples; i++) {
279 const float *src = (float *)in->data[0] + i * 2;
280 const float f = src[0] * src[1] / (src[0]*src[0] + src[1] * src[1]) * 2;
281 const float phase = isnan(f) ? 1 : f;
282 const int x = get_x(phase, s->w);
283
284 if (s->do_video) {
285 dst = out->data[0] + x * 4;
286 dst[0] = FFMIN(255, dst[0] + rc);
287 dst[1] = FFMIN(255, dst[1] + gc);
288 dst[2] = FFMIN(255, dst[2] + bc);
289 dst[3] = 255;
290 }
291 fphase += phase;
292 }
293 fphase /= in->nb_samples;
294 s->phase = fphase;
295
296 if (s->do_video) {
297 if (s->draw_median_phase) {
298 dst = out->data[0] + get_x(fphase, s->w) * 4;
299 AV_WL32(dst, AV_RL32(s->mpc));
300 }
301
302 for (i = 1; i < 10 && i < outlink->h; i++)
303 memcpy(out->data[0] + i * out->linesize[0], out->data[0], outlink->w * 4);
304 }
305
306 metadata = &in->metadata;
307 if (metadata) {
308 uint8_t value[128];
309
310 snprintf(value, sizeof(value), "%f", fphase);
311 add_metadata(in, "phase", value);
312 }
313
314 if (s->do_phasing_detection) {
315 s->time_base = inlink->time_base;
316 s->frame_end = in->pts + av_rescale_q(in->nb_samples,
317 (AVRational){ 1, in->sample_rate }, inlink->time_base);
318
319 mono_measurement = (tolerance - fphase) < FLT_EPSILON;
320 out_phase_measurement = (angle - fphase) > FLT_EPSILON;
321
322 update_mono_detection(s, in, mono_measurement);
323 update_out_phase_detection(s, in, out_phase_measurement);
324 }
325
326 if (s->do_video) {
327 AVFrame *clone;
328
329 s->out->pts = in->pts;
330 clone = av_frame_clone(s->out);
331 if (!clone)
332 return AVERROR(ENOMEM);
333 ff_filter_frame(outlink, clone);
334 }
335 return ff_filter_frame(aoutlink, in);
336 }
337
uninit(AVFilterContext * ctx)338 static av_cold void uninit(AVFilterContext *ctx)
339 {
340 AudioPhaseMeterContext *s = ctx->priv;
341
342 if (s->do_phasing_detection) {
343 update_mono_detection(s, NULL, 0);
344 update_out_phase_detection(s, NULL, 0);
345 }
346 av_frame_free(&s->out);
347 }
348
init(AVFilterContext * ctx)349 static av_cold int init(AVFilterContext *ctx)
350 {
351 AudioPhaseMeterContext *s = ctx->priv;
352 AVFilterPad pad;
353 int ret;
354
355 pad = (AVFilterPad){
356 .name = "out0",
357 .type = AVMEDIA_TYPE_AUDIO,
358 };
359 ret = ff_append_outpad(ctx, &pad);
360 if (ret < 0)
361 return ret;
362
363 if (s->do_video) {
364 pad = (AVFilterPad){
365 .name = "out1",
366 .type = AVMEDIA_TYPE_VIDEO,
367 .config_props = config_video_output,
368 };
369 ret = ff_append_outpad(ctx, &pad);
370 if (ret < 0)
371 return ret;
372 }
373
374 return 0;
375 }
376
377 static const AVFilterPad inputs[] = {
378 {
379 .name = "default",
380 .type = AVMEDIA_TYPE_AUDIO,
381 .config_props = config_input,
382 .filter_frame = filter_frame,
383 },
384 };
385
386 const AVFilter ff_avf_aphasemeter = {
387 .name = "aphasemeter",
388 .description = NULL_IF_CONFIG_SMALL("Convert input audio to phase meter video output."),
389 .init = init,
390 .uninit = uninit,
391 .priv_size = sizeof(AudioPhaseMeterContext),
392 FILTER_INPUTS(inputs),
393 .outputs = NULL,
394 FILTER_QUERY_FUNC(query_formats),
395 .priv_class = &aphasemeter_class,
396 .flags = AVFILTER_FLAG_DYNAMIC_OUTPUTS,
397 };
398