1 /*
2 * Copyright (c) 2013 Nicolas George
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public License
8 * as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with FFmpeg; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #include "libavutil/avassert.h"
22 #include "libavutil/opt.h"
23 #include "avfilter.h"
24 #include "filters.h"
25 #include "framesync.h"
26 #include "internal.h"
27
28 #define OFFSET(member) offsetof(FFFrameSync, member)
29 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM
30
framesync_name(void * ptr)31 static const char *framesync_name(void *ptr)
32 {
33 return "framesync";
34 }
35
36 static const AVOption framesync_options[] = {
37 { "eof_action", "Action to take when encountering EOF from secondary input ",
38 OFFSET(opt_eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT },
39 EOF_ACTION_REPEAT, EOF_ACTION_PASS, .flags = FLAGS, "eof_action" },
40 { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, "eof_action" },
41 { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, "eof_action" },
42 { "pass", "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS, "eof_action" },
43 { "shortest", "force termination when the shortest input terminates", OFFSET(opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
44 { "repeatlast", "extend last frame of secondary streams beyond EOF", OFFSET(opt_repeatlast), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, FLAGS },
45 { NULL }
46 };
47 static const AVClass framesync_class = {
48 .version = LIBAVUTIL_VERSION_INT,
49 .class_name = "framesync",
50 .item_name = framesync_name,
51 .category = AV_CLASS_CATEGORY_FILTER,
52 .option = framesync_options,
53 .parent_log_context_offset = OFFSET(parent),
54 };
55
ff_framesync_child_class_iterate(void ** iter)56 const AVClass *ff_framesync_child_class_iterate(void **iter)
57 {
58 const AVClass *c = *iter ? NULL : &framesync_class;
59 *iter = (void *)(uintptr_t)c;
60 return c;
61 }
62
63 enum {
64 STATE_BOF,
65 STATE_RUN,
66 STATE_EOF,
67 };
68
69 static int consume_from_fifos(FFFrameSync *fs);
70
ff_framesync_preinit(FFFrameSync * fs)71 void ff_framesync_preinit(FFFrameSync *fs)
72 {
73 if (fs->class)
74 return;
75 fs->class = &framesync_class;
76 av_opt_set_defaults(fs);
77 }
78
ff_framesync_init(FFFrameSync * fs,AVFilterContext * parent,unsigned nb_in)79 int ff_framesync_init(FFFrameSync *fs, AVFilterContext *parent, unsigned nb_in)
80 {
81 /* For filters with several outputs, we will not be able to assume which
82 output is relevant for ff_outlink_frame_wanted() and
83 ff_outlink_set_status(). To be designed when needed. */
84 av_assert0(parent->nb_outputs == 1);
85
86 ff_framesync_preinit(fs);
87 fs->parent = parent;
88 fs->nb_in = nb_in;
89
90 fs->in = av_calloc(nb_in, sizeof(*fs->in));
91 if (!fs->in)
92 return AVERROR(ENOMEM);
93 return 0;
94 }
95
framesync_eof(FFFrameSync * fs)96 static void framesync_eof(FFFrameSync *fs)
97 {
98 fs->eof = 1;
99 fs->frame_ready = 0;
100 ff_outlink_set_status(fs->parent->outputs[0], AVERROR_EOF, AV_NOPTS_VALUE);
101 }
102
framesync_sync_level_update(FFFrameSync * fs)103 static void framesync_sync_level_update(FFFrameSync *fs)
104 {
105 unsigned i, level = 0;
106
107 for (i = 0; i < fs->nb_in; i++)
108 if (fs->in[i].state != STATE_EOF)
109 level = FFMAX(level, fs->in[i].sync);
110 av_assert0(level <= fs->sync_level);
111 if (level < fs->sync_level)
112 av_log(fs, AV_LOG_VERBOSE, "Sync level %u\n", level);
113 if (level)
114 fs->sync_level = level;
115 else
116 framesync_eof(fs);
117 }
118
ff_framesync_configure(FFFrameSync * fs)119 int ff_framesync_configure(FFFrameSync *fs)
120 {
121 unsigned i;
122
123 if (!fs->opt_repeatlast || fs->opt_eof_action == EOF_ACTION_PASS) {
124 fs->opt_repeatlast = 0;
125 fs->opt_eof_action = EOF_ACTION_PASS;
126 }
127 if (fs->opt_shortest || fs->opt_eof_action == EOF_ACTION_ENDALL) {
128 fs->opt_shortest = 1;
129 fs->opt_eof_action = EOF_ACTION_ENDALL;
130 }
131 if (!fs->opt_repeatlast) {
132 for (i = 1; i < fs->nb_in; i++) {
133 fs->in[i].after = EXT_NULL;
134 fs->in[i].sync = 0;
135 }
136 }
137 if (fs->opt_shortest) {
138 for (i = 0; i < fs->nb_in; i++)
139 fs->in[i].after = EXT_STOP;
140 }
141
142 if (!fs->time_base.num) {
143 for (i = 0; i < fs->nb_in; i++) {
144 if (fs->in[i].sync) {
145 if (fs->time_base.num) {
146 fs->time_base = av_gcd_q(fs->time_base, fs->in[i].time_base,
147 AV_TIME_BASE / 2, AV_TIME_BASE_Q);
148 } else {
149 fs->time_base = fs->in[i].time_base;
150 }
151 }
152 }
153 if (!fs->time_base.num) {
154 av_log(fs, AV_LOG_ERROR, "Impossible to set time base\n");
155 return AVERROR(EINVAL);
156 }
157 av_log(fs, AV_LOG_VERBOSE, "Selected %d/%d time base\n",
158 fs->time_base.num, fs->time_base.den);
159 }
160
161 for (i = 0; i < fs->nb_in; i++)
162 fs->in[i].pts = fs->in[i].pts_next = AV_NOPTS_VALUE;
163 fs->sync_level = UINT_MAX;
164 framesync_sync_level_update(fs);
165
166 return 0;
167 }
168
framesync_advance(FFFrameSync * fs)169 static int framesync_advance(FFFrameSync *fs)
170 {
171 unsigned i;
172 int64_t pts;
173 int ret;
174
175 while (!(fs->frame_ready || fs->eof)) {
176 ret = consume_from_fifos(fs);
177 if (ret <= 0)
178 return ret;
179
180 pts = INT64_MAX;
181 for (i = 0; i < fs->nb_in; i++)
182 if (fs->in[i].have_next && fs->in[i].pts_next < pts)
183 pts = fs->in[i].pts_next;
184 if (pts == INT64_MAX) {
185 framesync_eof(fs);
186 break;
187 }
188 for (i = 0; i < fs->nb_in; i++) {
189 if (fs->in[i].pts_next == pts ||
190 (fs->in[i].before == EXT_INFINITY &&
191 fs->in[i].state == STATE_BOF)) {
192 av_frame_free(&fs->in[i].frame);
193 fs->in[i].frame = fs->in[i].frame_next;
194 fs->in[i].pts = fs->in[i].pts_next;
195 fs->in[i].frame_next = NULL;
196 fs->in[i].pts_next = AV_NOPTS_VALUE;
197 fs->in[i].have_next = 0;
198 fs->in[i].state = fs->in[i].frame ? STATE_RUN : STATE_EOF;
199 if (fs->in[i].sync == fs->sync_level && fs->in[i].frame)
200 fs->frame_ready = 1;
201 if (fs->in[i].state == STATE_EOF &&
202 fs->in[i].after == EXT_STOP)
203 framesync_eof(fs);
204 }
205 }
206 if (fs->frame_ready)
207 for (i = 0; i < fs->nb_in; i++)
208 if ((fs->in[i].state == STATE_BOF &&
209 fs->in[i].before == EXT_STOP))
210 fs->frame_ready = 0;
211 fs->pts = pts;
212 }
213 return 0;
214 }
215
framesync_pts_extrapolate(FFFrameSync * fs,unsigned in,int64_t pts)216 static int64_t framesync_pts_extrapolate(FFFrameSync *fs, unsigned in,
217 int64_t pts)
218 {
219 /* Possible enhancement: use the link's frame rate */
220 return pts + 1;
221 }
222
framesync_inject_frame(FFFrameSync * fs,unsigned in,AVFrame * frame)223 static void framesync_inject_frame(FFFrameSync *fs, unsigned in, AVFrame *frame)
224 {
225 int64_t pts;
226
227 av_assert0(!fs->in[in].have_next);
228 av_assert0(frame);
229 pts = av_rescale_q(frame->pts, fs->in[in].time_base, fs->time_base);
230 frame->pts = pts;
231 fs->in[in].frame_next = frame;
232 fs->in[in].pts_next = pts;
233 fs->in[in].have_next = 1;
234 }
235
framesync_inject_status(FFFrameSync * fs,unsigned in,int status,int64_t pts)236 static void framesync_inject_status(FFFrameSync *fs, unsigned in, int status, int64_t pts)
237 {
238 av_assert0(!fs->in[in].have_next);
239 pts = fs->in[in].state != STATE_RUN || fs->in[in].after == EXT_INFINITY
240 ? INT64_MAX : framesync_pts_extrapolate(fs, in, fs->in[in].pts);
241 fs->in[in].sync = 0;
242 framesync_sync_level_update(fs);
243 fs->in[in].frame_next = NULL;
244 fs->in[in].pts_next = pts;
245 fs->in[in].have_next = 1;
246 }
247
ff_framesync_get_frame(FFFrameSync * fs,unsigned in,AVFrame ** rframe,unsigned get)248 int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe,
249 unsigned get)
250 {
251 AVFrame *frame;
252 unsigned need_copy = 0, i;
253 int64_t pts_next;
254 int ret;
255
256 if (!fs->in[in].frame) {
257 *rframe = NULL;
258 return 0;
259 }
260 frame = fs->in[in].frame;
261 if (get) {
262 /* Find out if we need to copy the frame: is there another sync
263 stream, and do we know if its current frame will outlast this one? */
264 pts_next = fs->in[in].have_next ? fs->in[in].pts_next : INT64_MAX;
265 for (i = 0; i < fs->nb_in && !need_copy; i++)
266 if (i != in && fs->in[i].sync &&
267 (!fs->in[i].have_next || fs->in[i].pts_next < pts_next))
268 need_copy = 1;
269 if (need_copy) {
270 if (!(frame = av_frame_clone(frame)))
271 return AVERROR(ENOMEM);
272 if ((ret = av_frame_make_writable(frame)) < 0) {
273 av_frame_free(&frame);
274 return ret;
275 }
276 } else {
277 fs->in[in].frame = NULL;
278 }
279 fs->frame_ready = 0;
280 }
281 *rframe = frame;
282 return 0;
283 }
284
ff_framesync_uninit(FFFrameSync * fs)285 void ff_framesync_uninit(FFFrameSync *fs)
286 {
287 unsigned i;
288
289 for (i = 0; i < fs->nb_in; i++) {
290 av_frame_free(&fs->in[i].frame);
291 av_frame_free(&fs->in[i].frame_next);
292 }
293
294 av_freep(&fs->in);
295 }
296
consume_from_fifos(FFFrameSync * fs)297 static int consume_from_fifos(FFFrameSync *fs)
298 {
299 AVFilterContext *ctx = fs->parent;
300 AVFrame *frame = NULL;
301 int64_t pts;
302 unsigned i, nb_active, nb_miss;
303 int ret, status;
304
305 nb_active = nb_miss = 0;
306 for (i = 0; i < fs->nb_in; i++) {
307 if (fs->in[i].have_next || fs->in[i].state == STATE_EOF)
308 continue;
309 nb_active++;
310 ret = ff_inlink_consume_frame(ctx->inputs[i], &frame);
311 if (ret < 0)
312 return ret;
313 if (ret) {
314 av_assert0(frame);
315 framesync_inject_frame(fs, i, frame);
316 } else {
317 ret = ff_inlink_acknowledge_status(ctx->inputs[i], &status, &pts);
318 if (ret > 0) {
319 framesync_inject_status(fs, i, status, pts);
320 } else if (!ret) {
321 nb_miss++;
322 }
323 }
324 }
325 if (nb_miss) {
326 if (nb_miss == nb_active && !ff_outlink_frame_wanted(ctx->outputs[0]))
327 return FFERROR_NOT_READY;
328 for (i = 0; i < fs->nb_in; i++)
329 if (!fs->in[i].have_next && fs->in[i].state != STATE_EOF)
330 ff_inlink_request_frame(ctx->inputs[i]);
331 return 0;
332 }
333 return 1;
334 }
335
ff_framesync_activate(FFFrameSync * fs)336 int ff_framesync_activate(FFFrameSync *fs)
337 {
338 int ret;
339
340 ret = framesync_advance(fs);
341 if (ret < 0)
342 return ret;
343 if (fs->eof || !fs->frame_ready)
344 return 0;
345 ret = fs->on_event(fs);
346 if (ret < 0)
347 return ret;
348 fs->frame_ready = 0;
349
350 return 0;
351 }
352
ff_framesync_init_dualinput(FFFrameSync * fs,AVFilterContext * parent)353 int ff_framesync_init_dualinput(FFFrameSync *fs, AVFilterContext *parent)
354 {
355 int ret;
356
357 ret = ff_framesync_init(fs, parent, 2);
358 if (ret < 0)
359 return ret;
360 fs->in[0].time_base = parent->inputs[0]->time_base;
361 fs->in[1].time_base = parent->inputs[1]->time_base;
362 fs->in[0].sync = 2;
363 fs->in[0].before = EXT_STOP;
364 fs->in[0].after = EXT_INFINITY;
365 fs->in[1].sync = 1;
366 fs->in[1].before = EXT_NULL;
367 fs->in[1].after = EXT_INFINITY;
368 return 0;
369 }
370
ff_framesync_dualinput_get(FFFrameSync * fs,AVFrame ** f0,AVFrame ** f1)371 int ff_framesync_dualinput_get(FFFrameSync *fs, AVFrame **f0, AVFrame **f1)
372 {
373 AVFilterContext *ctx = fs->parent;
374 AVFrame *mainpic = NULL, *secondpic = NULL;
375 int ret;
376
377 if ((ret = ff_framesync_get_frame(fs, 0, &mainpic, 1)) < 0 ||
378 (ret = ff_framesync_get_frame(fs, 1, &secondpic, 0)) < 0) {
379 av_frame_free(&mainpic);
380 return ret;
381 }
382 av_assert0(mainpic);
383 mainpic->pts = av_rescale_q(fs->pts, fs->time_base, ctx->outputs[0]->time_base);
384 if (ctx->is_disabled)
385 secondpic = NULL;
386 *f0 = mainpic;
387 *f1 = secondpic;
388 return 0;
389 }
390
ff_framesync_dualinput_get_writable(FFFrameSync * fs,AVFrame ** f0,AVFrame ** f1)391 int ff_framesync_dualinput_get_writable(FFFrameSync *fs, AVFrame **f0, AVFrame **f1)
392 {
393 int ret;
394
395 ret = ff_framesync_dualinput_get(fs, f0, f1);
396 if (ret < 0)
397 return ret;
398 ret = ff_inlink_make_frame_writable(fs->parent->inputs[0], f0);
399 if (ret < 0) {
400 av_frame_free(f0);
401 *f1 = NULL;
402 return ret;
403 }
404 return 0;
405 }
406