1 /*
2 * Copyright (c) 2013 Vittorio Giovara
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 /**
22 * @file
23 * Generate a frame packed video, by combining two views in a single surface.
24 */
25
26 #include <string.h>
27
28 #include "libavutil/common.h"
29 #include "libavutil/imgutils.h"
30 #include "libavutil/opt.h"
31 #include "libavutil/pixdesc.h"
32 #include "libavutil/rational.h"
33 #include "libavutil/stereo3d.h"
34
35 #include "avfilter.h"
36 #include "filters.h"
37 #include "formats.h"
38 #include "internal.h"
39 #include "video.h"
40
41 #define LEFT 0
42 #define RIGHT 1
43
44 typedef struct FramepackContext {
45 const AVClass *class;
46
47 int depth;
48 const AVPixFmtDescriptor *pix_desc; ///< agreed pixel format
49
50 enum AVStereo3DType format; ///< frame pack type output
51
52 AVFrame *input_views[2]; ///< input frames
53 } FramepackContext;
54
55 static const enum AVPixelFormat formats_supported[] = {
56 AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY9,
57 AV_PIX_FMT_GRAY10, AV_PIX_FMT_GRAY12, AV_PIX_FMT_GRAY14,
58 AV_PIX_FMT_GRAY16,
59 AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
60 AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
61 AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,
62 AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P,
63 AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P,
64 AV_PIX_FMT_YUVJ411P,
65 AV_PIX_FMT_YUV420P9, AV_PIX_FMT_YUV422P9, AV_PIX_FMT_YUV444P9,
66 AV_PIX_FMT_YUV420P10, AV_PIX_FMT_YUV422P10, AV_PIX_FMT_YUV444P10,
67 AV_PIX_FMT_YUV440P10,
68 AV_PIX_FMT_YUV444P12, AV_PIX_FMT_YUV422P12, AV_PIX_FMT_YUV420P12,
69 AV_PIX_FMT_YUV440P12,
70 AV_PIX_FMT_YUV444P14, AV_PIX_FMT_YUV422P14, AV_PIX_FMT_YUV420P14,
71 AV_PIX_FMT_YUV420P16, AV_PIX_FMT_YUV422P16, AV_PIX_FMT_YUV444P16,
72 AV_PIX_FMT_GBRP, AV_PIX_FMT_GBRP9, AV_PIX_FMT_GBRP10,
73 AV_PIX_FMT_GBRP12, AV_PIX_FMT_GBRP14, AV_PIX_FMT_GBRP16,
74 AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P,
75 AV_PIX_FMT_YUVA444P9, AV_PIX_FMT_YUVA444P10, AV_PIX_FMT_YUVA444P12, AV_PIX_FMT_YUVA444P16,
76 AV_PIX_FMT_YUVA422P9, AV_PIX_FMT_YUVA422P10, AV_PIX_FMT_YUVA422P12, AV_PIX_FMT_YUVA422P16,
77 AV_PIX_FMT_YUVA420P9, AV_PIX_FMT_YUVA420P10, AV_PIX_FMT_YUVA420P16,
78 AV_PIX_FMT_GBRAP, AV_PIX_FMT_GBRAP10, AV_PIX_FMT_GBRAP12, AV_PIX_FMT_GBRAP16,
79 AV_PIX_FMT_NONE
80 };
81
framepack_uninit(AVFilterContext * ctx)82 static av_cold void framepack_uninit(AVFilterContext *ctx)
83 {
84 FramepackContext *s = ctx->priv;
85
86 // clean any leftover frame
87 av_frame_free(&s->input_views[LEFT]);
88 av_frame_free(&s->input_views[RIGHT]);
89 }
90
config_output(AVFilterLink * outlink)91 static int config_output(AVFilterLink *outlink)
92 {
93 AVFilterContext *ctx = outlink->src;
94 FramepackContext *s = outlink->src->priv;
95
96 int width = ctx->inputs[LEFT]->w;
97 int height = ctx->inputs[LEFT]->h;
98 AVRational time_base = ctx->inputs[LEFT]->time_base;
99 AVRational frame_rate = ctx->inputs[LEFT]->frame_rate;
100
101 // check size and fps match on the other input
102 if (width != ctx->inputs[RIGHT]->w ||
103 height != ctx->inputs[RIGHT]->h) {
104 av_log(ctx, AV_LOG_ERROR,
105 "Left and right sizes differ (%dx%d vs %dx%d).\n",
106 width, height,
107 ctx->inputs[RIGHT]->w, ctx->inputs[RIGHT]->h);
108 return AVERROR_INVALIDDATA;
109 } else if (av_cmp_q(time_base, ctx->inputs[RIGHT]->time_base) != 0) {
110 av_log(ctx, AV_LOG_ERROR,
111 "Left and right time bases differ (%d/%d vs %d/%d).\n",
112 time_base.num, time_base.den,
113 ctx->inputs[RIGHT]->time_base.num,
114 ctx->inputs[RIGHT]->time_base.den);
115 return AVERROR_INVALIDDATA;
116 } else if (av_cmp_q(frame_rate, ctx->inputs[RIGHT]->frame_rate) != 0) {
117 av_log(ctx, AV_LOG_ERROR,
118 "Left and right framerates differ (%d/%d vs %d/%d).\n",
119 frame_rate.num, frame_rate.den,
120 ctx->inputs[RIGHT]->frame_rate.num,
121 ctx->inputs[RIGHT]->frame_rate.den);
122 return AVERROR_INVALIDDATA;
123 }
124
125 s->pix_desc = av_pix_fmt_desc_get(outlink->format);
126 if (!s->pix_desc)
127 return AVERROR_BUG;
128 s->depth = s->pix_desc->comp[0].depth;
129
130 // modify output properties as needed
131 switch (s->format) {
132 case AV_STEREO3D_FRAMESEQUENCE:
133 time_base.den *= 2;
134 frame_rate.num *= 2;
135 break;
136 case AV_STEREO3D_COLUMNS:
137 case AV_STEREO3D_SIDEBYSIDE:
138 width *= 2;
139 break;
140 case AV_STEREO3D_LINES:
141 case AV_STEREO3D_TOPBOTTOM:
142 height *= 2;
143 break;
144 default:
145 av_log(ctx, AV_LOG_ERROR, "Unknown packing mode.");
146 return AVERROR_INVALIDDATA;
147 }
148
149 outlink->w = width;
150 outlink->h = height;
151 outlink->time_base = time_base;
152 outlink->frame_rate = frame_rate;
153
154 return 0;
155 }
156
horizontal_frame_pack(AVFilterLink * outlink,AVFrame * out,int interleaved)157 static void horizontal_frame_pack(AVFilterLink *outlink,
158 AVFrame *out,
159 int interleaved)
160 {
161 AVFilterContext *ctx = outlink->src;
162 FramepackContext *s = ctx->priv;
163 int i, plane;
164
165 if (interleaved && s->depth <= 8) {
166 const uint8_t *leftp = s->input_views[LEFT]->data[0];
167 const uint8_t *rightp = s->input_views[RIGHT]->data[0];
168 uint8_t *dstp = out->data[0];
169 int length = out->width / 2;
170 int lines = out->height;
171
172 for (plane = 0; plane < s->pix_desc->nb_components; plane++) {
173 if (plane == 1 || plane == 2) {
174 length = AV_CEIL_RSHIFT(out->width / 2, s->pix_desc->log2_chroma_w);
175 lines = AV_CEIL_RSHIFT(out->height, s->pix_desc->log2_chroma_h);
176 }
177 for (i = 0; i < lines; i++) {
178 int j;
179 leftp = s->input_views[LEFT]->data[plane] +
180 s->input_views[LEFT]->linesize[plane] * i;
181 rightp = s->input_views[RIGHT]->data[plane] +
182 s->input_views[RIGHT]->linesize[plane] * i;
183 dstp = out->data[plane] + out->linesize[plane] * i;
184 for (j = 0; j < length; j++) {
185 // interpolate chroma as necessary
186 if ((s->pix_desc->log2_chroma_w ||
187 s->pix_desc->log2_chroma_h) &&
188 (plane == 1 || plane == 2)) {
189 *dstp++ = (*leftp + *rightp) / 2;
190 *dstp++ = (*leftp + *rightp) / 2;
191 } else {
192 *dstp++ = *leftp;
193 *dstp++ = *rightp;
194 }
195 leftp += 1;
196 rightp += 1;
197 }
198 }
199 }
200 } else if (interleaved && s->depth > 8) {
201 const uint16_t *leftp = (const uint16_t *)s->input_views[LEFT]->data[0];
202 const uint16_t *rightp = (const uint16_t *)s->input_views[RIGHT]->data[0];
203 uint16_t *dstp = (uint16_t *)out->data[0];
204 int length = out->width / 2;
205 int lines = out->height;
206
207 for (plane = 0; plane < s->pix_desc->nb_components; plane++) {
208 if (plane == 1 || plane == 2) {
209 length = AV_CEIL_RSHIFT(out->width / 2, s->pix_desc->log2_chroma_w);
210 lines = AV_CEIL_RSHIFT(out->height, s->pix_desc->log2_chroma_h);
211 }
212 for (i = 0; i < lines; i++) {
213 int j;
214 leftp = (const uint16_t *)s->input_views[LEFT]->data[plane] +
215 s->input_views[LEFT]->linesize[plane] * i / 2;
216 rightp = (const uint16_t *)s->input_views[RIGHT]->data[plane] +
217 s->input_views[RIGHT]->linesize[plane] * i / 2;
218 dstp = (uint16_t *)out->data[plane] + out->linesize[plane] * i / 2;
219 for (j = 0; j < length; j++) {
220 // interpolate chroma as necessary
221 if ((s->pix_desc->log2_chroma_w ||
222 s->pix_desc->log2_chroma_h) &&
223 (plane == 1 || plane == 2)) {
224 *dstp++ = (*leftp + *rightp) / 2;
225 *dstp++ = (*leftp + *rightp) / 2;
226 } else {
227 *dstp++ = *leftp;
228 *dstp++ = *rightp;
229 }
230 leftp += 1;
231 rightp += 1;
232 }
233 }
234 }
235 } else {
236 for (i = 0; i < 2; i++) {
237 const int psize = 1 + (s->depth > 8);
238 const uint8_t *src[4];
239 uint8_t *dst[4];
240 int sub_w = psize * s->input_views[i]->width >> s->pix_desc->log2_chroma_w;
241
242 src[0] = s->input_views[i]->data[0];
243 src[1] = s->input_views[i]->data[1];
244 src[2] = s->input_views[i]->data[2];
245
246 dst[0] = out->data[0] + i * s->input_views[i]->width * psize;
247 dst[1] = out->data[1] + i * sub_w;
248 dst[2] = out->data[2] + i * sub_w;
249
250 av_image_copy(dst, out->linesize, src, s->input_views[i]->linesize,
251 s->input_views[i]->format,
252 s->input_views[i]->width,
253 s->input_views[i]->height);
254 }
255 }
256 }
257
vertical_frame_pack(AVFilterLink * outlink,AVFrame * out,int interleaved)258 static void vertical_frame_pack(AVFilterLink *outlink,
259 AVFrame *out,
260 int interleaved)
261 {
262 AVFilterContext *ctx = outlink->src;
263 FramepackContext *s = ctx->priv;
264 int i;
265
266 for (i = 0; i < 2; i++) {
267 const uint8_t *src[4];
268 uint8_t *dst[4];
269 int linesizes[4];
270 int sub_h = s->input_views[i]->height >> s->pix_desc->log2_chroma_h;
271
272 src[0] = s->input_views[i]->data[0];
273 src[1] = s->input_views[i]->data[1];
274 src[2] = s->input_views[i]->data[2];
275
276 dst[0] = out->data[0] + i * out->linesize[0] *
277 (interleaved + s->input_views[i]->height * (1 - interleaved));
278 dst[1] = out->data[1] + i * out->linesize[1] *
279 (interleaved + sub_h * (1 - interleaved));
280 dst[2] = out->data[2] + i * out->linesize[2] *
281 (interleaved + sub_h * (1 - interleaved));
282
283 linesizes[0] = out->linesize[0] +
284 interleaved * out->linesize[0];
285 linesizes[1] = out->linesize[1] +
286 interleaved * out->linesize[1];
287 linesizes[2] = out->linesize[2] +
288 interleaved * out->linesize[2];
289
290 av_image_copy(dst, linesizes, src, s->input_views[i]->linesize,
291 s->input_views[i]->format,
292 s->input_views[i]->width,
293 s->input_views[i]->height);
294 }
295 }
296
spatial_frame_pack(AVFilterLink * outlink,AVFrame * dst)297 static av_always_inline void spatial_frame_pack(AVFilterLink *outlink,
298 AVFrame *dst)
299 {
300 AVFilterContext *ctx = outlink->src;
301 FramepackContext *s = ctx->priv;
302 switch (s->format) {
303 case AV_STEREO3D_SIDEBYSIDE:
304 horizontal_frame_pack(outlink, dst, 0);
305 break;
306 case AV_STEREO3D_COLUMNS:
307 horizontal_frame_pack(outlink, dst, 1);
308 break;
309 case AV_STEREO3D_TOPBOTTOM:
310 vertical_frame_pack(outlink, dst, 0);
311 break;
312 case AV_STEREO3D_LINES:
313 vertical_frame_pack(outlink, dst, 1);
314 break;
315 }
316 }
317
try_push_frame(AVFilterContext * ctx)318 static int try_push_frame(AVFilterContext *ctx)
319 {
320 FramepackContext *s = ctx->priv;
321 AVFilterLink *outlink = ctx->outputs[0];
322 AVStereo3D *stereo;
323 int ret, i;
324
325 if (!(s->input_views[0] && s->input_views[1]))
326 return 0;
327 if (s->format == AV_STEREO3D_FRAMESEQUENCE) {
328 int64_t pts = s->input_views[0]->pts;
329
330 for (i = 0; i < 2; i++) {
331 // set correct timestamps
332 if (pts != AV_NOPTS_VALUE)
333 s->input_views[i]->pts = i == 0 ? pts * 2 : pts * 2 + av_rescale_q(1, av_inv_q(outlink->frame_rate), outlink->time_base);
334
335 // set stereo3d side data
336 stereo = av_stereo3d_create_side_data(s->input_views[i]);
337 if (!stereo)
338 return AVERROR(ENOMEM);
339 stereo->type = s->format;
340 stereo->view = i == LEFT ? AV_STEREO3D_VIEW_LEFT
341 : AV_STEREO3D_VIEW_RIGHT;
342
343 // filter the frame and immediately relinquish its pointer
344 ret = ff_filter_frame(outlink, s->input_views[i]);
345 s->input_views[i] = NULL;
346 if (ret < 0)
347 return ret;
348 }
349 return ret;
350 } else {
351 AVFrame *dst = ff_get_video_buffer(outlink, outlink->w, outlink->h);
352 if (!dst)
353 return AVERROR(ENOMEM);
354
355 spatial_frame_pack(outlink, dst);
356
357 // get any property from the original frame
358 ret = av_frame_copy_props(dst, s->input_views[LEFT]);
359 if (ret < 0) {
360 av_frame_free(&dst);
361 return ret;
362 }
363
364 for (i = 0; i < 2; i++)
365 av_frame_free(&s->input_views[i]);
366
367 // set stereo3d side data
368 stereo = av_stereo3d_create_side_data(dst);
369 if (!stereo) {
370 av_frame_free(&dst);
371 return AVERROR(ENOMEM);
372 }
373 stereo->type = s->format;
374
375 return ff_filter_frame(outlink, dst);
376 }
377 }
378
activate(AVFilterContext * ctx)379 static int activate(AVFilterContext *ctx)
380 {
381 AVFilterLink *outlink = ctx->outputs[0];
382 FramepackContext *s = ctx->priv;
383 int ret;
384
385 FF_FILTER_FORWARD_STATUS_BACK_ALL(outlink, ctx);
386
387 if (!s->input_views[0]) {
388 ret = ff_inlink_consume_frame(ctx->inputs[0], &s->input_views[0]);
389 if (ret < 0)
390 return ret;
391 }
392
393 if (!s->input_views[1]) {
394 ret = ff_inlink_consume_frame(ctx->inputs[1], &s->input_views[1]);
395 if (ret < 0)
396 return ret;
397 }
398
399 if (s->input_views[0] && s->input_views[1])
400 return try_push_frame(ctx);
401
402 FF_FILTER_FORWARD_STATUS(ctx->inputs[0], outlink);
403 FF_FILTER_FORWARD_STATUS(ctx->inputs[1], outlink);
404
405 if (ff_outlink_frame_wanted(ctx->outputs[0]) &&
406 !ff_outlink_get_status(ctx->inputs[0]) &&
407 !s->input_views[0]) {
408 ff_inlink_request_frame(ctx->inputs[0]);
409 return 0;
410 }
411
412 if (ff_outlink_frame_wanted(ctx->outputs[0]) &&
413 !ff_outlink_get_status(ctx->inputs[1]) &&
414 !s->input_views[1]) {
415 ff_inlink_request_frame(ctx->inputs[1]);
416 return 0;
417 }
418
419 return FFERROR_NOT_READY;
420 }
421
422 #define OFFSET(x) offsetof(FramepackContext, x)
423 #define VF AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM
424 static const AVOption framepack_options[] = {
425 { "format", "Frame pack output format", OFFSET(format), AV_OPT_TYPE_INT,
426 { .i64 = AV_STEREO3D_SIDEBYSIDE }, 0, INT_MAX, .flags = VF, .unit = "format" },
427 { "sbs", "Views are packed next to each other", 0, AV_OPT_TYPE_CONST,
428 { .i64 = AV_STEREO3D_SIDEBYSIDE }, INT_MIN, INT_MAX, .flags = VF, .unit = "format" },
429 { "tab", "Views are packed on top of each other", 0, AV_OPT_TYPE_CONST,
430 { .i64 = AV_STEREO3D_TOPBOTTOM }, INT_MIN, INT_MAX, .flags = VF, .unit = "format" },
431 { "frameseq", "Views are one after the other", 0, AV_OPT_TYPE_CONST,
432 { .i64 = AV_STEREO3D_FRAMESEQUENCE }, INT_MIN, INT_MAX, .flags = VF, .unit = "format" },
433 { "lines", "Views are interleaved by lines", 0, AV_OPT_TYPE_CONST,
434 { .i64 = AV_STEREO3D_LINES }, INT_MIN, INT_MAX, .flags = VF, .unit = "format" },
435 { "columns", "Views are interleaved by columns", 0, AV_OPT_TYPE_CONST,
436 { .i64 = AV_STEREO3D_COLUMNS }, INT_MIN, INT_MAX, .flags = VF, .unit = "format" },
437 { NULL },
438 };
439
440 AVFILTER_DEFINE_CLASS(framepack);
441
442 static const AVFilterPad framepack_inputs[] = {
443 {
444 .name = "left",
445 .type = AVMEDIA_TYPE_VIDEO,
446 },
447 {
448 .name = "right",
449 .type = AVMEDIA_TYPE_VIDEO,
450 },
451 };
452
453 static const AVFilterPad framepack_outputs[] = {
454 {
455 .name = "packed",
456 .type = AVMEDIA_TYPE_VIDEO,
457 .config_props = config_output,
458 },
459 };
460
461 const AVFilter ff_vf_framepack = {
462 .name = "framepack",
463 .description = NULL_IF_CONFIG_SMALL("Generate a frame packed stereoscopic video."),
464 .priv_size = sizeof(FramepackContext),
465 .priv_class = &framepack_class,
466 FILTER_INPUTS(framepack_inputs),
467 FILTER_OUTPUTS(framepack_outputs),
468 FILTER_PIXFMTS_ARRAY(formats_supported),
469 .activate = activate,
470 .uninit = framepack_uninit,
471 };
472