1 /**
2 * Copyright (c) 2014-2015 Michael Niedermayer <michaelni@gmx.at>
3 * Copyright (c) 2016 Davinder Singh (DSM_) <ds.mudhar<@gmail.com>
4 *
5 * This file is part of FFmpeg.
6 *
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 #include "motion_estimation.h"
23 #include "libavcodec/mathops.h"
24 #include "libavutil/common.h"
25 #include "libavutil/motion_vector.h"
26 #include "libavutil/opt.h"
27 #include "libavutil/pixdesc.h"
28 #include "avfilter.h"
29 #include "formats.h"
30 #include "internal.h"
31 #include "video.h"
32 #include "scene_sad.h"
33
34 #define ME_MODE_BIDIR 0
35 #define ME_MODE_BILAT 1
36
37 #define MC_MODE_OBMC 0
38 #define MC_MODE_AOBMC 1
39
40 #define SCD_METHOD_NONE 0
41 #define SCD_METHOD_FDIFF 1
42
43 #define NB_FRAMES 4
44 #define NB_PIXEL_MVS 32
45 #define NB_CLUSTERS 128
46
47 #define ALPHA_MAX 1024
48 #define CLUSTER_THRESHOLD 4
49 #define PX_WEIGHT_MAX 255
50 #define COST_PRED_SCALE 64
51
52 static const uint8_t obmc_linear32[1024] = {
53 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 8, 8, 8, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0,
54 0, 4, 4, 4, 8, 8, 8, 12, 12, 16, 16, 16, 20, 20, 20, 24, 24, 20, 20, 20, 16, 16, 16, 12, 12, 8, 8, 8, 4, 4, 4, 0,
55 0, 4, 8, 8, 12, 12, 16, 20, 20, 24, 28, 28, 32, 32, 36, 40, 40, 36, 32, 32, 28, 28, 24, 20, 20, 16, 12, 12, 8, 8, 4, 0,
56 0, 4, 8, 12, 16, 20, 24, 28, 28, 32, 36, 40, 44, 48, 52, 56, 56, 52, 48, 44, 40, 36, 32, 28, 28, 24, 20, 16, 12, 8, 4, 0,
57 4, 8, 12, 16, 20, 24, 28, 32, 40, 44, 48, 52, 56, 60, 64, 68, 68, 64, 60, 56, 52, 48, 44, 40, 32, 28, 24, 20, 16, 12, 8, 4,
58 4, 8, 12, 20, 24, 32, 36, 40, 48, 52, 56, 64, 68, 76, 80, 84, 84, 80, 76, 68, 64, 56, 52, 48, 40, 36, 32, 24, 20, 12, 8, 4,
59 4, 8, 16, 24, 28, 36, 44, 48, 56, 60, 68, 76, 80, 88, 96,100,100, 96, 88, 80, 76, 68, 60, 56, 48, 44, 36, 28, 24, 16, 8, 4,
60 4, 12, 20, 28, 32, 40, 48, 56, 64, 72, 80, 88, 92,100,108,116,116,108,100, 92, 88, 80, 72, 64, 56, 48, 40, 32, 28, 20, 12, 4,
61 4, 12, 20, 28, 40, 48, 56, 64, 72, 80, 88, 96,108,116,124,132,132,124,116,108, 96, 88, 80, 72, 64, 56, 48, 40, 28, 20, 12, 4,
62 4, 16, 24, 32, 44, 52, 60, 72, 80, 92,100,108,120,128,136,148,148,136,128,120,108,100, 92, 80, 72, 60, 52, 44, 32, 24, 16, 4,
63 4, 16, 28, 36, 48, 56, 68, 80, 88,100,112,120,132,140,152,164,164,152,140,132,120,112,100, 88, 80, 68, 56, 48, 36, 28, 16, 4,
64 4, 16, 28, 40, 52, 64, 76, 88, 96,108,120,132,144,156,168,180,180,168,156,144,132,120,108, 96, 88, 76, 64, 52, 40, 28, 16, 4,
65 8, 20, 32, 44, 56, 68, 80, 92,108,120,132,144,156,168,180,192,192,180,168,156,144,132,120,108, 92, 80, 68, 56, 44, 32, 20, 8,
66 8, 20, 32, 48, 60, 76, 88,100,116,128,140,156,168,184,196,208,208,196,184,168,156,140,128,116,100, 88, 76, 60, 48, 32, 20, 8,
67 8, 20, 36, 52, 64, 80, 96,108,124,136,152,168,180,196,212,224,224,212,196,180,168,152,136,124,108, 96, 80, 64, 52, 36, 20, 8,
68 8, 24, 40, 56, 68, 84,100,116,132,148,164,180,192,208,224,240,240,224,208,192,180,164,148,132,116,100, 84, 68, 56, 40, 24, 8,
69 8, 24, 40, 56, 68, 84,100,116,132,148,164,180,192,208,224,240,240,224,208,192,180,164,148,132,116,100, 84, 68, 56, 40, 24, 8,
70 8, 20, 36, 52, 64, 80, 96,108,124,136,152,168,180,196,212,224,224,212,196,180,168,152,136,124,108, 96, 80, 64, 52, 36, 20, 8,
71 8, 20, 32, 48, 60, 76, 88,100,116,128,140,156,168,184,196,208,208,196,184,168,156,140,128,116,100, 88, 76, 60, 48, 32, 20, 8,
72 8, 20, 32, 44, 56, 68, 80, 92,108,120,132,144,156,168,180,192,192,180,168,156,144,132,120,108, 92, 80, 68, 56, 44, 32, 20, 8,
73 4, 16, 28, 40, 52, 64, 76, 88, 96,108,120,132,144,156,168,180,180,168,156,144,132,120,108, 96, 88, 76, 64, 52, 40, 28, 16, 4,
74 4, 16, 28, 36, 48, 56, 68, 80, 88,100,112,120,132,140,152,164,164,152,140,132,120,112,100, 88, 80, 68, 56, 48, 36, 28, 16, 4,
75 4, 16, 24, 32, 44, 52, 60, 72, 80, 92,100,108,120,128,136,148,148,136,128,120,108,100, 92, 80, 72, 60, 52, 44, 32, 24, 16, 4,
76 4, 12, 20, 28, 40, 48, 56, 64, 72, 80, 88, 96,108,116,124,132,132,124,116,108, 96, 88, 80, 72, 64, 56, 48, 40, 28, 20, 12, 4,
77 4, 12, 20, 28, 32, 40, 48, 56, 64, 72, 80, 88, 92,100,108,116,116,108,100, 92, 88, 80, 72, 64, 56, 48, 40, 32, 28, 20, 12, 4,
78 4, 8, 16, 24, 28, 36, 44, 48, 56, 60, 68, 76, 80, 88, 96,100,100, 96, 88, 80, 76, 68, 60, 56, 48, 44, 36, 28, 24, 16, 8, 4,
79 4, 8, 12, 20, 24, 32, 36, 40, 48, 52, 56, 64, 68, 76, 80, 84, 84, 80, 76, 68, 64, 56, 52, 48, 40, 36, 32, 24, 20, 12, 8, 4,
80 4, 8, 12, 16, 20, 24, 28, 32, 40, 44, 48, 52, 56, 60, 64, 68, 68, 64, 60, 56, 52, 48, 44, 40, 32, 28, 24, 20, 16, 12, 8, 4,
81 0, 4, 8, 12, 16, 20, 24, 28, 28, 32, 36, 40, 44, 48, 52, 56, 56, 52, 48, 44, 40, 36, 32, 28, 28, 24, 20, 16, 12, 8, 4, 0,
82 0, 4, 8, 8, 12, 12, 16, 20, 20, 24, 28, 28, 32, 32, 36, 40, 40, 36, 32, 32, 28, 28, 24, 20, 20, 16, 12, 12, 8, 8, 4, 0,
83 0, 4, 4, 4, 8, 8, 8, 12, 12, 16, 16, 16, 20, 20, 20, 24, 24, 20, 20, 20, 16, 16, 16, 12, 12, 8, 8, 8, 4, 4, 4, 0,
84 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 8, 8, 8, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0,
85 };
86
87 static const uint8_t obmc_linear16[256] = {
88 0, 4, 4, 8, 8, 12, 12, 16, 16, 12, 12, 8, 8, 4, 4, 0,
89 4, 8, 16, 20, 28, 32, 40, 44, 44, 40, 32, 28, 20, 16, 8, 4,
90 4, 16, 24, 36, 44, 56, 64, 76, 76, 64, 56, 44, 36, 24, 16, 4,
91 8, 20, 36, 48, 64, 76, 92,104,104, 92, 76, 64, 48, 36, 20, 8,
92 8, 28, 44, 64, 80,100,116,136,136,116,100, 80, 64, 44, 28, 8,
93 12, 32, 56, 76,100,120,144,164,164,144,120,100, 76, 56, 32, 12,
94 12, 40, 64, 92,116,144,168,196,196,168,144,116, 92, 64, 40, 12,
95 16, 44, 76,104,136,164,196,224,224,196,164,136,104, 76, 44, 16,
96 16, 44, 76,104,136,164,196,224,224,196,164,136,104, 76, 44, 16,
97 12, 40, 64, 92,116,144,168,196,196,168,144,116, 92, 64, 40, 12,
98 12, 32, 56, 76,100,120,144,164,164,144,120,100, 76, 56, 32, 12,
99 8, 28, 44, 64, 80,100,116,136,136,116,100, 80, 64, 44, 28, 8,
100 8, 20, 36, 48, 64, 76, 92,104,104, 92, 76, 64, 48, 36, 20, 8,
101 4, 16, 24, 36, 44, 56, 64, 76, 76, 64, 56, 44, 36, 24, 16, 4,
102 4, 8, 16, 20, 28, 32, 40, 44, 44, 40, 32, 28, 20, 16, 8, 4,
103 0, 4, 4, 8, 8, 12, 12, 16, 16, 12, 12, 8, 8, 4, 4, 0,
104 };
105
106 static const uint8_t obmc_linear8[64] = {
107 4, 12, 20, 28, 28, 20, 12, 4,
108 12, 36, 60, 84, 84, 60, 36, 12,
109 20, 60,100,140,140,100, 60, 20,
110 28, 84,140,196,196,140, 84, 28,
111 28, 84,140,196,196,140, 84, 28,
112 20, 60,100,140,140,100, 60, 20,
113 12, 36, 60, 84, 84, 60, 36, 12,
114 4, 12, 20, 28, 28, 20, 12, 4,
115 };
116
117 static const uint8_t obmc_linear4[16] = {
118 16, 48, 48, 16,
119 48,144,144, 48,
120 48,144,144, 48,
121 16, 48, 48, 16,
122 };
123
124 static const uint8_t * const obmc_tab_linear[4]= {
125 obmc_linear32, obmc_linear16, obmc_linear8, obmc_linear4
126 };
127
128 enum MIMode {
129 MI_MODE_DUP = 0,
130 MI_MODE_BLEND = 1,
131 MI_MODE_MCI = 2,
132 };
133
134 typedef struct Cluster {
135 int64_t sum[2];
136 int nb;
137 } Cluster;
138
139 typedef struct Block {
140 int16_t mvs[2][2];
141 int cid;
142 uint64_t sbad;
143 int sb;
144 struct Block *subs;
145 } Block;
146
147 typedef struct PixelMVS {
148 int16_t mvs[NB_PIXEL_MVS][2];
149 } PixelMVS;
150
151 typedef struct PixelWeights {
152 uint32_t weights[NB_PIXEL_MVS];
153 } PixelWeights;
154
155 typedef struct PixelRefs {
156 int8_t refs[NB_PIXEL_MVS];
157 int nb;
158 } PixelRefs;
159
160 typedef struct Frame {
161 AVFrame *avf;
162 Block *blocks;
163 } Frame;
164
165 typedef struct MIContext {
166 const AVClass *class;
167 AVMotionEstContext me_ctx;
168 AVRational frame_rate;
169 enum MIMode mi_mode;
170 int mc_mode;
171 int me_mode;
172 int me_method;
173 int mb_size;
174 int search_param;
175 int vsbmc;
176
177 Frame frames[NB_FRAMES];
178 Cluster clusters[NB_CLUSTERS];
179 Block *int_blocks;
180 PixelMVS *pixel_mvs;
181 PixelWeights *pixel_weights;
182 PixelRefs *pixel_refs;
183 int (*mv_table[3])[2][2];
184 int64_t out_pts;
185 int b_width, b_height, b_count;
186 int log2_mb_size;
187 int bitdepth;
188
189 int scd_method;
190 int scene_changed;
191 ff_scene_sad_fn sad;
192 double prev_mafd;
193 double scd_threshold;
194
195 int log2_chroma_w;
196 int log2_chroma_h;
197 int nb_planes;
198 } MIContext;
199
200 #define OFFSET(x) offsetof(MIContext, x)
201 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
202 #define CONST(name, help, val, unit) { name, help, 0, AV_OPT_TYPE_CONST, {.i64=val}, 0, 0, FLAGS, unit }
203
204 static const AVOption minterpolate_options[] = {
205 { "fps", "output's frame rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str = "60"}, 0, INT_MAX, FLAGS },
206 { "mi_mode", "motion interpolation mode", OFFSET(mi_mode), AV_OPT_TYPE_INT, {.i64 = MI_MODE_MCI}, MI_MODE_DUP, MI_MODE_MCI, FLAGS, "mi_mode" },
207 CONST("dup", "duplicate frames", MI_MODE_DUP, "mi_mode"),
208 CONST("blend", "blend frames", MI_MODE_BLEND, "mi_mode"),
209 CONST("mci", "motion compensated interpolation", MI_MODE_MCI, "mi_mode"),
210 { "mc_mode", "motion compensation mode", OFFSET(mc_mode), AV_OPT_TYPE_INT, {.i64 = MC_MODE_OBMC}, MC_MODE_OBMC, MC_MODE_AOBMC, FLAGS, "mc_mode" },
211 CONST("obmc", "overlapped block motion compensation", MC_MODE_OBMC, "mc_mode"),
212 CONST("aobmc", "adaptive overlapped block motion compensation", MC_MODE_AOBMC, "mc_mode"),
213 { "me_mode", "motion estimation mode", OFFSET(me_mode), AV_OPT_TYPE_INT, {.i64 = ME_MODE_BILAT}, ME_MODE_BIDIR, ME_MODE_BILAT, FLAGS, "me_mode" },
214 CONST("bidir", "bidirectional motion estimation", ME_MODE_BIDIR, "me_mode"),
215 CONST("bilat", "bilateral motion estimation", ME_MODE_BILAT, "me_mode"),
216 { "me", "motion estimation method", OFFSET(me_method), AV_OPT_TYPE_INT, {.i64 = AV_ME_METHOD_EPZS}, AV_ME_METHOD_ESA, AV_ME_METHOD_UMH, FLAGS, "me" },
217 CONST("esa", "exhaustive search", AV_ME_METHOD_ESA, "me"),
218 CONST("tss", "three step search", AV_ME_METHOD_TSS, "me"),
219 CONST("tdls", "two dimensional logarithmic search", AV_ME_METHOD_TDLS, "me"),
220 CONST("ntss", "new three step search", AV_ME_METHOD_NTSS, "me"),
221 CONST("fss", "four step search", AV_ME_METHOD_FSS, "me"),
222 CONST("ds", "diamond search", AV_ME_METHOD_DS, "me"),
223 CONST("hexbs", "hexagon-based search", AV_ME_METHOD_HEXBS, "me"),
224 CONST("epzs", "enhanced predictive zonal search", AV_ME_METHOD_EPZS, "me"),
225 CONST("umh", "uneven multi-hexagon search", AV_ME_METHOD_UMH, "me"),
226 { "mb_size", "macroblock size", OFFSET(mb_size), AV_OPT_TYPE_INT, {.i64 = 16}, 4, 16, FLAGS },
227 { "search_param", "search parameter", OFFSET(search_param), AV_OPT_TYPE_INT, {.i64 = 32}, 4, INT_MAX, FLAGS },
228 { "vsbmc", "variable-size block motion compensation", OFFSET(vsbmc), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, FLAGS },
229 { "scd", "scene change detection method", OFFSET(scd_method), AV_OPT_TYPE_INT, {.i64 = SCD_METHOD_FDIFF}, SCD_METHOD_NONE, SCD_METHOD_FDIFF, FLAGS, "scene" },
230 CONST("none", "disable detection", SCD_METHOD_NONE, "scene"),
231 CONST("fdiff", "frame difference", SCD_METHOD_FDIFF, "scene"),
232 { "scd_threshold", "scene change threshold", OFFSET(scd_threshold), AV_OPT_TYPE_DOUBLE, {.dbl = 10.}, 0, 100.0, FLAGS },
233 { NULL }
234 };
235
236 AVFILTER_DEFINE_CLASS(minterpolate);
237
238 static const enum AVPixelFormat pix_fmts[] = {
239 AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
240 AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
241 AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,
242 AV_PIX_FMT_YUVJ444P, AV_PIX_FMT_YUVJ440P,
243 AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ420P,
244 AV_PIX_FMT_YUVJ411P,
245 AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P,
246 AV_PIX_FMT_GRAY8,
247 AV_PIX_FMT_NONE
248 };
249
get_sbad(AVMotionEstContext * me_ctx,int x,int y,int x_mv,int y_mv)250 static uint64_t get_sbad(AVMotionEstContext *me_ctx, int x, int y, int x_mv, int y_mv)
251 {
252 uint8_t *data_cur = me_ctx->data_cur;
253 uint8_t *data_next = me_ctx->data_ref;
254 int linesize = me_ctx->linesize;
255 int mv_x1 = x_mv - x;
256 int mv_y1 = y_mv - y;
257 int mv_x, mv_y, i, j;
258 uint64_t sbad = 0;
259
260 x = av_clip(x, me_ctx->x_min, me_ctx->x_max);
261 y = av_clip(y, me_ctx->y_min, me_ctx->y_max);
262 mv_x = av_clip(x_mv - x, -FFMIN(x - me_ctx->x_min, me_ctx->x_max - x), FFMIN(x - me_ctx->x_min, me_ctx->x_max - x));
263 mv_y = av_clip(y_mv - y, -FFMIN(y - me_ctx->y_min, me_ctx->y_max - y), FFMIN(y - me_ctx->y_min, me_ctx->y_max - y));
264
265 data_cur += (y + mv_y) * linesize;
266 data_next += (y - mv_y) * linesize;
267
268 for (j = 0; j < me_ctx->mb_size; j++)
269 for (i = 0; i < me_ctx->mb_size; i++)
270 sbad += FFABS(data_cur[x + mv_x + i + j * linesize] - data_next[x - mv_x + i + j * linesize]);
271
272 return sbad + (FFABS(mv_x1 - me_ctx->pred_x) + FFABS(mv_y1 - me_ctx->pred_y)) * COST_PRED_SCALE;
273 }
274
get_sbad_ob(AVMotionEstContext * me_ctx,int x,int y,int x_mv,int y_mv)275 static uint64_t get_sbad_ob(AVMotionEstContext *me_ctx, int x, int y, int x_mv, int y_mv)
276 {
277 uint8_t *data_cur = me_ctx->data_cur;
278 uint8_t *data_next = me_ctx->data_ref;
279 int linesize = me_ctx->linesize;
280 int x_min = me_ctx->x_min + me_ctx->mb_size / 2;
281 int x_max = me_ctx->x_max - me_ctx->mb_size / 2;
282 int y_min = me_ctx->y_min + me_ctx->mb_size / 2;
283 int y_max = me_ctx->y_max - me_ctx->mb_size / 2;
284 int mv_x1 = x_mv - x;
285 int mv_y1 = y_mv - y;
286 int mv_x, mv_y, i, j;
287 uint64_t sbad = 0;
288
289 x = av_clip(x, x_min, x_max);
290 y = av_clip(y, y_min, y_max);
291 mv_x = av_clip(x_mv - x, -FFMIN(x - x_min, x_max - x), FFMIN(x - x_min, x_max - x));
292 mv_y = av_clip(y_mv - y, -FFMIN(y - y_min, y_max - y), FFMIN(y - y_min, y_max - y));
293
294 for (j = -me_ctx->mb_size / 2; j < me_ctx->mb_size * 3 / 2; j++)
295 for (i = -me_ctx->mb_size / 2; i < me_ctx->mb_size * 3 / 2; i++)
296 sbad += FFABS(data_cur[x + mv_x + i + (y + mv_y + j) * linesize] - data_next[x - mv_x + i + (y - mv_y + j) * linesize]);
297
298 return sbad + (FFABS(mv_x1 - me_ctx->pred_x) + FFABS(mv_y1 - me_ctx->pred_y)) * COST_PRED_SCALE;
299 }
300
get_sad_ob(AVMotionEstContext * me_ctx,int x,int y,int x_mv,int y_mv)301 static uint64_t get_sad_ob(AVMotionEstContext *me_ctx, int x, int y, int x_mv, int y_mv)
302 {
303 uint8_t *data_ref = me_ctx->data_ref;
304 uint8_t *data_cur = me_ctx->data_cur;
305 int linesize = me_ctx->linesize;
306 int x_min = me_ctx->x_min + me_ctx->mb_size / 2;
307 int x_max = me_ctx->x_max - me_ctx->mb_size / 2;
308 int y_min = me_ctx->y_min + me_ctx->mb_size / 2;
309 int y_max = me_ctx->y_max - me_ctx->mb_size / 2;
310 int mv_x = x_mv - x;
311 int mv_y = y_mv - y;
312 int i, j;
313 uint64_t sad = 0;
314
315 x = av_clip(x, x_min, x_max);
316 y = av_clip(y, y_min, y_max);
317 x_mv = av_clip(x_mv, x_min, x_max);
318 y_mv = av_clip(y_mv, y_min, y_max);
319
320 for (j = -me_ctx->mb_size / 2; j < me_ctx->mb_size * 3 / 2; j++)
321 for (i = -me_ctx->mb_size / 2; i < me_ctx->mb_size * 3 / 2; i++)
322 sad += FFABS(data_ref[x_mv + i + (y_mv + j) * linesize] - data_cur[x + i + (y + j) * linesize]);
323
324 return sad + (FFABS(mv_x - me_ctx->pred_x) + FFABS(mv_y - me_ctx->pred_y)) * COST_PRED_SCALE;
325 }
326
config_input(AVFilterLink * inlink)327 static int config_input(AVFilterLink *inlink)
328 {
329 MIContext *mi_ctx = inlink->dst->priv;
330 AVMotionEstContext *me_ctx = &mi_ctx->me_ctx;
331 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
332 const int height = inlink->h;
333 const int width = inlink->w;
334 int i;
335
336 mi_ctx->log2_chroma_h = desc->log2_chroma_h;
337 mi_ctx->log2_chroma_w = desc->log2_chroma_w;
338 mi_ctx->bitdepth = desc->comp[0].depth;
339
340 mi_ctx->nb_planes = av_pix_fmt_count_planes(inlink->format);
341
342 mi_ctx->log2_mb_size = av_ceil_log2_c(mi_ctx->mb_size);
343 mi_ctx->mb_size = 1 << mi_ctx->log2_mb_size;
344
345 mi_ctx->b_width = width >> mi_ctx->log2_mb_size;
346 mi_ctx->b_height = height >> mi_ctx->log2_mb_size;
347 mi_ctx->b_count = mi_ctx->b_width * mi_ctx->b_height;
348
349 for (i = 0; i < NB_FRAMES; i++) {
350 Frame *frame = &mi_ctx->frames[i];
351 frame->blocks = av_calloc(mi_ctx->b_count, sizeof(*frame->blocks));
352 if (!frame->blocks)
353 return AVERROR(ENOMEM);
354 }
355
356 if (mi_ctx->mi_mode == MI_MODE_MCI) {
357 if (mi_ctx->b_width < 2 || mi_ctx->b_height < 2) {
358 av_log(inlink->dst, AV_LOG_ERROR, "Height or width < %d\n",
359 2 * mi_ctx->mb_size);
360 return AVERROR(EINVAL);
361 }
362 ff_me_init_context(me_ctx, mi_ctx->mb_size, mi_ctx->search_param,
363 width, height, 0, (mi_ctx->b_width - 1) << mi_ctx->log2_mb_size,
364 0, (mi_ctx->b_height - 1) << mi_ctx->log2_mb_size);
365
366 if (mi_ctx->me_mode == ME_MODE_BIDIR)
367 me_ctx->get_cost = &get_sad_ob;
368 else if (mi_ctx->me_mode == ME_MODE_BILAT)
369 me_ctx->get_cost = &get_sbad_ob;
370
371 mi_ctx->pixel_mvs = av_calloc(width * height, sizeof(*mi_ctx->pixel_mvs));
372 mi_ctx->pixel_weights = av_calloc(width * height, sizeof(*mi_ctx->pixel_weights));
373 mi_ctx->pixel_refs = av_calloc(width * height, sizeof(*mi_ctx->pixel_refs));
374 if (!mi_ctx->pixel_mvs || !mi_ctx->pixel_weights || !mi_ctx->pixel_refs)
375 return AVERROR(ENOMEM);
376
377 if (mi_ctx->me_mode == ME_MODE_BILAT)
378 if (!FF_ALLOCZ_TYPED_ARRAY(mi_ctx->int_blocks, mi_ctx->b_count))
379 return AVERROR(ENOMEM);
380
381 if (mi_ctx->me_method == AV_ME_METHOD_EPZS) {
382 for (i = 0; i < 3; i++) {
383 mi_ctx->mv_table[i] = av_calloc(mi_ctx->b_count, sizeof(*mi_ctx->mv_table[0]));
384 if (!mi_ctx->mv_table[i])
385 return AVERROR(ENOMEM);
386 }
387 }
388 }
389
390 if (mi_ctx->scd_method == SCD_METHOD_FDIFF) {
391 mi_ctx->sad = ff_scene_sad_get_fn(mi_ctx->bitdepth == 8 ? 8 : 16);
392 if (!mi_ctx->sad)
393 return AVERROR(EINVAL);
394 }
395
396 return 0;
397 }
398
config_output(AVFilterLink * outlink)399 static int config_output(AVFilterLink *outlink)
400 {
401 MIContext *mi_ctx = outlink->src->priv;
402
403 outlink->frame_rate = mi_ctx->frame_rate;
404 outlink->time_base = av_inv_q(mi_ctx->frame_rate);
405
406 return 0;
407 }
408
409 #define ADD_PRED(preds, px, py)\
410 do {\
411 preds.mvs[preds.nb][0] = px;\
412 preds.mvs[preds.nb][1] = py;\
413 preds.nb++;\
414 } while(0)
415
search_mv(MIContext * mi_ctx,Block * blocks,int mb_x,int mb_y,int dir)416 static void search_mv(MIContext *mi_ctx, Block *blocks, int mb_x, int mb_y, int dir)
417 {
418 AVMotionEstContext *me_ctx = &mi_ctx->me_ctx;
419 AVMotionEstPredictor *preds = me_ctx->preds;
420 Block *block = &blocks[mb_x + mb_y * mi_ctx->b_width];
421
422 const int x_mb = mb_x << mi_ctx->log2_mb_size;
423 const int y_mb = mb_y << mi_ctx->log2_mb_size;
424 const int mb_i = mb_x + mb_y * mi_ctx->b_width;
425 int mv[2] = {x_mb, y_mb};
426
427 switch (mi_ctx->me_method) {
428 case AV_ME_METHOD_ESA:
429 ff_me_search_esa(me_ctx, x_mb, y_mb, mv);
430 break;
431 case AV_ME_METHOD_TSS:
432 ff_me_search_tss(me_ctx, x_mb, y_mb, mv);
433 break;
434 case AV_ME_METHOD_TDLS:
435 ff_me_search_tdls(me_ctx, x_mb, y_mb, mv);
436 break;
437 case AV_ME_METHOD_NTSS:
438 ff_me_search_ntss(me_ctx, x_mb, y_mb, mv);
439 break;
440 case AV_ME_METHOD_FSS:
441 ff_me_search_fss(me_ctx, x_mb, y_mb, mv);
442 break;
443 case AV_ME_METHOD_DS:
444 ff_me_search_ds(me_ctx, x_mb, y_mb, mv);
445 break;
446 case AV_ME_METHOD_HEXBS:
447 ff_me_search_hexbs(me_ctx, x_mb, y_mb, mv);
448 break;
449 case AV_ME_METHOD_EPZS:
450
451 preds[0].nb = 0;
452 preds[1].nb = 0;
453
454 ADD_PRED(preds[0], 0, 0);
455
456 //left mb in current frame
457 if (mb_x > 0)
458 ADD_PRED(preds[0], mi_ctx->mv_table[0][mb_i - 1][dir][0], mi_ctx->mv_table[0][mb_i - 1][dir][1]);
459
460 //top mb in current frame
461 if (mb_y > 0)
462 ADD_PRED(preds[0], mi_ctx->mv_table[0][mb_i - mi_ctx->b_width][dir][0], mi_ctx->mv_table[0][mb_i - mi_ctx->b_width][dir][1]);
463
464 //top-right mb in current frame
465 if (mb_y > 0 && mb_x + 1 < mi_ctx->b_width)
466 ADD_PRED(preds[0], mi_ctx->mv_table[0][mb_i - mi_ctx->b_width + 1][dir][0], mi_ctx->mv_table[0][mb_i - mi_ctx->b_width + 1][dir][1]);
467
468 //median predictor
469 if (preds[0].nb == 4) {
470 me_ctx->pred_x = mid_pred(preds[0].mvs[1][0], preds[0].mvs[2][0], preds[0].mvs[3][0]);
471 me_ctx->pred_y = mid_pred(preds[0].mvs[1][1], preds[0].mvs[2][1], preds[0].mvs[3][1]);
472 } else if (preds[0].nb == 3) {
473 me_ctx->pred_x = mid_pred(0, preds[0].mvs[1][0], preds[0].mvs[2][0]);
474 me_ctx->pred_y = mid_pred(0, preds[0].mvs[1][1], preds[0].mvs[2][1]);
475 } else if (preds[0].nb == 2) {
476 me_ctx->pred_x = preds[0].mvs[1][0];
477 me_ctx->pred_y = preds[0].mvs[1][1];
478 } else {
479 me_ctx->pred_x = 0;
480 me_ctx->pred_y = 0;
481 }
482
483 //collocated mb in prev frame
484 ADD_PRED(preds[0], mi_ctx->mv_table[1][mb_i][dir][0], mi_ctx->mv_table[1][mb_i][dir][1]);
485
486 //accelerator motion vector of collocated block in prev frame
487 ADD_PRED(preds[1], mi_ctx->mv_table[1][mb_i][dir][0] + (mi_ctx->mv_table[1][mb_i][dir][0] - mi_ctx->mv_table[2][mb_i][dir][0]),
488 mi_ctx->mv_table[1][mb_i][dir][1] + (mi_ctx->mv_table[1][mb_i][dir][1] - mi_ctx->mv_table[2][mb_i][dir][1]));
489
490 //left mb in prev frame
491 if (mb_x > 0)
492 ADD_PRED(preds[1], mi_ctx->mv_table[1][mb_i - 1][dir][0], mi_ctx->mv_table[1][mb_i - 1][dir][1]);
493
494 //top mb in prev frame
495 if (mb_y > 0)
496 ADD_PRED(preds[1], mi_ctx->mv_table[1][mb_i - mi_ctx->b_width][dir][0], mi_ctx->mv_table[1][mb_i - mi_ctx->b_width][dir][1]);
497
498 //right mb in prev frame
499 if (mb_x + 1 < mi_ctx->b_width)
500 ADD_PRED(preds[1], mi_ctx->mv_table[1][mb_i + 1][dir][0], mi_ctx->mv_table[1][mb_i + 1][dir][1]);
501
502 //bottom mb in prev frame
503 if (mb_y + 1 < mi_ctx->b_height)
504 ADD_PRED(preds[1], mi_ctx->mv_table[1][mb_i + mi_ctx->b_width][dir][0], mi_ctx->mv_table[1][mb_i + mi_ctx->b_width][dir][1]);
505
506 ff_me_search_epzs(me_ctx, x_mb, y_mb, mv);
507
508 mi_ctx->mv_table[0][mb_i][dir][0] = mv[0] - x_mb;
509 mi_ctx->mv_table[0][mb_i][dir][1] = mv[1] - y_mb;
510
511 break;
512 case AV_ME_METHOD_UMH:
513
514 preds[0].nb = 0;
515
516 ADD_PRED(preds[0], 0, 0);
517
518 //left mb in current frame
519 if (mb_x > 0)
520 ADD_PRED(preds[0], blocks[mb_i - 1].mvs[dir][0], blocks[mb_i - 1].mvs[dir][1]);
521
522 if (mb_y > 0) {
523 //top mb in current frame
524 ADD_PRED(preds[0], blocks[mb_i - mi_ctx->b_width].mvs[dir][0], blocks[mb_i - mi_ctx->b_width].mvs[dir][1]);
525
526 //top-right mb in current frame
527 if (mb_x + 1 < mi_ctx->b_width)
528 ADD_PRED(preds[0], blocks[mb_i - mi_ctx->b_width + 1].mvs[dir][0], blocks[mb_i - mi_ctx->b_width + 1].mvs[dir][1]);
529 //top-left mb in current frame
530 else if (mb_x > 0)
531 ADD_PRED(preds[0], blocks[mb_i - mi_ctx->b_width - 1].mvs[dir][0], blocks[mb_i - mi_ctx->b_width - 1].mvs[dir][1]);
532 }
533
534 //median predictor
535 if (preds[0].nb == 4) {
536 me_ctx->pred_x = mid_pred(preds[0].mvs[1][0], preds[0].mvs[2][0], preds[0].mvs[3][0]);
537 me_ctx->pred_y = mid_pred(preds[0].mvs[1][1], preds[0].mvs[2][1], preds[0].mvs[3][1]);
538 } else if (preds[0].nb == 3) {
539 me_ctx->pred_x = mid_pred(0, preds[0].mvs[1][0], preds[0].mvs[2][0]);
540 me_ctx->pred_y = mid_pred(0, preds[0].mvs[1][1], preds[0].mvs[2][1]);
541 } else if (preds[0].nb == 2) {
542 me_ctx->pred_x = preds[0].mvs[1][0];
543 me_ctx->pred_y = preds[0].mvs[1][1];
544 } else {
545 me_ctx->pred_x = 0;
546 me_ctx->pred_y = 0;
547 }
548
549 ff_me_search_umh(me_ctx, x_mb, y_mb, mv);
550
551 break;
552 }
553
554 block->mvs[dir][0] = mv[0] - x_mb;
555 block->mvs[dir][1] = mv[1] - y_mb;
556 }
557
bilateral_me(MIContext * mi_ctx)558 static void bilateral_me(MIContext *mi_ctx)
559 {
560 Block *block;
561 int mb_x, mb_y;
562
563 for (mb_y = 0; mb_y < mi_ctx->b_height; mb_y++)
564 for (mb_x = 0; mb_x < mi_ctx->b_width; mb_x++) {
565 block = &mi_ctx->int_blocks[mb_x + mb_y * mi_ctx->b_width];
566
567 block->cid = 0;
568 block->sb = 0;
569
570 block->mvs[0][0] = 0;
571 block->mvs[0][1] = 0;
572 }
573
574 for (mb_y = 0; mb_y < mi_ctx->b_height; mb_y++)
575 for (mb_x = 0; mb_x < mi_ctx->b_width; mb_x++)
576 search_mv(mi_ctx, mi_ctx->int_blocks, mb_x, mb_y, 0);
577 }
578
var_size_bme(MIContext * mi_ctx,Block * block,int x_mb,int y_mb,int n)579 static int var_size_bme(MIContext *mi_ctx, Block *block, int x_mb, int y_mb, int n)
580 {
581 AVMotionEstContext *me_ctx = &mi_ctx->me_ctx;
582 uint64_t cost_sb, cost_old;
583 int mb_size = me_ctx->mb_size;
584 int search_param = me_ctx->search_param;
585 int mv_x, mv_y;
586 int x, y;
587 int ret;
588
589 me_ctx->mb_size = 1 << n;
590 cost_old = me_ctx->get_cost(me_ctx, x_mb, y_mb, x_mb + block->mvs[0][0], y_mb + block->mvs[0][1]);
591 me_ctx->mb_size = mb_size;
592
593 if (!cost_old) {
594 block->sb = 0;
595 return 0;
596 }
597
598 if (!block->subs) {
599 block->subs = av_mallocz(4 * sizeof(*block->subs));
600 if (!block->subs)
601 return AVERROR(ENOMEM);
602 }
603
604 block->sb = 1;
605
606 for (y = 0; y < 2; y++)
607 for (x = 0; x < 2; x++) {
608 Block *sb = &block->subs[x + y * 2];
609 int mv[2] = {x_mb + block->mvs[0][0], y_mb + block->mvs[0][1]};
610
611 me_ctx->mb_size = 1 << (n - 1);
612 me_ctx->search_param = 2;
613 me_ctx->pred_x = block->mvs[0][0];
614 me_ctx->pred_y = block->mvs[0][1];
615
616 cost_sb = ff_me_search_ds(&mi_ctx->me_ctx, x_mb + block->mvs[0][0], y_mb + block->mvs[0][1], mv);
617 mv_x = mv[0] - x_mb;
618 mv_y = mv[1] - y_mb;
619
620 me_ctx->mb_size = mb_size;
621 me_ctx->search_param = search_param;
622
623 if (cost_sb < cost_old / 4) {
624 sb->mvs[0][0] = mv_x;
625 sb->mvs[0][1] = mv_y;
626
627 if (n > 1) {
628 if (ret = var_size_bme(mi_ctx, sb, x_mb + (x << (n - 1)), y_mb + (y << (n - 1)), n - 1))
629 return ret;
630 } else
631 sb->sb = 0;
632 } else {
633 block->sb = 0;
634 return 0;
635 }
636 }
637
638 return 0;
639 }
640
cluster_mvs(MIContext * mi_ctx)641 static int cluster_mvs(MIContext *mi_ctx)
642 {
643 int changed, c, c_max = 0;
644 int mb_x, mb_y, x, y;
645 int mv_x, mv_y, avg_x, avg_y, dx, dy;
646 int d, ret;
647 Block *block;
648 Cluster *cluster, *cluster_new;
649
650 do {
651 changed = 0;
652 for (mb_y = 0; mb_y < mi_ctx->b_height; mb_y++)
653 for (mb_x = 0; mb_x < mi_ctx->b_width; mb_x++) {
654 block = &mi_ctx->int_blocks[mb_x + mb_y * mi_ctx->b_width];
655 c = block->cid;
656 cluster = &mi_ctx->clusters[c];
657 mv_x = block->mvs[0][0];
658 mv_y = block->mvs[0][1];
659
660 if (cluster->nb < 2)
661 continue;
662
663 avg_x = cluster->sum[0] / cluster->nb;
664 avg_y = cluster->sum[1] / cluster->nb;
665 dx = avg_x - mv_x;
666 dy = avg_y - mv_y;
667
668 if (FFABS(dx) > CLUSTER_THRESHOLD || FFABS(dy) > CLUSTER_THRESHOLD) {
669
670 for (d = 1; d < 5; d++)
671 for (y = FFMAX(mb_y - d, 0); y < FFMIN(mb_y + d + 1, mi_ctx->b_height); y++)
672 for (x = FFMAX(mb_x - d, 0); x < FFMIN(mb_x + d + 1, mi_ctx->b_width); x++) {
673 Block *nb = &mi_ctx->int_blocks[x + y * mi_ctx->b_width];
674 if (nb->cid > block->cid) {
675 if (nb->cid < c || c == block->cid)
676 c = nb->cid;
677 }
678 }
679
680 if (c == block->cid)
681 c = c_max + 1;
682
683 if (c >= NB_CLUSTERS) {
684 continue;
685 }
686
687 cluster_new = &mi_ctx->clusters[c];
688 cluster_new->sum[0] += mv_x;
689 cluster_new->sum[1] += mv_y;
690 cluster->sum[0] -= mv_x;
691 cluster->sum[1] -= mv_y;
692 cluster_new->nb++;
693 cluster->nb--;
694
695 c_max = FFMAX(c_max, c);
696 block->cid = c;
697
698 changed = 1;
699 }
700 }
701 } while (changed);
702
703 /* find boundaries */
704 for (mb_y = 0; mb_y < mi_ctx->b_height; mb_y++)
705 for (mb_x = 0; mb_x < mi_ctx->b_width; mb_x++) {
706 block = &mi_ctx->int_blocks[mb_x + mb_y * mi_ctx->b_width];
707 for (y = FFMAX(mb_y - 1, 0); y < FFMIN(mb_y + 2, mi_ctx->b_height); y++)
708 for (x = FFMAX(mb_x - 1, 0); x < FFMIN(mb_x + 2, mi_ctx->b_width); x++) {
709 dx = x - mb_x;
710 dy = y - mb_y;
711
712 if ((x - mb_x) && (y - mb_y) || !dx && !dy)
713 continue;
714
715 if (!mb_x || !mb_y || mb_x == mi_ctx->b_width - 1 || mb_y == mi_ctx->b_height - 1)
716 continue;
717
718 if (block->cid != mi_ctx->int_blocks[x + y * mi_ctx->b_width].cid) {
719 if (!dx && block->cid == mi_ctx->int_blocks[x + (mb_y - dy) * mi_ctx->b_width].cid ||
720 !dy && block->cid == mi_ctx->int_blocks[(mb_x - dx) + y * mi_ctx->b_width].cid) {
721 if (ret = var_size_bme(mi_ctx, block, mb_x << mi_ctx->log2_mb_size, mb_y << mi_ctx->log2_mb_size, mi_ctx->log2_mb_size))
722 return ret;
723 }
724 }
725 }
726 }
727
728 return 0;
729 }
730
inject_frame(AVFilterLink * inlink,AVFrame * avf_in)731 static int inject_frame(AVFilterLink *inlink, AVFrame *avf_in)
732 {
733 AVFilterContext *ctx = inlink->dst;
734 MIContext *mi_ctx = ctx->priv;
735 Frame frame_tmp;
736 int mb_x, mb_y, dir;
737
738 av_frame_free(&mi_ctx->frames[0].avf);
739 frame_tmp = mi_ctx->frames[0];
740 memmove(&mi_ctx->frames[0], &mi_ctx->frames[1], sizeof(mi_ctx->frames[0]) * (NB_FRAMES - 1));
741 mi_ctx->frames[NB_FRAMES - 1] = frame_tmp;
742 mi_ctx->frames[NB_FRAMES - 1].avf = avf_in;
743
744 if (mi_ctx->mi_mode == MI_MODE_MCI) {
745
746 if (mi_ctx->me_method == AV_ME_METHOD_EPZS) {
747 mi_ctx->mv_table[2] = memcpy(mi_ctx->mv_table[2], mi_ctx->mv_table[1], sizeof(*mi_ctx->mv_table[1]) * mi_ctx->b_count);
748 mi_ctx->mv_table[1] = memcpy(mi_ctx->mv_table[1], mi_ctx->mv_table[0], sizeof(*mi_ctx->mv_table[0]) * mi_ctx->b_count);
749 }
750
751 if (mi_ctx->me_mode == ME_MODE_BIDIR) {
752
753 if (mi_ctx->frames[1].avf) {
754 for (dir = 0; dir < 2; dir++) {
755 mi_ctx->me_ctx.linesize = mi_ctx->frames[2].avf->linesize[0];
756 mi_ctx->me_ctx.data_cur = mi_ctx->frames[2].avf->data[0];
757 mi_ctx->me_ctx.data_ref = mi_ctx->frames[dir ? 3 : 1].avf->data[0];
758
759 for (mb_y = 0; mb_y < mi_ctx->b_height; mb_y++)
760 for (mb_x = 0; mb_x < mi_ctx->b_width; mb_x++)
761 search_mv(mi_ctx, mi_ctx->frames[2].blocks, mb_x, mb_y, dir);
762 }
763 }
764
765 } else if (mi_ctx->me_mode == ME_MODE_BILAT) {
766 Block *block;
767 int i, ret;
768
769 if (!mi_ctx->frames[0].avf)
770 return 0;
771
772 mi_ctx->me_ctx.linesize = mi_ctx->frames[0].avf->linesize[0];
773 mi_ctx->me_ctx.data_cur = mi_ctx->frames[1].avf->data[0];
774 mi_ctx->me_ctx.data_ref = mi_ctx->frames[2].avf->data[0];
775
776 bilateral_me(mi_ctx);
777
778 if (mi_ctx->mc_mode == MC_MODE_AOBMC) {
779
780 for (mb_y = 0; mb_y < mi_ctx->b_height; mb_y++)
781 for (mb_x = 0; mb_x < mi_ctx->b_width; mb_x++) {
782 int x_mb = mb_x << mi_ctx->log2_mb_size;
783 int y_mb = mb_y << mi_ctx->log2_mb_size;
784 block = &mi_ctx->int_blocks[mb_x + mb_y * mi_ctx->b_width];
785
786 block->sbad = get_sbad(&mi_ctx->me_ctx, x_mb, y_mb, x_mb + block->mvs[0][0], y_mb + block->mvs[0][1]);
787 }
788 }
789
790 if (mi_ctx->vsbmc) {
791
792 for (i = 0; i < NB_CLUSTERS; i++) {
793 mi_ctx->clusters[i].sum[0] = 0;
794 mi_ctx->clusters[i].sum[1] = 0;
795 mi_ctx->clusters[i].nb = 0;
796 }
797
798 for (mb_y = 0; mb_y < mi_ctx->b_height; mb_y++)
799 for (mb_x = 0; mb_x < mi_ctx->b_width; mb_x++) {
800 block = &mi_ctx->int_blocks[mb_x + mb_y * mi_ctx->b_width];
801
802 mi_ctx->clusters[0].sum[0] += block->mvs[0][0];
803 mi_ctx->clusters[0].sum[1] += block->mvs[0][1];
804 }
805
806 mi_ctx->clusters[0].nb = mi_ctx->b_count;
807
808 if (ret = cluster_mvs(mi_ctx))
809 return ret;
810 }
811 }
812 }
813
814 return 0;
815 }
816
detect_scene_change(AVFilterContext * ctx)817 static int detect_scene_change(AVFilterContext *ctx)
818 {
819 MIContext *mi_ctx = ctx->priv;
820 AVFilterLink *input = ctx->inputs[0];
821 uint8_t *p1 = mi_ctx->frames[1].avf->data[0];
822 ptrdiff_t linesize1 = mi_ctx->frames[1].avf->linesize[0];
823 uint8_t *p2 = mi_ctx->frames[2].avf->data[0];
824 ptrdiff_t linesize2 = mi_ctx->frames[2].avf->linesize[0];
825
826 if (mi_ctx->scd_method == SCD_METHOD_FDIFF) {
827 double ret = 0, mafd, diff;
828 uint64_t sad;
829 mi_ctx->sad(p1, linesize1, p2, linesize2, input->w, input->h, &sad);
830 emms_c();
831 mafd = (double) sad * 100.0 / (input->h * input->w) / (1 << mi_ctx->bitdepth);
832 diff = fabs(mafd - mi_ctx->prev_mafd);
833 ret = av_clipf(FFMIN(mafd, diff), 0, 100.0);
834 mi_ctx->prev_mafd = mafd;
835
836 return ret >= mi_ctx->scd_threshold;
837 }
838
839 return 0;
840 }
841
842 #define ADD_PIXELS(b_weight, mv_x, mv_y)\
843 do {\
844 if (!b_weight || pixel_refs->nb + 1 >= NB_PIXEL_MVS)\
845 continue;\
846 pixel_refs->refs[pixel_refs->nb] = 1;\
847 pixel_weights->weights[pixel_refs->nb] = b_weight * (ALPHA_MAX - alpha);\
848 pixel_mvs->mvs[pixel_refs->nb][0] = av_clip((mv_x * alpha) / ALPHA_MAX, x_min, x_max);\
849 pixel_mvs->mvs[pixel_refs->nb][1] = av_clip((mv_y * alpha) / ALPHA_MAX, y_min, y_max);\
850 pixel_refs->nb++;\
851 pixel_refs->refs[pixel_refs->nb] = 2;\
852 pixel_weights->weights[pixel_refs->nb] = b_weight * alpha;\
853 pixel_mvs->mvs[pixel_refs->nb][0] = av_clip(-mv_x * (ALPHA_MAX - alpha) / ALPHA_MAX, x_min, x_max);\
854 pixel_mvs->mvs[pixel_refs->nb][1] = av_clip(-mv_y * (ALPHA_MAX - alpha) / ALPHA_MAX, y_min, y_max);\
855 pixel_refs->nb++;\
856 } while(0)
857
bidirectional_obmc(MIContext * mi_ctx,int alpha)858 static void bidirectional_obmc(MIContext *mi_ctx, int alpha)
859 {
860 int x, y;
861 int width = mi_ctx->frames[0].avf->width;
862 int height = mi_ctx->frames[0].avf->height;
863 int mb_y, mb_x, dir;
864
865 for (y = 0; y < height; y++)
866 for (x = 0; x < width; x++)
867 mi_ctx->pixel_refs[x + y * width].nb = 0;
868
869 for (dir = 0; dir < 2; dir++)
870 for (mb_y = 0; mb_y < mi_ctx->b_height; mb_y++)
871 for (mb_x = 0; mb_x < mi_ctx->b_width; mb_x++) {
872 int a = dir ? alpha : (ALPHA_MAX - alpha);
873 int mv_x = mi_ctx->frames[2 - dir].blocks[mb_x + mb_y * mi_ctx->b_width].mvs[dir][0];
874 int mv_y = mi_ctx->frames[2 - dir].blocks[mb_x + mb_y * mi_ctx->b_width].mvs[dir][1];
875 int start_x, start_y;
876 int startc_x, startc_y, endc_x, endc_y;
877
878 start_x = (mb_x << mi_ctx->log2_mb_size) - mi_ctx->mb_size / 2 + mv_x * a / ALPHA_MAX;
879 start_y = (mb_y << mi_ctx->log2_mb_size) - mi_ctx->mb_size / 2 + mv_y * a / ALPHA_MAX;
880
881 startc_x = av_clip(start_x, 0, width - 1);
882 startc_y = av_clip(start_y, 0, height - 1);
883 endc_x = av_clip(start_x + (2 << mi_ctx->log2_mb_size), 0, width - 1);
884 endc_y = av_clip(start_y + (2 << mi_ctx->log2_mb_size), 0, height - 1);
885
886 if (dir) {
887 mv_x = -mv_x;
888 mv_y = -mv_y;
889 }
890
891 for (y = startc_y; y < endc_y; y++) {
892 int y_min = -y;
893 int y_max = height - y - 1;
894 for (x = startc_x; x < endc_x; x++) {
895 int x_min = -x;
896 int x_max = width - x - 1;
897 int obmc_weight = obmc_tab_linear[4 - mi_ctx->log2_mb_size][(x - start_x) + ((y - start_y) << (mi_ctx->log2_mb_size + 1))];
898 PixelMVS *pixel_mvs = &mi_ctx->pixel_mvs[x + y * width];
899 PixelWeights *pixel_weights = &mi_ctx->pixel_weights[x + y * width];
900 PixelRefs *pixel_refs = &mi_ctx->pixel_refs[x + y * width];
901
902 ADD_PIXELS(obmc_weight, mv_x, mv_y);
903 }
904 }
905 }
906 }
907
set_frame_data(MIContext * mi_ctx,int alpha,AVFrame * avf_out)908 static void set_frame_data(MIContext *mi_ctx, int alpha, AVFrame *avf_out)
909 {
910 int x, y, plane;
911
912 for (plane = 0; plane < mi_ctx->nb_planes; plane++) {
913 int width = avf_out->width;
914 int height = avf_out->height;
915 int chroma = plane == 1 || plane == 2;
916
917 for (y = 0; y < height; y++)
918 for (x = 0; x < width; x++) {
919 int x_mv, y_mv;
920 int weight_sum = 0;
921 int i, val = 0;
922 PixelMVS *pixel_mvs = &mi_ctx->pixel_mvs[x + y * avf_out->width];
923 PixelWeights *pixel_weights = &mi_ctx->pixel_weights[x + y * avf_out->width];
924 PixelRefs *pixel_refs = &mi_ctx->pixel_refs[x + y * avf_out->width];
925
926 for (i = 0; i < pixel_refs->nb; i++)
927 weight_sum += pixel_weights->weights[i];
928
929 if (!weight_sum || !pixel_refs->nb) {
930 pixel_weights->weights[0] = ALPHA_MAX - alpha;
931 pixel_refs->refs[0] = 1;
932 pixel_mvs->mvs[0][0] = 0;
933 pixel_mvs->mvs[0][1] = 0;
934 pixel_weights->weights[1] = alpha;
935 pixel_refs->refs[1] = 2;
936 pixel_mvs->mvs[1][0] = 0;
937 pixel_mvs->mvs[1][1] = 0;
938 pixel_refs->nb = 2;
939
940 weight_sum = ALPHA_MAX;
941 }
942
943 for (i = 0; i < pixel_refs->nb; i++) {
944 Frame *frame = &mi_ctx->frames[pixel_refs->refs[i]];
945 if (chroma) {
946 x_mv = (x >> mi_ctx->log2_chroma_w) + pixel_mvs->mvs[i][0] / (1 << mi_ctx->log2_chroma_w);
947 y_mv = (y >> mi_ctx->log2_chroma_h) + pixel_mvs->mvs[i][1] / (1 << mi_ctx->log2_chroma_h);
948 } else {
949 x_mv = x + pixel_mvs->mvs[i][0];
950 y_mv = y + pixel_mvs->mvs[i][1];
951 }
952
953 val += pixel_weights->weights[i] * frame->avf->data[plane][x_mv + y_mv * frame->avf->linesize[plane]];
954 }
955
956 val = ROUNDED_DIV(val, weight_sum);
957
958 if (chroma)
959 avf_out->data[plane][(x >> mi_ctx->log2_chroma_w) + (y >> mi_ctx->log2_chroma_h) * avf_out->linesize[plane]] = val;
960 else
961 avf_out->data[plane][x + y * avf_out->linesize[plane]] = val;
962 }
963 }
964 }
965
var_size_bmc(MIContext * mi_ctx,Block * block,int x_mb,int y_mb,int n,int alpha)966 static void var_size_bmc(MIContext *mi_ctx, Block *block, int x_mb, int y_mb, int n, int alpha)
967 {
968 int sb_x, sb_y;
969 int width = mi_ctx->frames[0].avf->width;
970 int height = mi_ctx->frames[0].avf->height;
971
972 for (sb_y = 0; sb_y < 2; sb_y++)
973 for (sb_x = 0; sb_x < 2; sb_x++) {
974 Block *sb = &block->subs[sb_x + sb_y * 2];
975
976 if (sb->sb)
977 var_size_bmc(mi_ctx, sb, x_mb + (sb_x << (n - 1)), y_mb + (sb_y << (n - 1)), n - 1, alpha);
978 else {
979 int x, y;
980 int mv_x = sb->mvs[0][0] * 2;
981 int mv_y = sb->mvs[0][1] * 2;
982
983 int start_x = x_mb + (sb_x << (n - 1));
984 int start_y = y_mb + (sb_y << (n - 1));
985 int end_x = start_x + (1 << (n - 1));
986 int end_y = start_y + (1 << (n - 1));
987
988 for (y = start_y; y < end_y; y++) {
989 int y_min = -y;
990 int y_max = height - y - 1;
991 for (x = start_x; x < end_x; x++) {
992 int x_min = -x;
993 int x_max = width - x - 1;
994 PixelMVS *pixel_mvs = &mi_ctx->pixel_mvs[x + y * width];
995 PixelWeights *pixel_weights = &mi_ctx->pixel_weights[x + y * width];
996 PixelRefs *pixel_refs = &mi_ctx->pixel_refs[x + y * width];
997
998 ADD_PIXELS(PX_WEIGHT_MAX, mv_x, mv_y);
999 }
1000 }
1001 }
1002 }
1003 }
1004
bilateral_obmc(MIContext * mi_ctx,Block * block,int mb_x,int mb_y,int alpha)1005 static void bilateral_obmc(MIContext *mi_ctx, Block *block, int mb_x, int mb_y, int alpha)
1006 {
1007 int x, y;
1008 int width = mi_ctx->frames[0].avf->width;
1009 int height = mi_ctx->frames[0].avf->height;
1010
1011 Block *nb;
1012 int nb_x, nb_y;
1013 uint64_t sbads[9];
1014
1015 int mv_x = block->mvs[0][0] * 2;
1016 int mv_y = block->mvs[0][1] * 2;
1017 int start_x, start_y;
1018 int startc_x, startc_y, endc_x, endc_y;
1019
1020 if (mi_ctx->mc_mode == MC_MODE_AOBMC)
1021 for (nb_y = FFMAX(0, mb_y - 1); nb_y < FFMIN(mb_y + 2, mi_ctx->b_height); nb_y++)
1022 for (nb_x = FFMAX(0, mb_x - 1); nb_x < FFMIN(mb_x + 2, mi_ctx->b_width); nb_x++) {
1023 int x_nb = nb_x << mi_ctx->log2_mb_size;
1024 int y_nb = nb_y << mi_ctx->log2_mb_size;
1025
1026 if (nb_x - mb_x || nb_y - mb_y)
1027 sbads[nb_x - mb_x + 1 + (nb_y - mb_y + 1) * 3] = get_sbad(&mi_ctx->me_ctx, x_nb, y_nb, x_nb + block->mvs[0][0], y_nb + block->mvs[0][1]);
1028 }
1029
1030 start_x = (mb_x << mi_ctx->log2_mb_size) - mi_ctx->mb_size / 2;
1031 start_y = (mb_y << mi_ctx->log2_mb_size) - mi_ctx->mb_size / 2;
1032
1033 startc_x = av_clip(start_x, 0, width - 1);
1034 startc_y = av_clip(start_y, 0, height - 1);
1035 endc_x = av_clip(start_x + (2 << mi_ctx->log2_mb_size), 0, width - 1);
1036 endc_y = av_clip(start_y + (2 << mi_ctx->log2_mb_size), 0, height - 1);
1037
1038 for (y = startc_y; y < endc_y; y++) {
1039 int y_min = -y;
1040 int y_max = height - y - 1;
1041 for (x = startc_x; x < endc_x; x++) {
1042 int x_min = -x;
1043 int x_max = width - x - 1;
1044 int obmc_weight = obmc_tab_linear[4 - mi_ctx->log2_mb_size][(x - start_x) + ((y - start_y) << (mi_ctx->log2_mb_size + 1))];
1045 PixelMVS *pixel_mvs = &mi_ctx->pixel_mvs[x + y * width];
1046 PixelWeights *pixel_weights = &mi_ctx->pixel_weights[x + y * width];
1047 PixelRefs *pixel_refs = &mi_ctx->pixel_refs[x + y * width];
1048
1049 if (mi_ctx->mc_mode == MC_MODE_AOBMC) {
1050 nb_x = (((x - start_x) >> (mi_ctx->log2_mb_size - 1)) * 2 - 3) / 2;
1051 nb_y = (((y - start_y) >> (mi_ctx->log2_mb_size - 1)) * 2 - 3) / 2;
1052
1053 if (nb_x || nb_y) {
1054 uint64_t sbad = sbads[nb_x + 1 + (nb_y + 1) * 3];
1055 nb = &mi_ctx->int_blocks[mb_x + nb_x + (mb_y + nb_y) * mi_ctx->b_width];
1056
1057 if (sbad && sbad != UINT64_MAX && nb->sbad != UINT64_MAX) {
1058 int phi = av_clip(ALPHA_MAX * nb->sbad / sbad, 0, ALPHA_MAX);
1059 obmc_weight = obmc_weight * phi / ALPHA_MAX;
1060 }
1061 }
1062 }
1063
1064 ADD_PIXELS(obmc_weight, mv_x, mv_y);
1065 }
1066 }
1067 }
1068
interpolate(AVFilterLink * inlink,AVFrame * avf_out)1069 static void interpolate(AVFilterLink *inlink, AVFrame *avf_out)
1070 {
1071 AVFilterContext *ctx = inlink->dst;
1072 AVFilterLink *outlink = ctx->outputs[0];
1073 MIContext *mi_ctx = ctx->priv;
1074 int x, y;
1075 int plane, alpha;
1076 int64_t pts;
1077
1078 pts = av_rescale(avf_out->pts, (int64_t) ALPHA_MAX * outlink->time_base.num * inlink->time_base.den,
1079 (int64_t) outlink->time_base.den * inlink->time_base.num);
1080
1081 alpha = (pts - mi_ctx->frames[1].avf->pts * ALPHA_MAX) / (mi_ctx->frames[2].avf->pts - mi_ctx->frames[1].avf->pts);
1082 alpha = av_clip(alpha, 0, ALPHA_MAX);
1083
1084 if (alpha == 0 || alpha == ALPHA_MAX) {
1085 av_frame_copy(avf_out, alpha ? mi_ctx->frames[2].avf : mi_ctx->frames[1].avf);
1086 return;
1087 }
1088
1089 if (mi_ctx->scene_changed) {
1090 av_log(ctx, AV_LOG_DEBUG, "scene changed, input pts %"PRId64"\n", mi_ctx->frames[1].avf->pts);
1091 /* duplicate frame */
1092 av_frame_copy(avf_out, alpha > ALPHA_MAX / 2 ? mi_ctx->frames[2].avf : mi_ctx->frames[1].avf);
1093 return;
1094 }
1095
1096 switch(mi_ctx->mi_mode) {
1097 case MI_MODE_DUP:
1098 av_frame_copy(avf_out, alpha > ALPHA_MAX / 2 ? mi_ctx->frames[2].avf : mi_ctx->frames[1].avf);
1099
1100 break;
1101 case MI_MODE_BLEND:
1102 for (plane = 0; plane < mi_ctx->nb_planes; plane++) {
1103 int width = avf_out->width;
1104 int height = avf_out->height;
1105
1106 if (plane == 1 || plane == 2) {
1107 width = AV_CEIL_RSHIFT(width, mi_ctx->log2_chroma_w);
1108 height = AV_CEIL_RSHIFT(height, mi_ctx->log2_chroma_h);
1109 }
1110
1111 for (y = 0; y < height; y++) {
1112 for (x = 0; x < width; x++) {
1113 avf_out->data[plane][x + y * avf_out->linesize[plane]] =
1114 (alpha * mi_ctx->frames[2].avf->data[plane][x + y * mi_ctx->frames[2].avf->linesize[plane]] +
1115 (ALPHA_MAX - alpha) * mi_ctx->frames[1].avf->data[plane][x + y * mi_ctx->frames[1].avf->linesize[plane]] + 512) >> 10;
1116 }
1117 }
1118 }
1119
1120 break;
1121 case MI_MODE_MCI:
1122 if (mi_ctx->me_mode == ME_MODE_BIDIR) {
1123 bidirectional_obmc(mi_ctx, alpha);
1124 set_frame_data(mi_ctx, alpha, avf_out);
1125
1126 } else if (mi_ctx->me_mode == ME_MODE_BILAT) {
1127 int mb_x, mb_y;
1128 Block *block;
1129
1130 for (y = 0; y < mi_ctx->frames[0].avf->height; y++)
1131 for (x = 0; x < mi_ctx->frames[0].avf->width; x++)
1132 mi_ctx->pixel_refs[x + y * mi_ctx->frames[0].avf->width].nb = 0;
1133
1134 for (mb_y = 0; mb_y < mi_ctx->b_height; mb_y++)
1135 for (mb_x = 0; mb_x < mi_ctx->b_width; mb_x++) {
1136 block = &mi_ctx->int_blocks[mb_x + mb_y * mi_ctx->b_width];
1137
1138 if (block->sb)
1139 var_size_bmc(mi_ctx, block, mb_x << mi_ctx->log2_mb_size, mb_y << mi_ctx->log2_mb_size, mi_ctx->log2_mb_size, alpha);
1140
1141 bilateral_obmc(mi_ctx, block, mb_x, mb_y, alpha);
1142
1143 }
1144
1145 set_frame_data(mi_ctx, alpha, avf_out);
1146 }
1147
1148 break;
1149 }
1150 }
1151
filter_frame(AVFilterLink * inlink,AVFrame * avf_in)1152 static int filter_frame(AVFilterLink *inlink, AVFrame *avf_in)
1153 {
1154 AVFilterContext *ctx = inlink->dst;
1155 AVFilterLink *outlink = ctx->outputs[0];
1156 MIContext *mi_ctx = ctx->priv;
1157 int ret;
1158
1159 if (avf_in->pts == AV_NOPTS_VALUE) {
1160 ret = ff_filter_frame(ctx->outputs[0], avf_in);
1161 return ret;
1162 }
1163
1164 if (!mi_ctx->frames[NB_FRAMES - 1].avf || avf_in->pts < mi_ctx->frames[NB_FRAMES - 1].avf->pts) {
1165 av_log(ctx, AV_LOG_VERBOSE, "Initializing out pts from input pts %"PRId64"\n", avf_in->pts);
1166 mi_ctx->out_pts = av_rescale_q(avf_in->pts, inlink->time_base, outlink->time_base);
1167 }
1168
1169 if (!mi_ctx->frames[NB_FRAMES - 1].avf)
1170 if (ret = inject_frame(inlink, av_frame_clone(avf_in)))
1171 return ret;
1172
1173 if (ret = inject_frame(inlink, avf_in))
1174 return ret;
1175
1176 if (!mi_ctx->frames[0].avf)
1177 return 0;
1178
1179 mi_ctx->scene_changed = detect_scene_change(ctx);
1180
1181 for (;;) {
1182 AVFrame *avf_out;
1183
1184 if (av_compare_ts(mi_ctx->out_pts, outlink->time_base, mi_ctx->frames[2].avf->pts, inlink->time_base) > 0)
1185 break;
1186
1187 if (!(avf_out = ff_get_video_buffer(ctx->outputs[0], inlink->w, inlink->h)))
1188 return AVERROR(ENOMEM);
1189
1190 av_frame_copy_props(avf_out, mi_ctx->frames[NB_FRAMES - 1].avf);
1191 avf_out->pts = mi_ctx->out_pts++;
1192
1193 interpolate(inlink, avf_out);
1194
1195 if ((ret = ff_filter_frame(ctx->outputs[0], avf_out)) < 0)
1196 return ret;
1197 }
1198
1199 return 0;
1200 }
1201
free_blocks(Block * block,int sb)1202 static av_cold void free_blocks(Block *block, int sb)
1203 {
1204 if (block->subs)
1205 free_blocks(block->subs, 1);
1206 if (sb)
1207 av_freep(&block);
1208 }
1209
uninit(AVFilterContext * ctx)1210 static av_cold void uninit(AVFilterContext *ctx)
1211 {
1212 MIContext *mi_ctx = ctx->priv;
1213 int i, m;
1214
1215 av_freep(&mi_ctx->pixel_mvs);
1216 av_freep(&mi_ctx->pixel_weights);
1217 av_freep(&mi_ctx->pixel_refs);
1218 if (mi_ctx->int_blocks)
1219 for (m = 0; m < mi_ctx->b_count; m++)
1220 free_blocks(&mi_ctx->int_blocks[m], 0);
1221 av_freep(&mi_ctx->int_blocks);
1222
1223 for (i = 0; i < NB_FRAMES; i++) {
1224 Frame *frame = &mi_ctx->frames[i];
1225 av_freep(&frame->blocks);
1226 av_frame_free(&frame->avf);
1227 }
1228
1229 for (i = 0; i < 3; i++)
1230 av_freep(&mi_ctx->mv_table[i]);
1231 }
1232
1233 static const AVFilterPad minterpolate_inputs[] = {
1234 {
1235 .name = "default",
1236 .type = AVMEDIA_TYPE_VIDEO,
1237 .filter_frame = filter_frame,
1238 .config_props = config_input,
1239 },
1240 };
1241
1242 static const AVFilterPad minterpolate_outputs[] = {
1243 {
1244 .name = "default",
1245 .type = AVMEDIA_TYPE_VIDEO,
1246 .config_props = config_output,
1247 },
1248 };
1249
1250 const AVFilter ff_vf_minterpolate = {
1251 .name = "minterpolate",
1252 .description = NULL_IF_CONFIG_SMALL("Frame rate conversion using Motion Interpolation."),
1253 .priv_size = sizeof(MIContext),
1254 .priv_class = &minterpolate_class,
1255 .uninit = uninit,
1256 FILTER_INPUTS(minterpolate_inputs),
1257 FILTER_OUTPUTS(minterpolate_outputs),
1258 FILTER_PIXFMTS_ARRAY(pix_fmts),
1259 };
1260