1 /*
2 * Copyright (c) 2011 Smartjog S.A.S, Clément Bœsch <clement.boesch@smartjog.com>
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 /**
22 * @file
23 * Potential thumbnail lookup filter to reduce the risk of an inappropriate
24 * selection (such as a black frame) we could get with an absolute seek.
25 *
26 * Simplified version of algorithm by Vadim Zaliva <lord@crocodile.org>.
27 * @see http://notbrainsurgery.livejournal.com/29773.html
28 */
29
30 #include "libavutil/opt.h"
31 #include "libavutil/pixdesc.h"
32 #include "avfilter.h"
33 #include "internal.h"
34
35 #define HIST_SIZE (3*256)
36
37 struct thumb_frame {
38 AVFrame *buf; ///< cached frame
39 int histogram[HIST_SIZE]; ///< RGB color distribution histogram of the frame
40 };
41
42 typedef struct ThumbContext {
43 const AVClass *class;
44 int n; ///< current frame
45 int n_frames; ///< number of frames for analysis
46 struct thumb_frame *frames; ///< the n_frames frames
47 AVRational tb; ///< copy of the input timebase to ease access
48
49 int planewidth[4];
50 int planeheight[4];
51 } ThumbContext;
52
53 #define OFFSET(x) offsetof(ThumbContext, x)
54 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
55
56 static const AVOption thumbnail_options[] = {
57 { "n", "set the frames batch size", OFFSET(n_frames), AV_OPT_TYPE_INT, {.i64=100}, 2, INT_MAX, FLAGS },
58 { NULL }
59 };
60
61 AVFILTER_DEFINE_CLASS(thumbnail);
62
init(AVFilterContext * ctx)63 static av_cold int init(AVFilterContext *ctx)
64 {
65 ThumbContext *s = ctx->priv;
66
67 s->frames = av_calloc(s->n_frames, sizeof(*s->frames));
68 if (!s->frames) {
69 av_log(ctx, AV_LOG_ERROR,
70 "Allocation failure, try to lower the number of frames\n");
71 return AVERROR(ENOMEM);
72 }
73 av_log(ctx, AV_LOG_VERBOSE, "batch size: %d frames\n", s->n_frames);
74 return 0;
75 }
76
77 /**
78 * @brief Compute Sum-square deviation to estimate "closeness".
79 * @param hist color distribution histogram
80 * @param median average color distribution histogram
81 * @return sum of squared errors
82 */
frame_sum_square_err(const int * hist,const double * median)83 static double frame_sum_square_err(const int *hist, const double *median)
84 {
85 int i;
86 double err, sum_sq_err = 0;
87
88 for (i = 0; i < HIST_SIZE; i++) {
89 err = median[i] - (double)hist[i];
90 sum_sq_err += err*err;
91 }
92 return sum_sq_err;
93 }
94
get_best_frame(AVFilterContext * ctx)95 static AVFrame *get_best_frame(AVFilterContext *ctx)
96 {
97 AVFrame *picref;
98 ThumbContext *s = ctx->priv;
99 int i, j, best_frame_idx = 0;
100 int nb_frames = s->n;
101 double avg_hist[HIST_SIZE] = {0}, sq_err, min_sq_err = -1;
102
103 // average histogram of the N frames
104 for (j = 0; j < FF_ARRAY_ELEMS(avg_hist); j++) {
105 for (i = 0; i < nb_frames; i++)
106 avg_hist[j] += (double)s->frames[i].histogram[j];
107 avg_hist[j] /= nb_frames;
108 }
109
110 // find the frame closer to the average using the sum of squared errors
111 for (i = 0; i < nb_frames; i++) {
112 sq_err = frame_sum_square_err(s->frames[i].histogram, avg_hist);
113 if (i == 0 || sq_err < min_sq_err)
114 best_frame_idx = i, min_sq_err = sq_err;
115 }
116
117 // free and reset everything (except the best frame buffer)
118 for (i = 0; i < nb_frames; i++) {
119 memset(s->frames[i].histogram, 0, sizeof(s->frames[i].histogram));
120 if (i != best_frame_idx)
121 av_frame_free(&s->frames[i].buf);
122 }
123 s->n = 0;
124
125 // raise the chosen one
126 picref = s->frames[best_frame_idx].buf;
127 av_log(ctx, AV_LOG_INFO, "frame id #%d (pts_time=%f) selected "
128 "from a set of %d images\n", best_frame_idx,
129 picref->pts * av_q2d(s->tb), nb_frames);
130 s->frames[best_frame_idx].buf = NULL;
131
132 return picref;
133 }
134
filter_frame(AVFilterLink * inlink,AVFrame * frame)135 static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
136 {
137 int i, j;
138 AVFilterContext *ctx = inlink->dst;
139 ThumbContext *s = ctx->priv;
140 AVFilterLink *outlink = ctx->outputs[0];
141 int *hist = s->frames[s->n].histogram;
142 const uint8_t *p = frame->data[0];
143
144 // keep a reference of each frame
145 s->frames[s->n].buf = frame;
146
147 // update current frame histogram
148 switch (inlink->format) {
149 case AV_PIX_FMT_RGB24:
150 case AV_PIX_FMT_BGR24:
151 for (j = 0; j < inlink->h; j++) {
152 for (i = 0; i < inlink->w; i++) {
153 hist[0*256 + p[i*3 ]]++;
154 hist[1*256 + p[i*3 + 1]]++;
155 hist[2*256 + p[i*3 + 2]]++;
156 }
157 p += frame->linesize[0];
158 }
159 break;
160 case AV_PIX_FMT_RGB0:
161 case AV_PIX_FMT_BGR0:
162 case AV_PIX_FMT_RGBA:
163 case AV_PIX_FMT_BGRA:
164 for (j = 0; j < inlink->h; j++) {
165 for (i = 0; i < inlink->w; i++) {
166 hist[0*256 + p[i*4 ]]++;
167 hist[1*256 + p[i*4 + 1]]++;
168 hist[2*256 + p[i*4 + 2]]++;
169 }
170 p += frame->linesize[0];
171 }
172 break;
173 case AV_PIX_FMT_0RGB:
174 case AV_PIX_FMT_0BGR:
175 case AV_PIX_FMT_ARGB:
176 case AV_PIX_FMT_ABGR:
177 for (j = 0; j < inlink->h; j++) {
178 for (i = 0; i < inlink->w; i++) {
179 hist[0*256 + p[i*4 + 1]]++;
180 hist[1*256 + p[i*4 + 2]]++;
181 hist[2*256 + p[i*4 + 3]]++;
182 }
183 p += frame->linesize[0];
184 }
185 break;
186 default:
187 for (int plane = 0; plane < 3; plane++) {
188 const uint8_t *p = frame->data[plane];
189 for (j = 0; j < s->planeheight[plane]; j++) {
190 for (i = 0; i < s->planewidth[plane]; i++)
191 hist[256*plane + p[i]]++;
192 p += frame->linesize[plane];
193 }
194 }
195 break;
196 }
197
198 // no selection until the buffer of N frames is filled up
199 s->n++;
200 if (s->n < s->n_frames)
201 return 0;
202
203 return ff_filter_frame(outlink, get_best_frame(ctx));
204 }
205
uninit(AVFilterContext * ctx)206 static av_cold void uninit(AVFilterContext *ctx)
207 {
208 int i;
209 ThumbContext *s = ctx->priv;
210 for (i = 0; i < s->n_frames && s->frames && s->frames[i].buf; i++)
211 av_frame_free(&s->frames[i].buf);
212 av_freep(&s->frames);
213 }
214
request_frame(AVFilterLink * link)215 static int request_frame(AVFilterLink *link)
216 {
217 AVFilterContext *ctx = link->src;
218 ThumbContext *s = ctx->priv;
219 int ret = ff_request_frame(ctx->inputs[0]);
220
221 if (ret == AVERROR_EOF && s->n) {
222 ret = ff_filter_frame(link, get_best_frame(ctx));
223 if (ret < 0)
224 return ret;
225 ret = AVERROR_EOF;
226 }
227 if (ret < 0)
228 return ret;
229 return 0;
230 }
231
config_props(AVFilterLink * inlink)232 static int config_props(AVFilterLink *inlink)
233 {
234 AVFilterContext *ctx = inlink->dst;
235 ThumbContext *s = ctx->priv;
236 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
237
238 s->tb = inlink->time_base;
239 s->planewidth[1] = s->planewidth[2] = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w);
240 s->planewidth[0] = s->planewidth[3] = inlink->w;
241 s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
242 s->planeheight[0] = s->planeheight[3] = inlink->h;
243
244 return 0;
245 }
246
query_formats(AVFilterContext * ctx)247 static int query_formats(AVFilterContext *ctx)
248 {
249 static const enum AVPixelFormat pix_fmts[] = {
250 AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,
251 AV_PIX_FMT_RGBA, AV_PIX_FMT_BGRA,
252 AV_PIX_FMT_RGB0, AV_PIX_FMT_BGR0,
253 AV_PIX_FMT_ABGR, AV_PIX_FMT_ARGB,
254 AV_PIX_FMT_0BGR, AV_PIX_FMT_0RGB,
255 AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
256 AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
257 AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,
258 AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P,
259 AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P,
260 AV_PIX_FMT_YUVJ411P,
261 AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P,
262 AV_PIX_FMT_GBRP, AV_PIX_FMT_GBRAP,
263 AV_PIX_FMT_NONE
264 };
265 AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
266 if (!fmts_list)
267 return AVERROR(ENOMEM);
268 return ff_set_common_formats(ctx, fmts_list);
269 }
270
271 static const AVFilterPad thumbnail_inputs[] = {
272 {
273 .name = "default",
274 .type = AVMEDIA_TYPE_VIDEO,
275 .config_props = config_props,
276 .filter_frame = filter_frame,
277 },
278 { NULL }
279 };
280
281 static const AVFilterPad thumbnail_outputs[] = {
282 {
283 .name = "default",
284 .type = AVMEDIA_TYPE_VIDEO,
285 .request_frame = request_frame,
286 },
287 { NULL }
288 };
289
290 AVFilter ff_vf_thumbnail = {
291 .name = "thumbnail",
292 .description = NULL_IF_CONFIG_SMALL("Select the most representative frame in a given sequence of consecutive frames."),
293 .priv_size = sizeof(ThumbContext),
294 .init = init,
295 .uninit = uninit,
296 .query_formats = query_formats,
297 .inputs = thumbnail_inputs,
298 .outputs = thumbnail_outputs,
299 .priv_class = &thumbnail_class,
300 .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC,
301 };
302