1 /*
2 * Copyright (c) 2019 Vladimir Panteleev
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #include <float.h>
22
23 #include "libavutil/imgutils.h"
24 #include "libavutil/opt.h"
25 #include "libavutil/pixdesc.h"
26 #include "avfilter.h"
27
28 #include "formats.h"
29 #include "internal.h"
30 #include "video.h"
31
32 #define MAX_FRAMES 240
33 #define GRID_SIZE 8
34 #define NUM_CHANNELS 3
35
36 typedef struct PhotosensitivityFrame {
37 uint8_t grid[GRID_SIZE][GRID_SIZE][4];
38 } PhotosensitivityFrame;
39
40 typedef struct PhotosensitivityContext {
41 const AVClass *class;
42
43 int nb_frames;
44 int skip;
45 float threshold_multiplier;
46 int bypass;
47
48 int badness_threshold;
49
50 /* Circular buffer */
51 int history[MAX_FRAMES];
52 int history_pos;
53
54 PhotosensitivityFrame last_frame_e;
55 AVFrame *last_frame_av;
56 } PhotosensitivityContext;
57
58 #define OFFSET(x) offsetof(PhotosensitivityContext, x)
59 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
60
61 static const AVOption photosensitivity_options[] = {
62 { "frames", "set how many frames to use", OFFSET(nb_frames), AV_OPT_TYPE_INT, {.i64=30}, 2, MAX_FRAMES, FLAGS },
63 { "f", "set how many frames to use", OFFSET(nb_frames), AV_OPT_TYPE_INT, {.i64=30}, 2, MAX_FRAMES, FLAGS },
64 { "threshold", "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0.1, FLT_MAX, FLAGS },
65 { "t", "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0.1, FLT_MAX, FLAGS },
66 { "skip", "set pixels to skip when sampling frames", OFFSET(skip), AV_OPT_TYPE_INT, {.i64=1}, 1, 1024, FLAGS },
67 { "bypass", "leave frames unchanged", OFFSET(bypass), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS },
68 { NULL }
69 };
70
71 AVFILTER_DEFINE_CLASS(photosensitivity);
72
query_formats(AVFilterContext * ctx)73 static int query_formats(AVFilterContext *ctx)
74 {
75 static const enum AVPixelFormat pixel_fmts[] = {
76 AV_PIX_FMT_RGB24,
77 AV_PIX_FMT_BGR24,
78 AV_PIX_FMT_NONE
79 };
80 AVFilterFormats *formats = ff_make_format_list(pixel_fmts);
81 if (!formats)
82 return AVERROR(ENOMEM);
83 return ff_set_common_formats(ctx, formats);
84 }
85
86 typedef struct ThreadData_convert_frame
87 {
88 AVFrame *in;
89 PhotosensitivityFrame *out;
90 int skip;
91 } ThreadData_convert_frame;
92
93 #define NUM_CELLS (GRID_SIZE * GRID_SIZE)
94
convert_frame_partial(AVFilterContext * ctx,void * arg,int jobnr,int nb_jobs)95 static int convert_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
96 {
97 int cell, gx, gy, x0, x1, y0, y1, x, y, c, area;
98 int sum[NUM_CHANNELS];
99 const uint8_t *p;
100
101 ThreadData_convert_frame *td = arg;
102
103 const int slice_start = (NUM_CELLS * jobnr) / nb_jobs;
104 const int slice_end = (NUM_CELLS * (jobnr+1)) / nb_jobs;
105
106 int width = td->in->width, height = td->in->height, linesize = td->in->linesize[0], skip = td->skip;
107 const uint8_t *data = td->in->data[0];
108
109 for (cell = slice_start; cell < slice_end; cell++) {
110 gx = cell % GRID_SIZE;
111 gy = cell / GRID_SIZE;
112
113 x0 = width * gx / GRID_SIZE;
114 x1 = width * (gx+1) / GRID_SIZE;
115 y0 = height * gy / GRID_SIZE;
116 y1 = height * (gy+1) / GRID_SIZE;
117
118 for (c = 0; c < NUM_CHANNELS; c++) {
119 sum[c] = 0;
120 }
121 for (y = y0; y < y1; y += skip) {
122 p = data + y * linesize + x0 * NUM_CHANNELS;
123 for (x = x0; x < x1; x += skip) {
124 //av_log(NULL, AV_LOG_VERBOSE, "%d %d %d : (%d,%d) (%d,%d) -> %d,%d | *%d\n", c, gx, gy, x0, y0, x1, y1, x, y, (int)row);
125 sum[0] += p[0];
126 sum[1] += p[1];
127 sum[2] += p[2];
128 p += NUM_CHANNELS * skip;
129 // TODO: variable size
130 }
131 }
132
133 area = ((x1 - x0 + skip - 1) / skip) * ((y1 - y0 + skip - 1) / skip);
134 for (c = 0; c < NUM_CHANNELS; c++) {
135 if (area)
136 sum[c] /= area;
137 td->out->grid[gy][gx][c] = sum[c];
138 }
139 }
140 return 0;
141 }
142
convert_frame(AVFilterContext * ctx,AVFrame * in,PhotosensitivityFrame * out,int skip)143 static void convert_frame(AVFilterContext *ctx, AVFrame *in, PhotosensitivityFrame *out, int skip)
144 {
145 ThreadData_convert_frame td;
146 td.in = in;
147 td.out = out;
148 td.skip = skip;
149 ctx->internal->execute(ctx, convert_frame_partial, &td, NULL, FFMIN(NUM_CELLS, ff_filter_get_nb_threads(ctx)));
150 }
151
152 typedef struct ThreadData_blend_frame
153 {
154 AVFrame *target;
155 AVFrame *source;
156 uint16_t s_mul;
157 } ThreadData_blend_frame;
158
blend_frame_partial(AVFilterContext * ctx,void * arg,int jobnr,int nb_jobs)159 static int blend_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
160 {
161 int x, y;
162 uint8_t *t, *s;
163
164 ThreadData_blend_frame *td = arg;
165 const uint16_t s_mul = td->s_mul;
166 const uint16_t t_mul = 0x100 - s_mul;
167 const int slice_start = (td->target->height * jobnr) / nb_jobs;
168 const int slice_end = (td->target->height * (jobnr+1)) / nb_jobs;
169 const int linesize = td->target->linesize[0];
170
171 for (y = slice_start; y < slice_end; y++) {
172 t = td->target->data[0] + y * td->target->linesize[0];
173 s = td->source->data[0] + y * td->source->linesize[0];
174 for (x = 0; x < linesize; x++) {
175 *t = (*t * t_mul + *s * s_mul) >> 8;
176 t++; s++;
177 }
178 }
179 return 0;
180 }
181
blend_frame(AVFilterContext * ctx,AVFrame * target,AVFrame * source,float factor)182 static void blend_frame(AVFilterContext *ctx, AVFrame *target, AVFrame *source, float factor)
183 {
184 ThreadData_blend_frame td;
185 td.target = target;
186 td.source = source;
187 td.s_mul = (uint16_t)(factor * 0x100);
188 ctx->internal->execute(ctx, blend_frame_partial, &td, NULL, FFMIN(ctx->outputs[0]->h, ff_filter_get_nb_threads(ctx)));
189 }
190
get_badness(PhotosensitivityFrame * a,PhotosensitivityFrame * b)191 static int get_badness(PhotosensitivityFrame *a, PhotosensitivityFrame *b)
192 {
193 int badness, x, y, c;
194 badness = 0;
195 for (c = 0; c < NUM_CHANNELS; c++) {
196 for (y = 0; y < GRID_SIZE; y++) {
197 for (x = 0; x < GRID_SIZE; x++) {
198 badness += abs((int)a->grid[y][x][c] - (int)b->grid[y][x][c]);
199 //av_log(NULL, AV_LOG_VERBOSE, "%d - %d -> %d \n", a->grid[y][x], b->grid[y][x], badness);
200 //av_log(NULL, AV_LOG_VERBOSE, "%d -> %d \n", abs((int)a->grid[y][x] - (int)b->grid[y][x]), badness);
201 }
202 }
203 }
204 return badness;
205 }
206
config_input(AVFilterLink * inlink)207 static int config_input(AVFilterLink *inlink)
208 {
209 /* const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); */
210 AVFilterContext *ctx = inlink->dst;
211 PhotosensitivityContext *s = ctx->priv;
212
213 s->badness_threshold = (int)(GRID_SIZE * GRID_SIZE * 4 * 256 * s->nb_frames * s->threshold_multiplier / 128);
214
215 return 0;
216 }
217
filter_frame(AVFilterLink * inlink,AVFrame * in)218 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
219 {
220 int this_badness, current_badness, fixed_badness, new_badness, i, res;
221 PhotosensitivityFrame ef;
222 AVFrame *src, *out;
223 int free_in = 0;
224 float factor;
225 AVDictionary **metadata;
226
227 AVFilterContext *ctx = inlink->dst;
228 AVFilterLink *outlink = ctx->outputs[0];
229 PhotosensitivityContext *s = ctx->priv;
230
231 /* weighted moving average */
232 current_badness = 0;
233 for (i = 1; i < s->nb_frames; i++)
234 current_badness += i * s->history[(s->history_pos + i) % s->nb_frames];
235 current_badness /= s->nb_frames;
236
237 convert_frame(ctx, in, &ef, s->skip);
238 this_badness = get_badness(&ef, &s->last_frame_e);
239 new_badness = current_badness + this_badness;
240 av_log(s, AV_LOG_VERBOSE, "badness: %6d -> %6d / %6d (%3d%% - %s)\n",
241 current_badness, new_badness, s->badness_threshold,
242 100 * new_badness / s->badness_threshold, new_badness < s->badness_threshold ? "OK" : "EXCEEDED");
243
244 fixed_badness = new_badness;
245 if (new_badness < s->badness_threshold || !s->last_frame_av || s->bypass) {
246 factor = 1; /* for metadata */
247 av_frame_free(&s->last_frame_av);
248 s->last_frame_av = src = in;
249 s->last_frame_e = ef;
250 s->history[s->history_pos] = this_badness;
251 } else {
252 factor = (float)(s->badness_threshold - current_badness) / (new_badness - current_badness);
253 if (factor <= 0) {
254 /* just duplicate the frame */
255 s->history[s->history_pos] = 0; /* frame was duplicated, thus, delta is zero */
256 } else {
257 res = av_frame_make_writable(s->last_frame_av);
258 if (res) {
259 av_frame_free(&in);
260 return res;
261 }
262 blend_frame(ctx, s->last_frame_av, in, factor);
263
264 convert_frame(ctx, s->last_frame_av, &ef, s->skip);
265 this_badness = get_badness(&ef, &s->last_frame_e);
266 fixed_badness = current_badness + this_badness;
267 av_log(s, AV_LOG_VERBOSE, " fixed: %6d -> %6d / %6d (%3d%%) factor=%5.3f\n",
268 current_badness, fixed_badness, s->badness_threshold,
269 100 * new_badness / s->badness_threshold, factor);
270 s->last_frame_e = ef;
271 s->history[s->history_pos] = this_badness;
272 }
273 src = s->last_frame_av;
274 free_in = 1;
275 }
276 s->history_pos = (s->history_pos + 1) % s->nb_frames;
277
278 out = ff_get_video_buffer(outlink, in->width, in->height);
279 if (!out) {
280 if (free_in == 1)
281 av_frame_free(&in);
282 return AVERROR(ENOMEM);
283 }
284 av_frame_copy_props(out, in);
285 metadata = &out->metadata;
286 if (metadata) {
287 char value[128];
288
289 snprintf(value, sizeof(value), "%f", (float)new_badness / s->badness_threshold);
290 av_dict_set(metadata, "lavfi.photosensitivity.badness", value, 0);
291
292 snprintf(value, sizeof(value), "%f", (float)fixed_badness / s->badness_threshold);
293 av_dict_set(metadata, "lavfi.photosensitivity.fixed-badness", value, 0);
294
295 snprintf(value, sizeof(value), "%f", (float)this_badness / s->badness_threshold);
296 av_dict_set(metadata, "lavfi.photosensitivity.frame-badness", value, 0);
297
298 snprintf(value, sizeof(value), "%f", factor);
299 av_dict_set(metadata, "lavfi.photosensitivity.factor", value, 0);
300 }
301 av_frame_copy(out, src);
302 if (free_in == 1)
303 av_frame_free(&in);
304 return ff_filter_frame(outlink, out);
305 }
306
uninit(AVFilterContext * ctx)307 static av_cold void uninit(AVFilterContext *ctx)
308 {
309 PhotosensitivityContext *s = ctx->priv;
310
311 av_frame_free(&s->last_frame_av);
312 }
313
314 static const AVFilterPad inputs[] = {
315 {
316 .name = "default",
317 .type = AVMEDIA_TYPE_VIDEO,
318 .filter_frame = filter_frame,
319 .config_props = config_input,
320 },
321 { NULL }
322 };
323
324 static const AVFilterPad outputs[] = {
325 {
326 .name = "default",
327 .type = AVMEDIA_TYPE_VIDEO,
328 },
329 { NULL }
330 };
331
332 AVFilter ff_vf_photosensitivity = {
333 .name = "photosensitivity",
334 .description = NULL_IF_CONFIG_SMALL("Filter out photosensitive epilepsy seizure-inducing flashes."),
335 .priv_size = sizeof(PhotosensitivityContext),
336 .priv_class = &photosensitivity_class,
337 .uninit = uninit,
338 .query_formats = query_formats,
339 .inputs = inputs,
340 .outputs = outputs,
341 };
342