1 /*
2 * Copyright (c) 2017 Paul B Mahol
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 /**
22 * @file
23 * Filter for reading closed captioning data (EIA-608).
24 * See also https://en.wikipedia.org/wiki/EIA-608
25 */
26
27 #include <string.h>
28
29 #include "libavutil/internal.h"
30 #include "libavutil/opt.h"
31 #include "libavutil/pixdesc.h"
32 #include "libavutil/timestamp.h"
33
34 #include "avfilter.h"
35 #include "formats.h"
36 #include "internal.h"
37 #include "video.h"
38
39 #define LAG 25
40 #define CLOCK_BITSIZE_MIN 0.2f
41 #define CLOCK_BITSIZE_MAX 1.5f
42 #define SYNC_BITSIZE_MIN 12.f
43 #define SYNC_BITSIZE_MAX 15.f
44
45 typedef struct LineItem {
46 int input;
47 int output;
48
49 float unfiltered;
50 float filtered;
51 float average;
52 float deviation;
53 } LineItem;
54
55 typedef struct CodeItem {
56 uint8_t bit;
57 int size;
58 } CodeItem;
59
60 typedef struct ScanItem {
61 int nb_line;
62 int found;
63 int white;
64 int black;
65 uint64_t *histogram;
66 uint8_t byte[2];
67
68 CodeItem *code;
69 LineItem *line;
70 } ScanItem;
71
72 typedef struct ReadEIA608Context {
73 const AVClass *class;
74
75 int start, end;
76 float spw;
77 int chp;
78 int lp;
79
80 int depth;
81 int max;
82 int nb_allocated;
83 ScanItem *scan;
84
85 void (*read_line[2])(AVFrame *in, int nb_line,
86 LineItem *line, int lp, int w);
87 } ReadEIA608Context;
88
89 #define OFFSET(x) offsetof(ReadEIA608Context, x)
90 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
91
92 static const AVOption readeia608_options[] = {
93 { "scan_min", "set from which line to scan for codes", OFFSET(start), AV_OPT_TYPE_INT, {.i64=0}, 0, INT_MAX, FLAGS },
94 { "scan_max", "set to which line to scan for codes", OFFSET(end), AV_OPT_TYPE_INT, {.i64=29}, 0, INT_MAX, FLAGS },
95 { "spw", "set ratio of width reserved for sync code detection", OFFSET(spw), AV_OPT_TYPE_FLOAT, {.dbl=.27}, 0.1, 0.7, FLAGS },
96 { "chp", "check and apply parity bit", OFFSET(chp), AV_OPT_TYPE_BOOL, {.i64= 0}, 0, 1, FLAGS },
97 { "lp", "lowpass line prior to processing", OFFSET(lp), AV_OPT_TYPE_BOOL, {.i64= 1}, 0, 1, FLAGS },
98 { NULL }
99 };
100
101 AVFILTER_DEFINE_CLASS(readeia608);
102
query_formats(AVFilterContext * ctx)103 static int query_formats(AVFilterContext *ctx)
104 {
105 static const enum AVPixelFormat pixel_fmts[] = {
106 AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY9,
107 AV_PIX_FMT_GRAY10, AV_PIX_FMT_GRAY12, AV_PIX_FMT_GRAY14,
108 AV_PIX_FMT_GRAY16,
109 AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
110 AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
111 AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,
112 AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P,
113 AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P,
114 AV_PIX_FMT_YUVJ411P,
115 AV_PIX_FMT_YUV420P9, AV_PIX_FMT_YUV422P9, AV_PIX_FMT_YUV444P9,
116 AV_PIX_FMT_YUV420P10, AV_PIX_FMT_YUV422P10, AV_PIX_FMT_YUV444P10,
117 AV_PIX_FMT_YUV440P10,
118 AV_PIX_FMT_YUV444P12, AV_PIX_FMT_YUV422P12, AV_PIX_FMT_YUV420P12,
119 AV_PIX_FMT_YUV440P12,
120 AV_PIX_FMT_YUV444P14, AV_PIX_FMT_YUV422P14, AV_PIX_FMT_YUV420P14,
121 AV_PIX_FMT_YUV420P16, AV_PIX_FMT_YUV422P16, AV_PIX_FMT_YUV444P16,
122 AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P,
123 AV_PIX_FMT_YUVA444P9, AV_PIX_FMT_YUVA444P10, AV_PIX_FMT_YUVA444P12, AV_PIX_FMT_YUVA444P16,
124 AV_PIX_FMT_YUVA422P9, AV_PIX_FMT_YUVA422P10, AV_PIX_FMT_YUVA422P12, AV_PIX_FMT_YUVA422P16,
125 AV_PIX_FMT_YUVA420P9, AV_PIX_FMT_YUVA420P10, AV_PIX_FMT_YUVA420P16,
126 AV_PIX_FMT_NONE
127 };
128 AVFilterFormats *formats = ff_make_format_list(pixel_fmts);
129 if (!formats)
130 return AVERROR(ENOMEM);
131 return ff_set_common_formats(ctx, formats);
132 }
133
config_filter(AVFilterContext * ctx,int start,int end)134 static int config_filter(AVFilterContext *ctx, int start, int end)
135 {
136 ReadEIA608Context *s = ctx->priv;
137 AVFilterLink *inlink = ctx->inputs[0];
138 int size = inlink->w + LAG;
139
140 if (end >= inlink->h) {
141 av_log(ctx, AV_LOG_WARNING, "Last line to scan too large, clipping.\n");
142 end = inlink->h - 1;
143 }
144
145 if (start > end) {
146 av_log(ctx, AV_LOG_ERROR, "Invalid range.\n");
147 return AVERROR(EINVAL);
148 }
149
150 if (s->nb_allocated < end - start + 1) {
151 const int diff = end - start + 1 - s->nb_allocated;
152
153 s->scan = av_realloc_f(s->scan, end - start + 1, sizeof(*s->scan));
154 if (!s->scan)
155 return AVERROR(ENOMEM);
156 memset(&s->scan[s->nb_allocated], 0, diff * sizeof(*s->scan));
157 s->nb_allocated = end - start + 1;
158 }
159
160 for (int i = 0; i < s->nb_allocated; i++) {
161 ScanItem *scan = &s->scan[i];
162
163 if (!scan->histogram)
164 scan->histogram = av_calloc(s->max + 1, sizeof(*scan->histogram));
165 if (!scan->line)
166 scan->line = av_calloc(size, sizeof(*scan->line));
167 if (!scan->code)
168 scan->code = av_calloc(size, sizeof(*scan->code));
169 if (!scan->line || !scan->code || !scan->histogram)
170 return AVERROR(ENOMEM);
171 }
172
173 s->start = start;
174 s->end = end;
175
176 return 0;
177 }
178
build_histogram(ReadEIA608Context * s,ScanItem * scan,const LineItem * line,int len)179 static void build_histogram(ReadEIA608Context *s, ScanItem *scan, const LineItem *line, int len)
180 {
181 memset(scan->histogram, 0, (s->max + 1) * sizeof(*scan->histogram));
182
183 for (int i = LAG; i < len + LAG; i++)
184 scan->histogram[line[i].input]++;
185 }
186
find_black_and_white(ReadEIA608Context * s,ScanItem * scan)187 static void find_black_and_white(ReadEIA608Context *s, ScanItem *scan)
188 {
189 const int max = s->max;
190 int start = 0, end = 0, middle;
191 int black = 0, white = 0;
192 int cnt;
193
194 for (int i = 0; i <= max; i++) {
195 if (scan->histogram[i]) {
196 start = i;
197 break;
198 }
199 }
200
201 for (int i = max; i >= 0; i--) {
202 if (scan->histogram[i]) {
203 end = i;
204 break;
205 }
206 }
207
208 middle = start + (end - start) / 2;
209
210 cnt = 0;
211 for (int i = start; i <= middle; i++) {
212 if (scan->histogram[i] > cnt) {
213 cnt = scan->histogram[i];
214 black = i;
215 }
216 }
217
218 cnt = 0;
219 for (int i = end; i >= middle; i--) {
220 if (scan->histogram[i] > cnt) {
221 cnt = scan->histogram[i];
222 white = i;
223 }
224 }
225
226 scan->black = black;
227 scan->white = white;
228 }
229
meanf(const LineItem * line,int len)230 static float meanf(const LineItem *line, int len)
231 {
232 float sum = 0.0, mean = 0.0;
233
234 for (int i = 0; i < len; i++)
235 sum += line[i].filtered;
236
237 mean = sum / len;
238
239 return mean;
240 }
241
stddevf(const LineItem * line,int len)242 static float stddevf(const LineItem *line, int len)
243 {
244 float m = meanf(line, len);
245 float standard_deviation = 0.f;
246
247 for (int i = 0; i < len; i++)
248 standard_deviation += (line[i].filtered - m) * (line[i].filtered - m);
249
250 return sqrtf(standard_deviation / (len - 1));
251 }
252
thresholding(ReadEIA608Context * s,ScanItem * scan,LineItem * line,int lag,float threshold,float influence,int len)253 static void thresholding(ReadEIA608Context *s, ScanItem *scan, LineItem *line,
254 int lag, float threshold, float influence, int len)
255 {
256 for (int i = lag; i < len + lag; i++) {
257 line[i].unfiltered = line[i].input / 255.f;
258 line[i].filtered = line[i].unfiltered;
259 }
260
261 for (int i = 0; i < lag; i++) {
262 line[i].unfiltered = meanf(line, len * s->spw);
263 line[i].filtered = line[i].unfiltered;
264 }
265
266 line[lag - 1].average = meanf(line, lag);
267 line[lag - 1].deviation = stddevf(line, lag);
268
269 for (int i = lag; i < len + lag; i++) {
270 if (fabsf(line[i].unfiltered - line[i-1].average) > threshold * line[i-1].deviation) {
271 if (line[i].unfiltered > line[i-1].average) {
272 line[i].output = 255;
273 } else {
274 line[i].output = 0;
275 }
276
277 line[i].filtered = influence * line[i].unfiltered + (1.f - influence) * line[i-1].filtered;
278 } else {
279 int distance_from_black, distance_from_white;
280
281 distance_from_black = FFABS(line[i].input - scan->black);
282 distance_from_white = FFABS(line[i].input - scan->white);
283
284 line[i].output = distance_from_black <= distance_from_white ? 0 : 255;
285 }
286
287 line[i].average = meanf(line + i - lag, lag);
288 line[i].deviation = stddevf(line + i - lag, lag);
289 }
290 }
291
periods(const LineItem * line,CodeItem * code,int len)292 static int periods(const LineItem *line, CodeItem *code, int len)
293 {
294 int hold = line[LAG].output, cnt = 0;
295 int last = LAG;
296
297 memset(code, 0, len * sizeof(*code));
298
299 for (int i = LAG + 1; i < len + LAG; i++) {
300 if (line[i].output != hold) {
301 code[cnt].size = i - last;
302 code[cnt].bit = hold;
303 hold = line[i].output;
304 last = i;
305 cnt++;
306 }
307 }
308
309 code[cnt].size = LAG + len - last;
310 code[cnt].bit = hold;
311
312 return cnt + 1;
313 }
314
dump_code(AVFilterContext * ctx,ScanItem * scan,int len,int item)315 static void dump_code(AVFilterContext *ctx, ScanItem *scan, int len, int item)
316 {
317 av_log(ctx, AV_LOG_DEBUG, "%d:", item);
318 for (int i = 0; i < len; i++) {
319 av_log(ctx, AV_LOG_DEBUG, " %03d", scan->code[i].size);
320 }
321 av_log(ctx, AV_LOG_DEBUG, "\n");
322 }
323
324 #define READ_LINE(type, name) \
325 static void read_##name(AVFrame *in, int nb_line, LineItem *line, int lp, int w) \
326 { \
327 const type *src = (const type *)(&in->data[0][nb_line * in->linesize[0]]);\
328 \
329 if (lp) { \
330 for (int i = 0; i < w; i++) { \
331 int a = FFMAX(i - 3, 0); \
332 int b = FFMAX(i - 2, 0); \
333 int c = FFMAX(i - 1, 0); \
334 int d = FFMIN(i + 3, w-1); \
335 int e = FFMIN(i + 2, w-1); \
336 int f = FFMIN(i + 1, w-1); \
337 \
338 line[LAG + i].input = (src[a] + src[b] + src[c] + src[i] + \
339 src[d] + src[e] + src[f] + 6) / 7; \
340 } \
341 } else { \
342 for (int i = 0; i < w; i++) { \
343 line[LAG + i].input = src[i]; \
344 } \
345 } \
346 }
347
READ_LINE(uint8_t,byte)348 READ_LINE(uint8_t, byte)
349 READ_LINE(uint16_t, word)
350
351 static int config_input(AVFilterLink *inlink)
352 {
353 AVFilterContext *ctx = inlink->dst;
354 ReadEIA608Context *s = ctx->priv;
355 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
356
357 if (!desc)
358 return AVERROR_BUG;
359 s->depth = desc->comp[0].depth;
360 s->max = (1 << desc->comp[0].depth) - 1;
361 s->read_line[0] = read_byte;
362 s->read_line[1] = read_word;
363
364 return config_filter(ctx, s->start, s->end);
365 }
366
extract_line(AVFilterContext * ctx,AVFrame * in,ScanItem * scan,int w,int nb_line)367 static void extract_line(AVFilterContext *ctx, AVFrame *in, ScanItem *scan, int w, int nb_line)
368 {
369 ReadEIA608Context *s = ctx->priv;
370 LineItem *line = scan->line;
371 int i, j, ch, len;
372 uint8_t codes[19] = { 0 };
373 float bit_size = 0.f;
374 int parity;
375
376 memset(line, 0, (w + LAG) * sizeof(*line));
377 scan->byte[0] = scan->byte[1] = 0;
378 scan->found = 0;
379
380 s->read_line[s->depth > 8](in, nb_line, line, s->lp, w);
381
382 build_histogram(s, scan, line, w);
383 find_black_and_white(s, scan);
384 if (scan->white - scan->black < 5)
385 return;
386
387 thresholding(s, scan, line, LAG, 1, 0, w);
388 len = periods(line, scan->code, w);
389 dump_code(ctx, scan, len, nb_line);
390 if (len < 15 ||
391 scan->code[14].bit != 0 ||
392 w / (float)scan->code[14].size < SYNC_BITSIZE_MIN ||
393 w / (float)scan->code[14].size > SYNC_BITSIZE_MAX) {
394 return;
395 }
396
397 for (i = 14; i < len; i++) {
398 bit_size += scan->code[i].size;
399 }
400
401 bit_size /= 19.f;
402 for (i = 1; i < 14; i++) {
403 if (scan->code[i].size / bit_size > CLOCK_BITSIZE_MAX ||
404 scan->code[i].size / bit_size < CLOCK_BITSIZE_MIN) {
405 return;
406 }
407 }
408
409 if (scan->code[15].size / bit_size < 0.45f) {
410 return;
411 }
412
413 for (j = 0, i = 14; i < len; i++) {
414 int run, bit;
415
416 run = lrintf(scan->code[i].size / bit_size);
417 bit = scan->code[i].bit;
418
419 for (int k = 0; j < 19 && k < run; k++) {
420 codes[j++] = bit;
421 }
422
423 if (j >= 19)
424 break;
425 }
426
427 for (ch = 0; ch < 2; ch++) {
428 for (parity = 0, i = 0; i < 8; i++) {
429 int b = codes[3 + ch * 8 + i];
430
431 if (b == 255) {
432 parity++;
433 b = 1;
434 } else {
435 b = 0;
436 }
437 scan->byte[ch] |= b << i;
438 }
439
440 if (s->chp) {
441 if (!(parity & 1)) {
442 scan->byte[ch] = 0x7F;
443 }
444 }
445 }
446
447 scan->nb_line = nb_line;
448 scan->found = 1;
449 }
450
extract_lines(AVFilterContext * ctx,void * arg,int job,int nb_jobs)451 static int extract_lines(AVFilterContext *ctx, void *arg,
452 int job, int nb_jobs)
453 {
454 ReadEIA608Context *s = ctx->priv;
455 AVFilterLink *inlink = ctx->inputs[0];
456 const int h = s->end - s->start + 1;
457 const int start = (h * job) / nb_jobs;
458 const int end = (h * (job+1)) / nb_jobs;
459 AVFrame *in = arg;
460
461 for (int i = start; i < end; i++) {
462 ScanItem *scan = &s->scan[i];
463
464 extract_line(ctx, in, scan, inlink->w, i);
465 }
466
467 return 0;
468 }
469
filter_frame(AVFilterLink * inlink,AVFrame * in)470 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
471 {
472 AVFilterContext *ctx = inlink->dst;
473 AVFilterLink *outlink = ctx->outputs[0];
474 ReadEIA608Context *s = ctx->priv;
475 int nb_found;
476
477 ctx->internal->execute(ctx, extract_lines, in, NULL, FFMIN(FFMAX(s->end - s->start + 1, 1),
478 ff_filter_get_nb_threads(ctx)));
479
480 nb_found = 0;
481 for (int i = 0; i < s->end - s->start + 1; i++) {
482 ScanItem *scan = &s->scan[i];
483 uint8_t key[128], value[128];
484
485 if (!scan->found)
486 continue;
487
488 //snprintf(key, sizeof(key), "lavfi.readeia608.%d.bits", nb_found);
489 //snprintf(value, sizeof(value), "0b%d%d%d%d%d%d%d%d 0b%d%d%d%d%d%d%d%d", codes[3]==255,codes[4]==255,codes[5]==255,codes[6]==255,codes[7]==255,codes[8]==255,codes[9]==255,codes[10]==255,codes[11]==255,codes[12]==255,codes[13]==255,codes[14]==255,codes[15]==255,codes[16]==255,codes[17]==255,codes[18]==255);
490 //av_dict_set(&in->metadata, key, value, 0);
491
492 snprintf(key, sizeof(key), "lavfi.readeia608.%d.cc", nb_found);
493 snprintf(value, sizeof(value), "0x%02X%02X", scan->byte[0], scan->byte[1]);
494 av_dict_set(&in->metadata, key, value, 0);
495
496 snprintf(key, sizeof(key), "lavfi.readeia608.%d.line", nb_found);
497 snprintf(value, sizeof(value), "%d", scan->nb_line);
498 av_dict_set(&in->metadata, key, value, 0);
499
500 nb_found++;
501 }
502
503 return ff_filter_frame(outlink, in);
504 }
505
uninit(AVFilterContext * ctx)506 static av_cold void uninit(AVFilterContext *ctx)
507 {
508 ReadEIA608Context *s = ctx->priv;
509
510 for (int i = 0; i < s->nb_allocated; i++) {
511 ScanItem *scan = &s->scan[i];
512
513 av_freep(&scan->histogram);
514 av_freep(&scan->code);
515 av_freep(&scan->line);
516 }
517
518 s->nb_allocated = 0;
519 av_freep(&s->scan);
520 }
521
process_command(AVFilterContext * ctx,const char * cmd,const char * args,char * res,int res_len,int flags)522 static int process_command(AVFilterContext *ctx, const char *cmd, const char *args,
523 char *res, int res_len, int flags)
524 {
525 ReadEIA608Context *s = ctx->priv;
526 int ret, start = s->start, end = s->end;
527
528 ret = ff_filter_process_command(ctx, cmd, args, res, res_len, flags);
529 if (ret < 0)
530 return ret;
531
532 ret = config_filter(ctx, s->start, s->end);
533 if (ret < 0) {
534 s->start = start;
535 s->end = end;
536 }
537
538 return 0;
539 }
540
541 static const AVFilterPad readeia608_inputs[] = {
542 {
543 .name = "default",
544 .type = AVMEDIA_TYPE_VIDEO,
545 .filter_frame = filter_frame,
546 .config_props = config_input,
547 },
548 { NULL }
549 };
550
551 static const AVFilterPad readeia608_outputs[] = {
552 {
553 .name = "default",
554 .type = AVMEDIA_TYPE_VIDEO,
555 },
556 { NULL }
557 };
558
559 AVFilter ff_vf_readeia608 = {
560 .name = "readeia608",
561 .description = NULL_IF_CONFIG_SMALL("Read EIA-608 Closed Caption codes from input video and write them to frame metadata."),
562 .priv_size = sizeof(ReadEIA608Context),
563 .priv_class = &readeia608_class,
564 .query_formats = query_formats,
565 .inputs = readeia608_inputs,
566 .outputs = readeia608_outputs,
567 .uninit = uninit,
568 .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC | AVFILTER_FLAG_SLICE_THREADS,
569 .process_command = process_command,
570 };
571