1 /* GStreamer
2 * Copyright (C) 2020 He Junyan <junyan.he@intel.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 /*
21 * SECTION:element-av1parse
22 * @title: av1parse
23 * @short_description: An AV1 stream parse.
24 *
25 * The minimal unit should be the BYTE.
26 * There are four types of AV1 alignment in the AV1 stream.
27 *
28 * alignment: byte, obu, frame, tu
29 *
30 * 1. Aligned to byte. The basic and default one for input.
31 * 2. Aligned to obu(Open Bitstream Units).
32 * 3. Aligned to frame. The default one for output. This ensures that
33 * each buffer contains only one frame or frame header with the
34 * show_existing flag for the base or sub layer. It is useful for
35 * the decoder.
36 * 4. Aligned to tu(Temporal Unit). A temporal unit consists of all the
37 * OBUs that are associated with a specific, distinct time instant.
38 * When scalability is disabled, it contains just exact one showing
39 * frame(may contain several unshowing frames). When scalability is
40 * enabled, it contains frames depending on the layer number. It should
41 * begin with a temporal delimiter obu. It may be useful for mux/demux
42 * to index the data of some timestamp.
43 *
44 * The annex B define a special format for the temporal unit. The size of
45 * each temporal unit is extract out to the header of the buffer, and no
46 * size field inside the each obu. There are two stream formats:
47 *
48 * stream-format: obu-stream, annexb
49 *
50 * 1. obu-stream. The basic and default one.
51 * 2. annexb. A special stream of temporal unit. It also implies that the
52 * alignment should be TU.
53 *
54 * This AV1 parse implements the conversion between the alignments and the
55 * stream-formats. If the input and output have the same alignment and the
56 * same stream-format, it will check and bypass the data.
57 *
58 * ## Example launch line to generate annex B format AV1 stream:
59 * ```
60 * gst-launch-1.0 filesrc location=sample.av1 ! ivfparse ! av1parse ! \
61 * video/x-av1,alignment=\(string\)tu,stream-format=\(string\)annexb ! \
62 * filesink location=matroskamux ! filesink location=trans.mkv
63 * ```
64 *
65 * Since: 1.20
66 */
67
68 #ifdef HAVE_CONFIG_H
69 #include "config.h"
70 #endif
71
72 #include <gst/base/gstbitreader.h>
73 #include <gst/base/gstbitwriter.h>
74 #include <gst/codecparsers/gstav1parser.h>
75 #include <gst/video/video.h>
76 #include "gstvideoparserselements.h"
77 #include "gstav1parse.h"
78
79 #include <string.h>
80
81 #define GST_AV1_MAX_LEB_128_SIZE 8
82
83 GST_DEBUG_CATEGORY (av1_parse_debug);
84 #define GST_CAT_DEFAULT av1_parse_debug
85
86 /* We combine the stream format and the alignment
87 together. When stream format is annexb, the
88 alignment must be TU. */
89 typedef enum
90 {
91 GST_AV1_PARSE_ALIGN_ERROR = -1,
92 GST_AV1_PARSE_ALIGN_NONE = 0,
93 GST_AV1_PARSE_ALIGN_BYTE,
94 GST_AV1_PARSE_ALIGN_OBU,
95 GST_AV1_PARSE_ALIGN_FRAME,
96 GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT,
97 GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B,
98 } GstAV1ParseAligment;
99
100 struct _GstAV1Parse
101 {
102 GstBaseParse parent;
103
104 gint width;
105 gint height;
106 gint subsampling_x;
107 gint subsampling_y;
108 gboolean mono_chrome;
109 guint8 bit_depth;
110 gchar *colorimetry;
111 GstAV1Profile profile;
112
113 GstAV1ParseAligment in_align;
114 GstAV1ParseAligment align;
115
116 GstAV1Parser *parser;
117 GstAdapter *cache_out;
118 guint last_parsed_offset;
119 GstAdapter *frame_cache;
120 guint highest_spatial_id;
121 gint last_shown_frame_temporal_id;
122 gint last_shown_frame_spatial_id;
123 gboolean within_one_frame;
124 gboolean update_caps;
125 gboolean discont;
126 gboolean header;
127 gboolean keyframe;
128 gboolean show_frame;
129 };
130
131 static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
132 GST_PAD_SINK,
133 GST_PAD_ALWAYS,
134 GST_STATIC_CAPS ("video/x-av1"));
135
136 static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
137 GST_PAD_SRC,
138 GST_PAD_ALWAYS,
139 GST_STATIC_CAPS ("video/x-av1, parsed = (boolean) true, "
140 "stream-format=(string) { obu-stream, annexb }, "
141 "alignment=(string) { obu, tu, frame }"));
142
143 #define parent_class gst_av1_parse_parent_class
144 G_DEFINE_TYPE (GstAV1Parse, gst_av1_parse, GST_TYPE_BASE_PARSE);
145 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (av1parse, "av1parse", GST_RANK_SECONDARY,
146 GST_TYPE_AV1_PARSE, videoparsers_element_init (plugin));
147
148 static void
remove_fields(GstCaps * caps,gboolean all)149 remove_fields (GstCaps * caps, gboolean all)
150 {
151 guint i, n;
152
153 n = gst_caps_get_size (caps);
154 for (i = 0; i < n; i++) {
155 GstStructure *s = gst_caps_get_structure (caps, i);
156
157 if (all) {
158 gst_structure_remove_field (s, "alignment");
159 gst_structure_remove_field (s, "stream-format");
160 }
161 gst_structure_remove_field (s, "parsed");
162 }
163 }
164
165 static const gchar *
_obu_name(GstAV1OBUType type)166 _obu_name (GstAV1OBUType type)
167 {
168 switch (type) {
169 case GST_AV1_OBU_SEQUENCE_HEADER:
170 return "sequence header";
171 case GST_AV1_OBU_TEMPORAL_DELIMITER:
172 return "temporal delimiter";
173 case GST_AV1_OBU_FRAME_HEADER:
174 return "frame header";
175 case GST_AV1_OBU_TILE_GROUP:
176 return "tile group";
177 case GST_AV1_OBU_METADATA:
178 return "metadata";
179 case GST_AV1_OBU_FRAME:
180 return "frame";
181 case GST_AV1_OBU_REDUNDANT_FRAME_HEADER:
182 return "redundant frame header";
183 case GST_AV1_OBU_TILE_LIST:
184 return "tile list";
185 case GST_AV1_OBU_PADDING:
186 return "padding";
187 default:
188 return "unknown";
189 }
190
191 return NULL;
192 }
193
194 static guint32
_read_leb128(guint8 * data,GstAV1ParserResult * retval,guint32 * comsumed)195 _read_leb128 (guint8 * data, GstAV1ParserResult * retval, guint32 * comsumed)
196 {
197 guint8 leb128_byte = 0;
198 guint64 value = 0;
199 gint i;
200 gboolean result;
201 GstBitReader br;
202 guint32 cur_pos;
203
204 gst_bit_reader_init (&br, data, 8);
205
206 cur_pos = gst_bit_reader_get_pos (&br);
207 for (i = 0; i < 8; i++) {
208 leb128_byte = 0;
209 result = gst_bit_reader_get_bits_uint8 (&br, &leb128_byte, 8);
210 if (result == FALSE) {
211 *retval = GST_AV1_PARSER_BITSTREAM_ERROR;
212 return 0;
213 }
214
215 value |= (((gint) leb128_byte & 0x7f) << (i * 7));
216 if (!(leb128_byte & 0x80))
217 break;
218 }
219
220 *comsumed = (gst_bit_reader_get_pos (&br) - cur_pos) / 8;
221 /* check for bitstream conformance see chapter4.10.5 */
222 if (value < G_MAXUINT32) {
223 *retval = GST_AV1_PARSER_OK;
224 return (guint32) value;
225 } else {
226 GST_WARNING ("invalid leb128");
227 *retval = GST_AV1_PARSER_BITSTREAM_ERROR;
228 return 0;
229 }
230 }
231
232 static gsize
_leb_size_in_bytes(guint64 value)233 _leb_size_in_bytes (guint64 value)
234 {
235 gsize size = 0;
236 do {
237 ++size;
238 } while ((value >>= 7) != 0);
239
240 return size;
241 }
242
243 static gboolean
_write_leb128(guint8 * data,guint * len,guint64 value)244 _write_leb128 (guint8 * data, guint * len, guint64 value)
245 {
246 guint leb_size = _leb_size_in_bytes (value);
247 guint i;
248
249 if (value > G_MAXUINT32 || leb_size > GST_AV1_MAX_LEB_128_SIZE)
250 return FALSE;
251
252 for (i = 0; i < leb_size; ++i) {
253 guint8 byte = value & 0x7f;
254 value >>= 7;
255
256 /* Signal that more bytes follow. */
257 if (value != 0)
258 byte |= 0x80;
259
260 *(data + i) = byte;
261 }
262
263 *len = leb_size;
264 return TRUE;
265 }
266
267 static gboolean gst_av1_parse_start (GstBaseParse * parse);
268 static gboolean gst_av1_parse_stop (GstBaseParse * parse);
269 static GstFlowReturn gst_av1_parse_handle_frame (GstBaseParse * parse,
270 GstBaseParseFrame * frame, gint * skipsize);
271 static gboolean gst_av1_parse_set_sink_caps (GstBaseParse * parse,
272 GstCaps * caps);
273 static GstCaps *gst_av1_parse_get_sink_caps (GstBaseParse * parse,
274 GstCaps * filter);
275
276 /* Clear the parse state related to data kind OBUs. */
277 static void
gst_av1_parse_reset_obu_data_state(GstAV1Parse * self)278 gst_av1_parse_reset_obu_data_state (GstAV1Parse * self)
279 {
280 self->last_shown_frame_temporal_id = -1;
281 self->last_shown_frame_spatial_id = -1;
282 self->within_one_frame = FALSE;
283 }
284
285 static void
gst_av1_parse_reset(GstAV1Parse * self)286 gst_av1_parse_reset (GstAV1Parse * self)
287 {
288 self->width = 0;
289 self->height = 0;
290 self->subsampling_x = -1;
291 self->subsampling_y = -1;
292 self->mono_chrome = FALSE;
293 self->profile = GST_AV1_PROFILE_UNDEFINED;
294 self->bit_depth = 0;
295 self->align = GST_AV1_PARSE_ALIGN_NONE;
296 self->in_align = GST_AV1_PARSE_ALIGN_NONE;
297 self->discont = TRUE;
298 self->header = FALSE;
299 self->keyframe = FALSE;
300 self->show_frame = FALSE;
301 self->last_parsed_offset = 0;
302 self->highest_spatial_id = 0;
303 gst_av1_parse_reset_obu_data_state (self);
304 g_clear_pointer (&self->colorimetry, g_free);
305 g_clear_pointer (&self->parser, gst_av1_parser_free);
306 gst_adapter_clear (self->cache_out);
307 gst_adapter_clear (self->frame_cache);
308 }
309
310 static void
gst_av1_parse_init(GstAV1Parse * self)311 gst_av1_parse_init (GstAV1Parse * self)
312 {
313 gst_base_parse_set_pts_interpolation (GST_BASE_PARSE (self), FALSE);
314 gst_base_parse_set_infer_ts (GST_BASE_PARSE (self), FALSE);
315
316 GST_PAD_SET_ACCEPT_INTERSECT (GST_BASE_PARSE_SINK_PAD (self));
317 GST_PAD_SET_ACCEPT_TEMPLATE (GST_BASE_PARSE_SINK_PAD (self));
318
319 self->cache_out = gst_adapter_new ();
320 self->frame_cache = gst_adapter_new ();
321 }
322
323 static void
gst_av1_parse_finalize(GObject * object)324 gst_av1_parse_finalize (GObject * object)
325 {
326 GstAV1Parse *self = GST_AV1_PARSE (object);
327
328 gst_av1_parse_reset (self);
329 g_object_unref (self->cache_out);
330 g_object_unref (self->frame_cache);
331
332 G_OBJECT_CLASS (parent_class)->finalize (object);
333 }
334
335 static void
gst_av1_parse_class_init(GstAV1ParseClass * klass)336 gst_av1_parse_class_init (GstAV1ParseClass * klass)
337 {
338 GObjectClass *gobject_class = (GObjectClass *) klass;
339 GstBaseParseClass *parse_class = GST_BASE_PARSE_CLASS (klass);
340 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
341
342 gobject_class->finalize = gst_av1_parse_finalize;
343 parse_class->start = GST_DEBUG_FUNCPTR (gst_av1_parse_start);
344 parse_class->stop = GST_DEBUG_FUNCPTR (gst_av1_parse_stop);
345 parse_class->handle_frame = GST_DEBUG_FUNCPTR (gst_av1_parse_handle_frame);
346 parse_class->set_sink_caps = GST_DEBUG_FUNCPTR (gst_av1_parse_set_sink_caps);
347 parse_class->get_sink_caps = GST_DEBUG_FUNCPTR (gst_av1_parse_get_sink_caps);
348
349 gst_element_class_add_static_pad_template (element_class, &srctemplate);
350 gst_element_class_add_static_pad_template (element_class, &sinktemplate);
351
352 gst_element_class_set_static_metadata (element_class, "AV1 parser",
353 "Codec/Parser/Converter/Video",
354 "Parses AV1 streams", "He Junyan <junyan.he@intel.com>");
355
356 GST_DEBUG_CATEGORY_INIT (av1_parse_debug, "av1parse", 0, "av1 parser");
357 }
358
359 static gboolean
gst_av1_parse_start(GstBaseParse * parse)360 gst_av1_parse_start (GstBaseParse * parse)
361 {
362 GstAV1Parse *self = GST_AV1_PARSE (parse);
363
364 GST_DEBUG_OBJECT (self, "start");
365
366 gst_av1_parse_reset (self);
367 self->parser = gst_av1_parser_new ();
368
369 /* At least the OBU header. */
370 gst_base_parse_set_min_frame_size (parse, 1);
371
372 return TRUE;
373 }
374
375 static gboolean
gst_av1_parse_stop(GstBaseParse * parse)376 gst_av1_parse_stop (GstBaseParse * parse)
377 {
378 GstAV1Parse *self = GST_AV1_PARSE (parse);
379
380 GST_DEBUG_OBJECT (self, "stop");
381 g_clear_pointer (&self->parser, gst_av1_parser_free);
382
383 return TRUE;
384 }
385
386 static const gchar *
gst_av1_parse_profile_to_string(GstAV1Profile profile)387 gst_av1_parse_profile_to_string (GstAV1Profile profile)
388 {
389 switch (profile) {
390 case GST_AV1_PROFILE_0:
391 return "main";
392 case GST_AV1_PROFILE_1:
393 return "high";
394 case GST_AV1_PROFILE_2:
395 return "professional";
396 default:
397 break;
398 }
399
400 return NULL;
401 }
402
403 static GstAV1Profile
gst_av1_parse_profile_from_string(const gchar * profile)404 gst_av1_parse_profile_from_string (const gchar * profile)
405 {
406 if (!profile)
407 return GST_AV1_PROFILE_UNDEFINED;
408
409 if (g_strcmp0 (profile, "main") == 0)
410 return GST_AV1_PROFILE_0;
411 else if (g_strcmp0 (profile, "high") == 0)
412 return GST_AV1_PROFILE_1;
413 else if (g_strcmp0 (profile, "professional") == 0)
414 return GST_AV1_PROFILE_2;
415
416 return GST_AV1_PROFILE_UNDEFINED;
417 }
418
419 static const gchar *
gst_av1_parse_alignment_to_steam_format_string(GstAV1ParseAligment align)420 gst_av1_parse_alignment_to_steam_format_string (GstAV1ParseAligment align)
421 {
422 switch (align) {
423 case GST_AV1_PARSE_ALIGN_BYTE:
424 return "obu-stream";
425 case GST_AV1_PARSE_ALIGN_OBU:
426 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT:
427 case GST_AV1_PARSE_ALIGN_FRAME:
428 return "obu-stream";
429 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B:
430 return "annexb";
431 default:
432 GST_WARNING ("Unrecognized steam format");
433 break;
434 }
435
436 return NULL;
437 }
438
439 static const gchar *
gst_av1_parse_alignment_to_string(GstAV1ParseAligment align)440 gst_av1_parse_alignment_to_string (GstAV1ParseAligment align)
441 {
442 switch (align) {
443 case GST_AV1_PARSE_ALIGN_BYTE:
444 return "byte";
445 case GST_AV1_PARSE_ALIGN_OBU:
446 return "obu";
447 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT:
448 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B:
449 return "tu";
450 case GST_AV1_PARSE_ALIGN_FRAME:
451 return "frame";
452 default:
453 GST_WARNING ("Unrecognized alignment");
454 break;
455 }
456
457 return NULL;
458 }
459
460 static GstAV1ParseAligment
gst_av1_parse_alignment_from_string(const gchar * align,const gchar * stream_format)461 gst_av1_parse_alignment_from_string (const gchar * align,
462 const gchar * stream_format)
463 {
464 if (!align && !stream_format)
465 return GST_AV1_PARSE_ALIGN_NONE;
466
467 if (stream_format) {
468 if (g_strcmp0 (stream_format, "annexb") == 0) {
469 if (align && g_strcmp0 (align, "tu") != 0) {
470 /* annex b stream must align to TU. */
471 return GST_AV1_PARSE_ALIGN_ERROR;
472 } else {
473 return GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B;
474 }
475 } else if (g_strcmp0 (stream_format, "obu-stream") != 0) {
476 /* unrecognized */
477 return GST_AV1_PARSE_ALIGN_NONE;
478 }
479
480 /* stream-format is obu-stream, depends on align */
481 }
482
483 if (align) {
484 if (g_strcmp0 (align, "byte") == 0) {
485 return GST_AV1_PARSE_ALIGN_BYTE;
486 } else if (g_strcmp0 (align, "obu") == 0) {
487 return GST_AV1_PARSE_ALIGN_OBU;
488 } else if (g_strcmp0 (align, "tu") == 0) {
489 return GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
490 } else if (g_strcmp0 (align, "frame") == 0) {
491 return GST_AV1_PARSE_ALIGN_FRAME;
492 } else {
493 /* unrecognized */
494 return GST_AV1_PARSE_ALIGN_NONE;
495 }
496 }
497
498 return GST_AV1_PARSE_ALIGN_NONE;
499 }
500
501 static gboolean
gst_av1_parse_caps_has_alignment(GstCaps * caps,GstAV1ParseAligment alignment)502 gst_av1_parse_caps_has_alignment (GstCaps * caps, GstAV1ParseAligment alignment)
503 {
504 guint i, j, caps_size;
505 const gchar *cmp_align_str = NULL;
506 const gchar *cmp_stream_str = NULL;
507
508 GST_DEBUG ("Try to find alignment %d in caps: %" GST_PTR_FORMAT,
509 alignment, caps);
510
511 caps_size = gst_caps_get_size (caps);
512 if (caps_size == 0)
513 return FALSE;
514
515 switch (alignment) {
516 case GST_AV1_PARSE_ALIGN_BYTE:
517 cmp_align_str = "byte";
518 cmp_stream_str = "obu-stream";
519 break;
520 case GST_AV1_PARSE_ALIGN_OBU:
521 cmp_align_str = "obu";
522 cmp_stream_str = "obu-stream";
523 break;
524 case GST_AV1_PARSE_ALIGN_FRAME:
525 cmp_align_str = "frame";
526 cmp_stream_str = "obu-stream";
527 break;
528 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT:
529 cmp_align_str = "tu";
530 cmp_stream_str = "obu-stream";
531 break;
532 case GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B:
533 cmp_align_str = "tu";
534 cmp_stream_str = "annexb";
535 break;
536 default:
537 return FALSE;
538 }
539
540 for (i = 0; i < caps_size; i++) {
541 GstStructure *s = gst_caps_get_structure (caps, i);
542 const GValue *alignment_value = gst_structure_get_value (s, "alignment");
543 const GValue *stream_value = gst_structure_get_value (s, "stream-format");
544
545 if (!alignment_value || !stream_value)
546 continue;
547
548 if (G_VALUE_HOLDS_STRING (alignment_value)) {
549 const gchar *align_str = g_value_get_string (alignment_value);
550
551 if (g_strcmp0 (align_str, cmp_align_str) != 0)
552 continue;
553 } else if (GST_VALUE_HOLDS_LIST (alignment_value)) {
554 guint num_values = gst_value_list_get_size (alignment_value);
555
556 for (j = 0; j < num_values; j++) {
557 const GValue *v = gst_value_list_get_value (alignment_value, j);
558 const gchar *align_str = g_value_get_string (v);
559
560 if (g_strcmp0 (align_str, cmp_align_str) == 0)
561 break;
562 }
563
564 if (j == num_values)
565 continue;
566 }
567
568 if (G_VALUE_HOLDS_STRING (stream_value)) {
569 const gchar *stream_str = g_value_get_string (stream_value);
570
571 if (g_strcmp0 (stream_str, cmp_stream_str) != 0)
572 continue;
573 } else if (GST_VALUE_HOLDS_LIST (stream_value)) {
574 guint num_values = gst_value_list_get_size (stream_value);
575
576 for (j = 0; j < num_values; j++) {
577 const GValue *v = gst_value_list_get_value (stream_value, j);
578 const gchar *stream_str = g_value_get_string (v);
579
580 if (g_strcmp0 (stream_str, cmp_stream_str) == 0)
581 break;
582 }
583
584 if (j == num_values)
585 continue;
586 }
587
588 return TRUE;
589 }
590
591 return FALSE;
592 }
593
594 static GstAV1ParseAligment
gst_av1_parse_alignment_from_caps(GstCaps * caps)595 gst_av1_parse_alignment_from_caps (GstCaps * caps)
596 {
597 GstAV1ParseAligment align;
598
599 align = GST_AV1_PARSE_ALIGN_NONE;
600
601 GST_DEBUG ("parsing caps: %" GST_PTR_FORMAT, caps);
602
603 if (caps && gst_caps_get_size (caps) > 0) {
604 GstStructure *s = gst_caps_get_structure (caps, 0);
605 const gchar *str_align = NULL;
606 const gchar *str_stream = NULL;
607
608 str_align = gst_structure_get_string (s, "alignment");
609 str_stream = gst_structure_get_string (s, "stream-format");
610
611 if (str_align || str_stream)
612 align = gst_av1_parse_alignment_from_string (str_align, str_stream);
613 }
614
615 return align;
616 }
617
618 static void
gst_av1_parse_update_src_caps(GstAV1Parse * self,GstCaps * caps)619 gst_av1_parse_update_src_caps (GstAV1Parse * self, GstCaps * caps)
620 {
621 GstCaps *sink_caps, *src_caps;
622 GstCaps *final_caps = NULL;
623 GstStructure *s = NULL;
624 gint width, height;
625 gint par_n = 0, par_d = 0;
626 gint fps_n = 0, fps_d = 0;
627 const gchar *profile = NULL;
628
629 if (G_UNLIKELY (!gst_pad_has_current_caps (GST_BASE_PARSE_SRC_PAD (self))))
630 self->update_caps = TRUE;
631
632 if (!self->update_caps)
633 return;
634
635 /* if this is being called from the first _setcaps call, caps on the sinkpad
636 * aren't set yet and so they need to be passed as an argument */
637 if (caps)
638 sink_caps = gst_caps_ref (caps);
639 else
640 sink_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SINK_PAD (self));
641
642 /* carry over input caps as much as possible; override with our own stuff */
643 if (!sink_caps)
644 sink_caps = gst_caps_new_empty_simple ("video/x-av1");
645 else
646 s = gst_caps_get_structure (sink_caps, 0);
647
648 final_caps = gst_caps_copy (sink_caps);
649
650 if (s && gst_structure_has_field (s, "width") &&
651 gst_structure_has_field (s, "height")) {
652 gst_structure_get_int (s, "width", &width);
653 gst_structure_get_int (s, "height", &height);
654 } else {
655 width = self->width;
656 height = self->height;
657 }
658
659 if (width > 0 && height > 0)
660 gst_caps_set_simple (final_caps, "width", G_TYPE_INT, width,
661 "height", G_TYPE_INT, height, NULL);
662
663 if (s && gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d)) {
664 if (par_n != 0 && par_d != 0) {
665 gst_caps_set_simple (final_caps, "pixel-aspect-ratio",
666 GST_TYPE_FRACTION, par_n, par_d, NULL);
667 }
668 }
669
670 if (s && gst_structure_has_field (s, "framerate")) {
671 gst_structure_get_fraction (s, "framerate", &fps_n, &fps_d);
672 }
673
674 if (fps_n > 0 && fps_d > 0) {
675 gst_caps_set_simple (final_caps, "framerate",
676 GST_TYPE_FRACTION, fps_n, fps_d, NULL);
677 gst_base_parse_set_frame_rate (GST_BASE_PARSE (self), fps_n, fps_d, 0, 0);
678 }
679
680 /* When not RGB, the chroma format is needed. */
681 if (self->colorimetry == NULL ||
682 (g_strcmp0 (self->colorimetry, GST_VIDEO_COLORIMETRY_SRGB) != 0)) {
683 const gchar *chroma_format = NULL;
684
685 if (self->subsampling_x == 1 && self->subsampling_y == 1) {
686 if (!self->mono_chrome) {
687 chroma_format = "4:2:0";
688 } else {
689 chroma_format = "4:0:0";
690 }
691 } else if (self->subsampling_x == 1 && self->subsampling_y == 0) {
692 chroma_format = "4:2:2";
693 } else if (self->subsampling_x == 0 && self->subsampling_y == 0) {
694 chroma_format = "4:4:4";
695 }
696
697 if (chroma_format)
698 gst_caps_set_simple (final_caps,
699 "chroma-format", G_TYPE_STRING, chroma_format, NULL);
700 }
701
702 if (self->bit_depth)
703 gst_caps_set_simple (final_caps,
704 "bit-depth-luma", G_TYPE_UINT, self->bit_depth,
705 "bit-depth-chroma", G_TYPE_UINT, self->bit_depth, NULL);
706
707 if (self->colorimetry && (!s || !gst_structure_has_field (s, "colorimetry")))
708 gst_caps_set_simple (final_caps,
709 "colorimetry", G_TYPE_STRING, self->colorimetry, NULL);
710
711 g_assert (self->align > GST_AV1_PARSE_ALIGN_NONE);
712 gst_caps_set_simple (final_caps, "parsed", G_TYPE_BOOLEAN, TRUE,
713 "stream-format", G_TYPE_STRING,
714 gst_av1_parse_alignment_to_steam_format_string (self->align),
715 "alignment", G_TYPE_STRING,
716 gst_av1_parse_alignment_to_string (self->align), NULL);
717
718 profile = gst_av1_parse_profile_to_string (self->profile);
719 if (profile)
720 gst_caps_set_simple (final_caps, "profile", G_TYPE_STRING, profile, NULL);
721
722 src_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (self));
723
724 if (!(src_caps && gst_caps_is_strictly_equal (src_caps, final_caps))) {
725 GST_DEBUG_OBJECT (self, "Update src caps %" GST_PTR_FORMAT, final_caps);
726 gst_pad_set_caps (GST_BASE_PARSE_SRC_PAD (self), final_caps);
727 }
728
729 gst_clear_caps (&src_caps);
730 gst_caps_unref (final_caps);
731 gst_caps_unref (sink_caps);
732
733 self->update_caps = FALSE;
734 }
735
736 /* check downstream caps to configure format and alignment */
737 static void
gst_av1_parse_negotiate(GstAV1Parse * self,GstCaps * in_caps)738 gst_av1_parse_negotiate (GstAV1Parse * self, GstCaps * in_caps)
739 {
740 GstCaps *caps;
741 GstAV1ParseAligment align = GST_AV1_PARSE_ALIGN_NONE;
742
743 caps = gst_pad_get_allowed_caps (GST_BASE_PARSE_SRC_PAD (self));
744 GST_DEBUG_OBJECT (self, "allowed caps: %" GST_PTR_FORMAT, caps);
745
746 /* concentrate on leading structure, since decodebin parser
747 * capsfilter always includes parser template caps */
748 if (caps) {
749 caps = gst_caps_truncate (caps);
750 GST_DEBUG_OBJECT (self, "negotiating with caps: %" GST_PTR_FORMAT, caps);
751 }
752
753 /* prefer TU as default */
754 if (gst_av1_parse_caps_has_alignment (caps,
755 GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT)) {
756 align = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
757 goto done;
758 }
759
760 /* Both upsteam and downstream support, best */
761 if (in_caps && caps) {
762 if (gst_caps_can_intersect (in_caps, caps)) {
763 GST_DEBUG_OBJECT (self, "downstream accepts upstream caps");
764 align = gst_av1_parse_alignment_from_caps (in_caps);
765 gst_clear_caps (&caps);
766 }
767 }
768 if (align != GST_AV1_PARSE_ALIGN_NONE)
769 goto done;
770
771 /* Select first one of downstream support */
772 if (caps && !gst_caps_is_empty (caps)) {
773 /* fixate to avoid ambiguity with lists when parsing */
774 caps = gst_caps_fixate (caps);
775 align = gst_av1_parse_alignment_from_caps (caps);
776 }
777 if (align != GST_AV1_PARSE_ALIGN_NONE)
778 goto done;
779
780 /* default */
781 if (align == GST_AV1_PARSE_ALIGN_NONE)
782 align = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
783
784 done:
785 self->align = align;
786 GST_INFO_OBJECT (self, "selected alignment %s",
787 gst_av1_parse_alignment_to_string (align));
788
789 gst_clear_caps (&caps);
790 }
791
792 static GstCaps *
gst_av1_parse_get_sink_caps(GstBaseParse * parse,GstCaps * filter)793 gst_av1_parse_get_sink_caps (GstBaseParse * parse, GstCaps * filter)
794 {
795 GstCaps *peercaps, *templ;
796 GstCaps *res, *tmp, *pcopy;
797
798 templ = gst_pad_get_pad_template_caps (GST_BASE_PARSE_SINK_PAD (parse));
799 if (filter) {
800 GstCaps *fcopy = gst_caps_copy (filter);
801 /* Remove the fields we convert */
802 remove_fields (fcopy, TRUE);
803 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), fcopy);
804 gst_caps_unref (fcopy);
805 } else {
806 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), NULL);
807 }
808
809 pcopy = gst_caps_copy (peercaps);
810 remove_fields (pcopy, TRUE);
811
812 res = gst_caps_intersect_full (pcopy, templ, GST_CAPS_INTERSECT_FIRST);
813 gst_caps_unref (pcopy);
814 gst_caps_unref (templ);
815
816 if (filter) {
817 GstCaps *tmp = gst_caps_intersect_full (res, filter,
818 GST_CAPS_INTERSECT_FIRST);
819 gst_caps_unref (res);
820 res = tmp;
821 }
822
823 /* Try if we can put the downstream caps first */
824 pcopy = gst_caps_copy (peercaps);
825 remove_fields (pcopy, FALSE);
826 tmp = gst_caps_intersect_full (pcopy, res, GST_CAPS_INTERSECT_FIRST);
827 gst_caps_unref (pcopy);
828 if (!gst_caps_is_empty (tmp))
829 res = gst_caps_merge (tmp, res);
830 else
831 gst_caps_unref (tmp);
832
833 gst_caps_unref (peercaps);
834
835 return res;
836 }
837
838 static gboolean
gst_av1_parse_set_sink_caps(GstBaseParse * parse,GstCaps * caps)839 gst_av1_parse_set_sink_caps (GstBaseParse * parse, GstCaps * caps)
840 {
841 GstAV1Parse *self = GST_AV1_PARSE (parse);
842 GstStructure *str;
843 GstAV1ParseAligment align;
844 GstCaps *in_caps = NULL;
845 const gchar *profile;
846
847 str = gst_caps_get_structure (caps, 0);
848
849 /* accept upstream info if provided */
850 gst_structure_get_int (str, "width", &self->width);
851 gst_structure_get_int (str, "height", &self->height);
852 profile = gst_structure_get_string (str, "profile");
853 if (profile)
854 self->profile = gst_av1_parse_profile_from_string (profile);
855
856 /* get upstream align from caps */
857 align = gst_av1_parse_alignment_from_caps (caps);
858 if (align == GST_AV1_PARSE_ALIGN_ERROR) {
859 GST_ERROR_OBJECT (self, "Sink caps %" GST_PTR_FORMAT " set stream-format"
860 " and alignment conflict.", caps);
861 return FALSE;
862 }
863
864 in_caps = gst_caps_copy (caps);
865 /* default */
866 if (align == GST_AV1_PARSE_ALIGN_NONE)
867 gst_caps_set_simple (in_caps, "alignment", G_TYPE_STRING,
868 gst_av1_parse_alignment_to_string (GST_AV1_PARSE_ALIGN_BYTE), NULL);
869
870 /* negotiate with downstream, set output align */
871 gst_av1_parse_negotiate (self, in_caps);
872
873 self->update_caps = TRUE;
874
875 /* if all of decoder's capability related values are provided
876 * by upstream, update src caps now */
877 if (self->width > 0 && self->height > 0 && profile)
878 gst_av1_parse_update_src_caps (self, in_caps);
879
880 gst_caps_unref (in_caps);
881
882 self->in_align = align;
883
884 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
885 gst_av1_parser_reset (self->parser, TRUE);
886 } else {
887 gst_av1_parser_reset (self->parser, FALSE);
888 }
889
890 return TRUE;
891 }
892
893 static GstFlowReturn
gst_av1_parse_push_data(GstAV1Parse * self,GstBaseParseFrame * frame,guint32 finish_sz,gboolean frame_finished)894 gst_av1_parse_push_data (GstAV1Parse * self, GstBaseParseFrame * frame,
895 guint32 finish_sz, gboolean frame_finished)
896 {
897 gsize sz;
898 GstBuffer *buf, *header_buf;
899 GstBuffer *buffer = frame->buffer;
900 GstFlowReturn ret = GST_FLOW_OK;
901
902 /* Need to generate the final TU annex-b format */
903 if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
904 guint8 size_data[GST_AV1_MAX_LEB_128_SIZE];
905 guint size_len = 0;
906 guint len;
907
908 /* When push a TU, it must also be a frame end. */
909 g_assert (frame_finished);
910
911 /* Still some left in the frame cache */
912 len = gst_adapter_available (self->frame_cache);
913 if (len) {
914 buf = gst_adapter_take_buffer (self->frame_cache, len);
915
916 /* frame_unit_size */
917 _write_leb128 (size_data, &size_len, len);
918 header_buf = gst_buffer_new_memdup (size_data, size_len);
919 GST_BUFFER_PTS (header_buf) = GST_BUFFER_PTS (buf);
920 GST_BUFFER_DTS (header_buf) = GST_BUFFER_DTS (buf);
921 GST_BUFFER_DURATION (header_buf) = GST_BUFFER_DURATION (buf);
922
923 gst_adapter_push (self->cache_out, header_buf);
924 gst_adapter_push (self->cache_out, buf);
925 }
926
927 len = gst_adapter_available (self->cache_out);
928 if (len) {
929 buf = gst_adapter_take_buffer (self->cache_out, len);
930
931 /* temporal_unit_size */
932 _write_leb128 (size_data, &size_len, len);
933 header_buf = gst_buffer_new_memdup (size_data, size_len);
934 GST_BUFFER_PTS (header_buf) = GST_BUFFER_PTS (buf);
935 GST_BUFFER_DTS (header_buf) = GST_BUFFER_DTS (buf);
936 GST_BUFFER_DURATION (header_buf) = GST_BUFFER_DURATION (buf);
937
938 gst_adapter_push (self->cache_out, header_buf);
939 gst_adapter_push (self->cache_out, buf);
940 }
941 }
942
943 sz = gst_adapter_available (self->cache_out);
944 if (sz) {
945 buf = gst_adapter_take_buffer (self->cache_out, sz);
946 gst_buffer_copy_into (buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
947 if (self->discont) {
948 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DISCONT);
949 self->discont = FALSE;
950 }
951 if (self->header) {
952 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_HEADER);
953 self->header = FALSE;
954 }
955 if (self->keyframe) {
956 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
957 self->keyframe = FALSE;
958 } else {
959 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
960 }
961
962 if (frame_finished)
963 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_MARKER);
964
965 if (self->align == GST_AV1_PARSE_ALIGN_FRAME) {
966 if (!self->show_frame) {
967 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DECODE_ONLY);
968 } else {
969 GST_BUFFER_FLAG_UNSET (buf, GST_BUFFER_FLAG_DECODE_ONLY);
970 }
971 }
972
973 gst_buffer_replace (&frame->out_buffer, buf);
974 gst_buffer_unref (buf);
975
976 gst_av1_parse_update_src_caps (self, NULL);
977 GST_LOG_OBJECT (self, "comsumed %d, output one buffer with size %"
978 G_GSSIZE_FORMAT, finish_sz, sz);
979 ret = gst_base_parse_finish_frame (GST_BASE_PARSE (self), frame, finish_sz);
980 }
981
982 return ret;
983 }
984
985 static void
gst_av1_parse_convert_to_annexb(GstAV1Parse * self,GstBuffer * buffer,GstAV1OBU * obu,gboolean frame_complete)986 gst_av1_parse_convert_to_annexb (GstAV1Parse * self, GstBuffer * buffer,
987 GstAV1OBU * obu, gboolean frame_complete)
988 {
989 guint8 size_data[GST_AV1_MAX_LEB_128_SIZE];
990 guint size_len = 0;
991 GstBitWriter bs;
992 GstBuffer *buf, *buf2;
993 guint8 *data;
994 guint len, len2, offset;
995
996 /* obu_length */
997 _write_leb128 (size_data, &size_len,
998 obu->obu_size + 1 + obu->header.obu_extention_flag);
999
1000 gst_bit_writer_init_with_size (&bs, 128, FALSE);
1001 /* obu_forbidden_bit */
1002 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1003 /* obu_type */
1004 gst_bit_writer_put_bits_uint8 (&bs, obu->obu_type, 4);
1005 /* obu_extension_flag */
1006 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_extention_flag, 1);
1007 /* obu_has_size_field */
1008 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1009 /* obu_reserved_1bit */
1010 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1011 if (obu->header.obu_extention_flag) {
1012 /* temporal_id */
1013 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_temporal_id, 3);
1014 /* spatial_id */
1015 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_spatial_id, 2);
1016 /* extension_header_reserved_3bits */
1017 gst_bit_writer_put_bits_uint8 (&bs, 0, 3);
1018 }
1019 g_assert (GST_BIT_WRITER_BIT_SIZE (&bs) % 8 == 0);
1020
1021 len = size_len;
1022 len += GST_BIT_WRITER_BIT_SIZE (&bs) / 8;
1023 len += obu->obu_size;
1024
1025 data = g_malloc (len);
1026 offset = 0;
1027
1028 memcpy (data + offset, size_data, size_len);
1029 offset += size_len;
1030
1031 memcpy (data + offset, GST_BIT_WRITER_DATA (&bs),
1032 GST_BIT_WRITER_BIT_SIZE (&bs) / 8);
1033 offset += GST_BIT_WRITER_BIT_SIZE (&bs) / 8;
1034
1035 memcpy (data + offset, obu->data, obu->obu_size);
1036
1037 /* The buf of this OBU */
1038 buf = gst_buffer_new_wrapped (data, len);
1039 GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (buffer);
1040 GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (buffer);
1041 GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (buffer);
1042
1043 gst_adapter_push (self->frame_cache, buf);
1044
1045 if (frame_complete) {
1046 len2 = gst_adapter_available (self->frame_cache);
1047 buf2 = gst_adapter_take_buffer (self->frame_cache, len2);
1048
1049 /* frame_unit_size */
1050 _write_leb128 (size_data, &size_len, len2);
1051 buf = gst_buffer_new_memdup (size_data, size_len);
1052 GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (buf2);
1053 GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (buf2);
1054 GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (buf2);
1055
1056 gst_adapter_push (self->cache_out, buf);
1057 gst_adapter_push (self->cache_out, buf2);
1058 }
1059
1060 gst_bit_writer_reset (&bs);
1061 }
1062
1063 static void
gst_av1_parse_convert_from_annexb(GstAV1Parse * self,GstBuffer * buffer,GstAV1OBU * obu)1064 gst_av1_parse_convert_from_annexb (GstAV1Parse * self, GstBuffer * buffer,
1065 GstAV1OBU * obu)
1066 {
1067 guint8 size_data[GST_AV1_MAX_LEB_128_SIZE];
1068 guint size_len = 0;
1069 GstBuffer *buf;
1070 guint len, offset;
1071 guint8 *data;
1072 GstBitWriter bs;
1073
1074 _write_leb128 (size_data, &size_len, obu->obu_size);
1075
1076 /* obu_header */
1077 len = 1;
1078 if (obu->header.obu_extention_flag)
1079 len += 1;
1080 len += size_len;
1081 len += obu->obu_size;
1082
1083 gst_bit_writer_init_with_size (&bs, 128, FALSE);
1084 /* obu_forbidden_bit */
1085 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1086 /* obu_type */
1087 gst_bit_writer_put_bits_uint8 (&bs, obu->obu_type, 4);
1088 /* obu_extension_flag */
1089 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_extention_flag, 1);
1090 /* obu_has_size_field */
1091 gst_bit_writer_put_bits_uint8 (&bs, 1, 1);
1092 /* obu_reserved_1bit */
1093 gst_bit_writer_put_bits_uint8 (&bs, 0, 1);
1094 if (obu->header.obu_extention_flag) {
1095 /* temporal_id */
1096 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_temporal_id, 3);
1097 /* spatial_id */
1098 gst_bit_writer_put_bits_uint8 (&bs, obu->header.obu_spatial_id, 2);
1099 /* extension_header_reserved_3bits */
1100 gst_bit_writer_put_bits_uint8 (&bs, 0, 3);
1101 }
1102 g_assert (GST_BIT_WRITER_BIT_SIZE (&bs) % 8 == 0);
1103
1104 data = g_malloc (len);
1105 offset = 0;
1106 memcpy (data + offset, GST_BIT_WRITER_DATA (&bs),
1107 GST_BIT_WRITER_BIT_SIZE (&bs) / 8);
1108 offset += GST_BIT_WRITER_BIT_SIZE (&bs) / 8;
1109
1110 memcpy (data + offset, size_data, size_len);
1111 offset += size_len;
1112
1113 memcpy (data + offset, obu->data, obu->obu_size);
1114
1115 buf = gst_buffer_new_wrapped (data, len);
1116 GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (buffer);
1117 GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (buffer);
1118 GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (buffer);
1119
1120 gst_adapter_push (self->cache_out, buf);
1121
1122 gst_bit_writer_reset (&bs);
1123 }
1124
1125 static void
gst_av1_parse_cache_one_obu(GstAV1Parse * self,GstBuffer * buffer,GstAV1OBU * obu,guint8 * data,guint32 size,gboolean frame_complete)1126 gst_av1_parse_cache_one_obu (GstAV1Parse * self, GstBuffer * buffer,
1127 GstAV1OBU * obu, guint8 * data, guint32 size, gboolean frame_complete)
1128 {
1129 gboolean need_convert = FALSE;
1130 GstBuffer *buf;
1131
1132 if (self->in_align != self->align
1133 && (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B
1134 || self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B))
1135 need_convert = TRUE;
1136
1137 if (need_convert) {
1138 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
1139 gst_av1_parse_convert_from_annexb (self, buffer, obu);
1140 } else {
1141 gst_av1_parse_convert_to_annexb (self, buffer, obu, frame_complete);
1142 }
1143 } else if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
1144 g_assert (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B);
1145 gst_av1_parse_convert_to_annexb (self, buffer, obu, frame_complete);
1146 } else {
1147 buf = gst_buffer_new_memdup (data, size);
1148 GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (buffer);
1149 GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (buffer);
1150 GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (buffer);
1151
1152 gst_adapter_push (self->cache_out, buf);
1153 }
1154 }
1155
1156 static GstAV1ParserResult
gst_av1_parse_handle_sequence_obu(GstAV1Parse * self,GstAV1OBU * obu)1157 gst_av1_parse_handle_sequence_obu (GstAV1Parse * self, GstAV1OBU * obu)
1158 {
1159 GstAV1SequenceHeaderOBU seq_header;
1160 GstAV1ParserResult res;
1161 guint i;
1162 guint val;
1163
1164 res = gst_av1_parser_parse_sequence_header_obu (self->parser,
1165 obu, &seq_header);
1166 if (res != GST_AV1_PARSER_OK)
1167 return res;
1168
1169 if (self->width != seq_header.max_frame_width_minus_1 + 1) {
1170 self->width = seq_header.max_frame_width_minus_1 + 1;
1171 self->update_caps = TRUE;
1172 }
1173 if (self->height != seq_header.max_frame_height_minus_1 + 1) {
1174 self->height = seq_header.max_frame_height_minus_1 + 1;
1175 self->update_caps = TRUE;
1176 }
1177
1178 if (seq_header.color_config.color_description_present_flag) {
1179 GstVideoColorimetry cinfo;
1180 gchar *colorimetry = NULL;
1181
1182 if (seq_header.color_config.color_range)
1183 cinfo.range = GST_VIDEO_COLOR_RANGE_0_255;
1184 else
1185 cinfo.range = GST_VIDEO_COLOR_RANGE_16_235;
1186
1187 cinfo.matrix = gst_video_color_matrix_from_iso
1188 (seq_header.color_config.matrix_coefficients);
1189 cinfo.transfer = gst_video_transfer_function_from_iso
1190 (seq_header.color_config.transfer_characteristics);
1191 cinfo.primaries = gst_video_color_primaries_from_iso
1192 (seq_header.color_config.color_primaries);
1193
1194 colorimetry = gst_video_colorimetry_to_string (&cinfo);
1195
1196 if (g_strcmp0 (colorimetry, self->colorimetry) != 0) {
1197 g_free (self->colorimetry);
1198 self->colorimetry = colorimetry;
1199 colorimetry = NULL;
1200 self->update_caps = TRUE;
1201 }
1202
1203 g_clear_pointer (&colorimetry, g_free);
1204 }
1205
1206 if (self->subsampling_x != seq_header.color_config.subsampling_x) {
1207 self->subsampling_x = seq_header.color_config.subsampling_x;
1208 self->update_caps = TRUE;
1209 }
1210
1211 if (self->subsampling_y != seq_header.color_config.subsampling_y) {
1212 self->subsampling_y = seq_header.color_config.subsampling_y;
1213 self->update_caps = TRUE;
1214 }
1215
1216 if (self->mono_chrome != seq_header.color_config.mono_chrome) {
1217 self->mono_chrome = seq_header.color_config.mono_chrome;
1218 self->update_caps = TRUE;
1219 }
1220
1221 if (self->bit_depth != seq_header.bit_depth) {
1222 self->bit_depth = seq_header.bit_depth;
1223 self->update_caps = TRUE;
1224 }
1225
1226 if (self->profile != seq_header.seq_profile) {
1227 self->profile = seq_header.seq_profile;
1228 self->update_caps = TRUE;
1229 }
1230
1231 val = (self->parser->state.operating_point_idc >> 8) & 0x0f;
1232 for (i = 0; i < (1 << GST_AV1_MAX_SPATIAL_LAYERS); i++) {
1233 if (val & (1 << i))
1234 self->highest_spatial_id = i;
1235 }
1236
1237 return GST_AV1_PARSER_OK;
1238 }
1239
1240 /* Check whether the frame start a new TU.
1241 The obu here should be a shown frame/frame header. */
1242 static gboolean
gst_av1_parse_frame_start_new_temporal_unit(GstAV1Parse * self,GstAV1OBU * obu)1243 gst_av1_parse_frame_start_new_temporal_unit (GstAV1Parse * self,
1244 GstAV1OBU * obu)
1245 {
1246 gboolean ret = FALSE;
1247
1248 g_assert (obu->obu_type == GST_AV1_OBU_FRAME_HEADER
1249 || obu->obu_type == GST_AV1_OBU_FRAME);
1250
1251 /* 7.5.Ordering of OBUs: The value of temporal_id must be the same in all
1252 OBU extension headers that are contained in the same temporal unit. */
1253 if (self->last_shown_frame_temporal_id >= 0 &&
1254 obu->header.obu_temporal_id != self->last_shown_frame_temporal_id) {
1255 ret = TRUE;
1256 goto new_tu;
1257 }
1258
1259 /* If scalability is not being used, only one shown frame for each
1260 temporal unit. So the new frame belongs to a new temporal unit. */
1261 if (!self->within_one_frame && self->last_shown_frame_temporal_id >= 0 &&
1262 self->parser->state.operating_point_idc == 0) {
1263 ret = TRUE;
1264 goto new_tu;
1265 }
1266
1267 /* The new frame has the same layer IDs with the last shown frame,
1268 it should belong to a new temporal unit. */
1269 if (!self->within_one_frame &&
1270 obu->header.obu_temporal_id == self->last_shown_frame_temporal_id &&
1271 obu->header.obu_spatial_id == self->last_shown_frame_spatial_id) {
1272 ret = TRUE;
1273 goto new_tu;
1274 }
1275
1276 new_tu:
1277 if (ret) {
1278 if (self->within_one_frame)
1279 GST_WARNING_OBJECT (self,
1280 "Start a new temporal unit with incompleted frame.");
1281
1282 gst_av1_parse_reset_obu_data_state (self);
1283 }
1284
1285 return ret;
1286 }
1287
1288 /* frame_complete will be set true if it is the frame edge. */
1289 static GstAV1ParserResult
gst_av1_parse_handle_one_obu(GstAV1Parse * self,GstAV1OBU * obu,gboolean * frame_complete,gboolean * check_new_tu)1290 gst_av1_parse_handle_one_obu (GstAV1Parse * self, GstAV1OBU * obu,
1291 gboolean * frame_complete, gboolean * check_new_tu)
1292 {
1293 GstAV1ParserResult res = GST_AV1_PARSER_OK;
1294 GstAV1MetadataOBU metadata;
1295 GstAV1FrameHeaderOBU frame_header;
1296 GstAV1TileListOBU tile_list;
1297 GstAV1TileGroupOBU tile_group;
1298 GstAV1FrameOBU frame;
1299
1300 *frame_complete = FALSE;
1301
1302 switch (obu->obu_type) {
1303 case GST_AV1_OBU_TEMPORAL_DELIMITER:
1304 res = gst_av1_parser_parse_temporal_delimiter_obu (self->parser, obu);
1305 break;
1306 case GST_AV1_OBU_SEQUENCE_HEADER:
1307 res = gst_av1_parse_handle_sequence_obu (self, obu);
1308 break;
1309 case GST_AV1_OBU_REDUNDANT_FRAME_HEADER:
1310 res = gst_av1_parser_parse_frame_header_obu (self->parser, obu,
1311 &frame_header);
1312 break;
1313 case GST_AV1_OBU_FRAME_HEADER:
1314 res = gst_av1_parser_parse_frame_header_obu (self->parser, obu,
1315 &frame_header);
1316 break;
1317 case GST_AV1_OBU_FRAME:
1318 res = gst_av1_parser_parse_frame_obu (self->parser, obu, &frame);
1319 break;
1320 case GST_AV1_OBU_METADATA:
1321 res = gst_av1_parser_parse_metadata_obu (self->parser, obu, &metadata);
1322 break;
1323 case GST_AV1_OBU_TILE_GROUP:
1324 res =
1325 gst_av1_parser_parse_tile_group_obu (self->parser, obu, &tile_group);
1326 break;
1327 case GST_AV1_OBU_TILE_LIST:
1328 res = gst_av1_parser_parse_tile_list_obu (self->parser, obu, &tile_list);
1329 break;
1330 case GST_AV1_OBU_PADDING:
1331 break;
1332 default:
1333 GST_WARNING_OBJECT (self, "an unrecognized obu type %d", obu->obu_type);
1334 res = GST_AV1_PARSER_BITSTREAM_ERROR;
1335 break;
1336 }
1337
1338 GST_LOG_OBJECT (self, "parsing the obu %s, result is %d",
1339 _obu_name (obu->obu_type), res);
1340 if (res != GST_AV1_PARSER_OK)
1341 goto out;
1342
1343 /* 7.5:
1344 All OBU extension headers that are contained in the same temporal
1345 unit and have the same spatial_id value must have the same temporal_id
1346 value.
1347 And
1348 OBUs with spatial level IDs (spatial_id) greater than 0 must
1349 appear within a temporal unit in increasing order of the spatial
1350 level ID values. */
1351 if (obu->header.obu_spatial_id > self->highest_spatial_id) {
1352 GST_WARNING_OBJECT (self,
1353 "spatial_id %d is bigger than highest_spatial_id %d",
1354 obu->header.obu_spatial_id, self->highest_spatial_id);
1355 res = GST_AV1_PARSER_BITSTREAM_ERROR;
1356 goto out;
1357 }
1358
1359 /* If to check a new temporal starts, return early.
1360 In 7.5.Ordering of OBUs: Sequence header OBUs may appear in any order
1361 within a coded video sequence. So it is allowed to repeat the sequence
1362 header within one temporal unit, and sequence header does not definitely
1363 start a TU. We only check TD here. */
1364 if (obu->obu_type == GST_AV1_OBU_TEMPORAL_DELIMITER) {
1365 gst_av1_parse_reset_obu_data_state (self);
1366
1367 if (check_new_tu) {
1368 *check_new_tu = TRUE;
1369 res = GST_AV1_PARSER_OK;
1370 goto out;
1371 }
1372 }
1373
1374 if (obu->obu_type == GST_AV1_OBU_SEQUENCE_HEADER)
1375 self->header = TRUE;
1376
1377 if (obu->obu_type == GST_AV1_OBU_FRAME_HEADER
1378 || obu->obu_type == GST_AV1_OBU_FRAME
1379 || obu->obu_type == GST_AV1_OBU_REDUNDANT_FRAME_HEADER) {
1380 GstAV1FrameHeaderOBU *fh = &frame_header;
1381
1382 if (obu->obu_type == GST_AV1_OBU_FRAME)
1383 fh = &frame.frame_header;
1384
1385 self->show_frame = fh->show_frame || fh->show_existing_frame;
1386 if (self->show_frame) {
1387 /* Check whether a new temporal starts, and return early. */
1388 if (check_new_tu && obu->obu_type != GST_AV1_OBU_REDUNDANT_FRAME_HEADER
1389 && gst_av1_parse_frame_start_new_temporal_unit (self, obu)) {
1390 *check_new_tu = TRUE;
1391 res = GST_AV1_PARSER_OK;
1392 goto out;
1393 }
1394
1395 self->last_shown_frame_temporal_id = obu->header.obu_temporal_id;
1396 self->last_shown_frame_spatial_id = obu->header.obu_spatial_id;
1397 }
1398
1399 self->within_one_frame = TRUE;
1400
1401 /* if a show_existing_frame case, only update key frame.
1402 otherwise, update all type of frame. */
1403 if (!fh->show_existing_frame || fh->frame_type == GST_AV1_KEY_FRAME)
1404 res = gst_av1_parser_reference_frame_update (self->parser, fh);
1405
1406 if (res != GST_AV1_PARSER_OK)
1407 GST_WARNING_OBJECT (self, "update frame get result %d", res);
1408
1409 if (fh->show_existing_frame) {
1410 *frame_complete = TRUE;
1411 self->within_one_frame = FALSE;
1412 }
1413
1414 if (fh->frame_type == GST_AV1_KEY_FRAME)
1415 self->keyframe = TRUE;
1416 }
1417
1418 if (obu->obu_type == GST_AV1_OBU_TILE_GROUP
1419 || obu->obu_type == GST_AV1_OBU_FRAME) {
1420 GstAV1TileGroupOBU *tg = &tile_group;
1421
1422 self->within_one_frame = TRUE;
1423
1424 if (obu->obu_type == GST_AV1_OBU_FRAME)
1425 tg = &frame.tile_group;
1426
1427 if (tg->tg_end == tg->num_tiles - 1) {
1428 *frame_complete = TRUE;
1429 self->within_one_frame = FALSE;
1430 }
1431 }
1432
1433 out:
1434 if (res != GST_AV1_PARSER_OK) {
1435 /* Some verbose OBU can be skip */
1436 if (obu->obu_type == GST_AV1_OBU_REDUNDANT_FRAME_HEADER) {
1437 GST_WARNING_OBJECT (self, "Ignore a verbose %s OBU parsing error",
1438 _obu_name (obu->obu_type));
1439 gst_av1_parse_reset_obu_data_state (self);
1440 res = GST_AV1_PARSER_OK;
1441 }
1442 }
1443
1444 return res;
1445 }
1446
1447 static GstFlowReturn
gst_av1_parse_handle_obu_to_obu(GstBaseParse * parse,GstBaseParseFrame * frame,gint * skipsize)1448 gst_av1_parse_handle_obu_to_obu (GstBaseParse * parse,
1449 GstBaseParseFrame * frame, gint * skipsize)
1450 {
1451 GstAV1Parse *self = GST_AV1_PARSE (parse);
1452 GstMapInfo map_info;
1453 GstAV1OBU obu;
1454 GstFlowReturn ret = GST_FLOW_OK;
1455 GstAV1ParserResult res;
1456 GstBuffer *buffer = gst_buffer_ref (frame->buffer);
1457 guint32 consumed;
1458 gboolean frame_complete;
1459
1460 if (!gst_buffer_map (buffer, &map_info, GST_MAP_READ)) {
1461 *skipsize = 0;
1462 GST_ERROR_OBJECT (parse, "Couldn't map incoming buffer");
1463 return GST_FLOW_ERROR;
1464 }
1465
1466 consumed = 0;
1467 frame_complete = FALSE;
1468 res = gst_av1_parser_identify_one_obu (self->parser, map_info.data,
1469 map_info.size, &obu, &consumed);
1470 if (res == GST_AV1_PARSER_OK)
1471 res = gst_av1_parse_handle_one_obu (self, &obu, &frame_complete, NULL);
1472
1473 g_assert (consumed <= map_info.size);
1474
1475 if (res == GST_AV1_PARSER_BITSTREAM_ERROR ||
1476 res == GST_AV1_PARSER_MISSING_OBU_REFERENCE) {
1477 if (consumed) {
1478 *skipsize = consumed;
1479 } else {
1480 *skipsize = map_info.size;
1481 }
1482 GST_WARNING_OBJECT (parse, "Parse obu error, discard %d.", *skipsize);
1483 gst_av1_parse_reset_obu_data_state (self);
1484 ret = GST_FLOW_OK;
1485 goto out;
1486 } else if (res == GST_AV1_PARSER_NO_MORE_DATA) {
1487 *skipsize = 0;
1488
1489 if (self->in_align == GST_AV1_PARSE_ALIGN_OBU) {
1490 /* The buffer is already aligned to OBU, should not happen. */
1491 if (consumed) {
1492 *skipsize = consumed;
1493 } else {
1494 *skipsize = map_info.size;
1495 }
1496 GST_WARNING_OBJECT (parse, "Parse obu need more data, discard %d.",
1497 *skipsize);
1498 gst_av1_parse_reset_obu_data_state (self);
1499 }
1500 ret = GST_FLOW_OK;
1501 goto out;
1502 } else if (res == GST_AV1_PARSER_DROP) {
1503 GST_DEBUG_OBJECT (parse, "Drop %d data", consumed);
1504 *skipsize = consumed;
1505 gst_av1_parse_reset_obu_data_state (self);
1506 ret = GST_FLOW_OK;
1507 goto out;
1508 } else if (res != GST_AV1_PARSER_OK) {
1509 GST_ERROR_OBJECT (parse, "Parse obu get unexpect error %d", res);
1510 *skipsize = 0;
1511 ret = GST_FLOW_ERROR;
1512 goto out;
1513 }
1514
1515 g_assert (consumed);
1516
1517 gst_av1_parse_update_src_caps (self, NULL);
1518 if (self->discont) {
1519 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DISCONT);
1520 self->discont = FALSE;
1521 }
1522 if (self->header) {
1523 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_HEADER);
1524 self->header = FALSE;
1525 }
1526 /* happen to be a frame boundary */
1527 if (frame_complete)
1528 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_MARKER);
1529
1530 GST_LOG_OBJECT (self, "Output one buffer with size %d", consumed);
1531 ret = gst_base_parse_finish_frame (parse, frame, consumed);
1532 *skipsize = 0;
1533
1534 out:
1535 gst_buffer_unmap (buffer, &map_info);
1536 gst_buffer_unref (buffer);
1537 return ret;
1538 }
1539
1540 static GstFlowReturn
gst_av1_parse_handle_to_small_and_equal_align(GstBaseParse * parse,GstBaseParseFrame * frame,gint * skipsize)1541 gst_av1_parse_handle_to_small_and_equal_align (GstBaseParse * parse,
1542 GstBaseParseFrame * frame, gint * skipsize)
1543 {
1544 GstAV1Parse *self = GST_AV1_PARSE (parse);
1545 GstMapInfo map_info;
1546 GstAV1OBU obu;
1547 GstFlowReturn ret = GST_FLOW_OK;
1548 GstAV1ParserResult res = GST_AV1_PARSER_INVALID_OPERATION;
1549 GstBuffer *buffer = gst_buffer_ref (frame->buffer);
1550 guint32 total_consumed, consumed;
1551 gboolean frame_complete;
1552
1553 if (!gst_buffer_map (buffer, &map_info, GST_MAP_READ)) {
1554 GST_ERROR_OBJECT (parse, "Couldn't map incoming buffer");
1555 return GST_FLOW_ERROR;
1556 }
1557
1558 total_consumed = 0;
1559 frame_complete = FALSE;
1560 again:
1561 while (total_consumed < map_info.size) {
1562 res = gst_av1_parser_identify_one_obu (self->parser,
1563 map_info.data + total_consumed, map_info.size - total_consumed,
1564 &obu, &consumed);
1565 if (res == GST_AV1_PARSER_OK)
1566 res = gst_av1_parse_handle_one_obu (self, &obu, &frame_complete, NULL);
1567 if (res != GST_AV1_PARSER_OK)
1568 break;
1569
1570 if (obu.obu_type == GST_AV1_OBU_TEMPORAL_DELIMITER && total_consumed) {
1571 GST_DEBUG_OBJECT (self, "Encounter TD inside one %s aligned"
1572 " buffer, should not happen normally.",
1573 gst_av1_parse_alignment_to_string (self->in_align));
1574 frame_complete = TRUE;
1575 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
1576 gst_av1_parser_reset_annex_b (self->parser);
1577 /* Not include this TD obu, it should belong to the next TU or frame */
1578 break;
1579 }
1580
1581 gst_av1_parse_cache_one_obu (self, buffer, &obu,
1582 map_info.data + total_consumed, consumed, frame_complete);
1583
1584 total_consumed += consumed;
1585
1586 if (self->align == GST_AV1_PARSE_ALIGN_OBU)
1587 break;
1588
1589 if (self->align == GST_AV1_PARSE_ALIGN_FRAME && frame_complete)
1590 break;
1591 }
1592
1593 if (res == GST_AV1_PARSER_BITSTREAM_ERROR ||
1594 res == GST_AV1_PARSER_MISSING_OBU_REFERENCE) {
1595 /* Discard the whole frame */
1596 *skipsize = map_info.size;
1597 GST_WARNING_OBJECT (parse, "Parse obu error, discard %d", *skipsize);
1598 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
1599 gst_av1_parser_reset_annex_b (self->parser);
1600 gst_av1_parse_reset_obu_data_state (self);
1601 ret = GST_FLOW_OK;
1602 goto out;
1603 } else if (res == GST_AV1_PARSER_NO_MORE_DATA) {
1604 /* Discard the whole buffer */
1605 *skipsize = map_info.size;
1606 GST_WARNING_OBJECT (parse, "Parse obu need more data, discard %d.",
1607 *skipsize);
1608 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
1609 gst_av1_parser_reset_annex_b (self->parser);
1610
1611 gst_av1_parse_reset_obu_data_state (self);
1612 ret = GST_FLOW_OK;
1613 goto out;
1614 } else if (res == GST_AV1_PARSER_DROP) {
1615 GST_DEBUG_OBJECT (parse, "Drop %d data", consumed);
1616 total_consumed += consumed;
1617 gst_av1_parse_reset_obu_data_state (self);
1618 res = GST_AV1_PARSER_OK;
1619 goto again;
1620 } else if (res != GST_AV1_PARSER_OK) {
1621 GST_ERROR_OBJECT (parse, "Parse obu get unexpect error %d", res);
1622 *skipsize = 0;
1623 ret = GST_FLOW_ERROR;
1624 goto out;
1625 }
1626
1627 g_assert (total_consumed >= map_info.size || frame_complete
1628 || self->align == GST_AV1_PARSE_ALIGN_OBU);
1629
1630 if (total_consumed >= map_info.size && !frame_complete
1631 && self->align == GST_AV1_PARSE_ALIGN_FRAME) {
1632 /* Warning and still consider this frame as complete */
1633 GST_WARNING_OBJECT (self, "Exhaust the buffer but still incomplete frame,"
1634 " should not happend in %s alignment",
1635 gst_av1_parse_alignment_to_string (self->in_align));
1636 }
1637
1638 ret = gst_av1_parse_push_data (self, frame, total_consumed, frame_complete);
1639
1640 out:
1641 gst_buffer_unmap (buffer, &map_info);
1642 gst_buffer_unref (buffer);
1643 return ret;
1644 }
1645
1646 static GstFlowReturn
gst_av1_parse_handle_to_big_align(GstBaseParse * parse,GstBaseParseFrame * frame,gint * skipsize)1647 gst_av1_parse_handle_to_big_align (GstBaseParse * parse,
1648 GstBaseParseFrame * frame, gint * skipsize)
1649 {
1650 GstAV1Parse *self = GST_AV1_PARSE (parse);
1651 GstMapInfo map_info;
1652 GstAV1OBU obu;
1653 GstFlowReturn ret = GST_FLOW_OK;
1654 GstAV1ParserResult res = GST_AV1_PARSER_OK;
1655 GstBuffer *buffer = gst_buffer_ref (frame->buffer);
1656 guint32 consumed;
1657 gboolean frame_complete;
1658 gboolean check_new_tu;
1659 gboolean complete;
1660
1661 g_assert (self->in_align <= GST_AV1_PARSE_ALIGN_FRAME);
1662
1663 if (!gst_buffer_map (buffer, &map_info, GST_MAP_READ)) {
1664 *skipsize = 0;
1665 GST_ERROR_OBJECT (parse, "Couldn't map incoming buffer");
1666 return GST_FLOW_ERROR;
1667 }
1668
1669 complete = FALSE;
1670 again:
1671 while (self->last_parsed_offset < map_info.size) {
1672 res = gst_av1_parser_identify_one_obu (self->parser,
1673 map_info.data + self->last_parsed_offset,
1674 map_info.size - self->last_parsed_offset, &obu, &consumed);
1675 if (res != GST_AV1_PARSER_OK)
1676 break;
1677
1678 check_new_tu = FALSE;
1679 res = gst_av1_parse_handle_one_obu (self, &obu, &frame_complete,
1680 &check_new_tu);
1681 if (res != GST_AV1_PARSER_OK)
1682 break;
1683
1684 if (check_new_tu && (gst_adapter_available (self->cache_out) ||
1685 gst_adapter_available (self->frame_cache))) {
1686 complete = TRUE;
1687 break;
1688 }
1689
1690 if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT ||
1691 self->align == GST_AV1_PARSE_ALIGN_FRAME) {
1692 GstBuffer *buf = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
1693 self->last_parsed_offset, consumed);
1694 gst_adapter_push (self->cache_out, buf);
1695 } else if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B) {
1696 gst_av1_parse_convert_to_annexb (self, buffer, &obu, frame_complete);
1697 } else {
1698 g_assert_not_reached ();
1699 }
1700 self->last_parsed_offset += consumed;
1701
1702 if (self->align == GST_AV1_PARSE_ALIGN_FRAME && frame_complete)
1703 complete = TRUE;
1704
1705 if (complete)
1706 break;
1707 }
1708
1709 /* Finish a complete frame anyway */
1710 if (complete || GST_BASE_PARSE_DRAINING (parse)) {
1711 *skipsize = 0;
1712
1713 /* push the left anyway if no error */
1714 if (res == GST_AV1_PARSER_OK)
1715 ret = gst_av1_parse_push_data (self, frame,
1716 self->last_parsed_offset, TRUE);
1717
1718 self->last_parsed_offset = 0;
1719
1720 goto out;
1721 }
1722
1723 if (res == GST_AV1_PARSER_BITSTREAM_ERROR ||
1724 res == GST_AV1_PARSER_MISSING_OBU_REFERENCE) {
1725 *skipsize = map_info.size;
1726 GST_WARNING_OBJECT (parse, "Parse obu error, discard whole buffer %d.",
1727 *skipsize);
1728 /* The adapter will be cleared in next loop because of
1729 GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME flag */
1730 gst_av1_parse_reset_obu_data_state (self);
1731 ret = GST_FLOW_OK;
1732 } else if (res == GST_AV1_PARSER_NO_MORE_DATA) {
1733 *skipsize = 0;
1734
1735 if (self->in_align >= GST_AV1_PARSE_ALIGN_OBU) {
1736 /* The buffer is already aligned to OBU, should not happen.
1737 The adapter will be cleared in next loop because of
1738 GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME flag */
1739 *skipsize = map_info.size;
1740 gst_av1_parse_reset_obu_data_state (self);
1741 GST_WARNING_OBJECT (parse,
1742 "Parse obu need more data, discard whole buffer %d.", *skipsize);
1743 }
1744 ret = GST_FLOW_OK;
1745 } else if (res == GST_AV1_PARSER_DROP) {
1746 GST_DEBUG_OBJECT (parse, "Drop %d data", consumed);
1747 self->last_parsed_offset += consumed;
1748 gst_av1_parse_reset_obu_data_state (self);
1749 res = GST_AV1_PARSER_OK;
1750 goto again;
1751 } else if (res == GST_AV1_PARSER_OK) {
1752 /* Everything is correct but still not get a frame or tu,
1753 need more data */
1754 GST_DEBUG_OBJECT (parse, "Need more data");
1755 *skipsize = 0;
1756 ret = GST_FLOW_OK;
1757 } else {
1758 GST_ERROR_OBJECT (parse, "Parse obu get unexpect error %d", res);
1759 *skipsize = 0;
1760 ret = GST_FLOW_ERROR;
1761 }
1762
1763 out:
1764 gst_buffer_unmap (buffer, &map_info);
1765 gst_buffer_unref (buffer);
1766 return ret;
1767 }
1768
1769 /* Try to recognize whether the input is annex-b format. */
1770 static GstFlowReturn
gst_av1_parse_detect_alignment(GstBaseParse * parse,GstBaseParseFrame * frame,gint * skipsize,guint32 * total_consumed)1771 gst_av1_parse_detect_alignment (GstBaseParse * parse,
1772 GstBaseParseFrame * frame, gint * skipsize, guint32 * total_consumed)
1773 {
1774 GstAV1Parse *self = GST_AV1_PARSE (parse);
1775 GstMapInfo map_info;
1776 GstAV1OBU obu;
1777 GstAV1ParserResult res = GST_AV1_PARSER_INVALID_OPERATION;
1778 GstBuffer *buffer = gst_buffer_ref (frame->buffer);
1779 gboolean got_seq, got_frame;
1780 gboolean frame_complete;
1781 guint32 consumed;
1782 guint32 frame_sz;
1783 GstFlowReturn ret = GST_FLOW_OK;
1784
1785 if (!gst_buffer_map (buffer, &map_info, GST_MAP_READ)) {
1786 *skipsize = 0;
1787 GST_ERROR_OBJECT (parse, "Couldn't map incoming buffer");
1788 return GST_FLOW_ERROR;
1789 }
1790
1791 gst_av1_parser_reset (self->parser, FALSE);
1792
1793 /* Detect the alignment obu first */
1794 got_seq = FALSE;
1795 got_frame = FALSE;
1796 *total_consumed = 0;
1797 again:
1798 while (*total_consumed < map_info.size) {
1799 res = gst_av1_parser_identify_one_obu (self->parser,
1800 map_info.data + *total_consumed, map_info.size - *total_consumed,
1801 &obu, &consumed);
1802 if (res == GST_AV1_PARSER_OK) {
1803 *total_consumed += consumed;
1804 res = gst_av1_parse_handle_one_obu (self, &obu, &frame_complete, NULL);
1805 }
1806
1807 if (res != GST_AV1_PARSER_OK)
1808 break;
1809
1810 if (obu.obu_type == GST_AV1_OBU_SEQUENCE_HEADER)
1811 got_seq = TRUE;
1812 if (obu.obu_type == GST_AV1_OBU_REDUNDANT_FRAME_HEADER ||
1813 obu.obu_type == GST_AV1_OBU_FRAME ||
1814 obu.obu_type == GST_AV1_OBU_FRAME_HEADER)
1815 got_frame = TRUE;
1816
1817 if (got_seq || got_frame)
1818 break;
1819 }
1820
1821 gst_av1_parser_reset (self->parser, FALSE);
1822
1823 if (res == GST_AV1_PARSER_OK || res == GST_AV1_PARSER_NO_MORE_DATA) {
1824 *skipsize = 0;
1825
1826 /* If succeed recognize seq or frame, we can decide,
1827 otherwise, just skipsize to 0 and get more data. */
1828 if (got_seq || got_frame)
1829 self->in_align = GST_AV1_PARSE_ALIGN_BYTE;
1830
1831 ret = GST_FLOW_OK;
1832 goto out;
1833 } else if (res == GST_AV1_PARSER_DROP) {
1834 *total_consumed += consumed;
1835 res = GST_AV1_PARSER_OK;
1836 gst_av1_parse_reset_obu_data_state (self);
1837 goto again;
1838 }
1839
1840 /* Try the annexb. The buffer should hold the whole frame, and
1841 the buffer start with the frame size in leb128() format. */
1842 if (map_info.size < 8) {
1843 /* Get more data. */
1844 *skipsize = 0;
1845 ret = GST_FLOW_OK;
1846 goto out;
1847 }
1848
1849 frame_sz = _read_leb128 (map_info.data, &res, &consumed);
1850 if (frame_sz == 0 || res != GST_AV1_PARSER_OK) {
1851 /* Both modes does not match, we can decide a error */
1852 ret = GST_FLOW_ERROR;
1853 goto out;
1854 }
1855
1856 if (frame_sz + consumed != map_info.size) {
1857 GST_DEBUG_OBJECT (self, "Buffer size %" G_GSSIZE_FORMAT ", frame size %d,"
1858 " consumed %d, does not match annex b format.",
1859 map_info.size, frame_sz, consumed);
1860 /* Both modes does not match, we can decide a error */
1861 ret = GST_FLOW_ERROR;
1862 goto out;
1863 }
1864
1865 self->in_align = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B;
1866 gst_av1_parser_reset (self->parser, TRUE);
1867 ret = GST_FLOW_OK;
1868
1869 out:
1870 gst_av1_parse_reset_obu_data_state (self);
1871 gst_buffer_unmap (buffer, &map_info);
1872 gst_buffer_unref (buffer);
1873 return ret;
1874 }
1875
1876 static GstFlowReturn
gst_av1_parse_handle_frame(GstBaseParse * parse,GstBaseParseFrame * frame,gint * skipsize)1877 gst_av1_parse_handle_frame (GstBaseParse * parse,
1878 GstBaseParseFrame * frame, gint * skipsize)
1879 {
1880 GstAV1Parse *self = GST_AV1_PARSE (parse);
1881 GstFlowReturn ret = GST_FLOW_OK;
1882 guint in_level, out_level;
1883
1884 if (GST_BUFFER_FLAG_IS_SET (frame->buffer, GST_BUFFER_FLAG_DISCONT)) {
1885 self->discont = TRUE;
1886
1887 if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME)
1888 gst_av1_parse_reset_obu_data_state (self);
1889 } else {
1890 self->discont = FALSE;
1891 }
1892
1893 GST_LOG_OBJECT (self, "Input frame size %" G_GSSIZE_FORMAT,
1894 gst_buffer_get_size (frame->buffer));
1895
1896 /* avoid stale cached parsing state */
1897 if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME) {
1898 GST_LOG_OBJECT (self, "parsing new frame");
1899 gst_adapter_clear (self->cache_out);
1900 gst_adapter_clear (self->frame_cache);
1901 self->last_parsed_offset = 0;
1902 self->header = FALSE;
1903 self->keyframe = FALSE;
1904 self->show_frame = FALSE;
1905 } else {
1906 GST_LOG_OBJECT (self, "resuming frame parsing");
1907 }
1908
1909 /* When in pull mode, the sink pad has no caps, we may get the
1910 caps by query the upstream element */
1911 if (self->in_align == GST_AV1_PARSE_ALIGN_NONE) {
1912 GstCaps *upstream_caps;
1913
1914 upstream_caps =
1915 gst_pad_peer_query_caps (GST_BASE_PARSE_SINK_PAD (self), NULL);
1916 if (upstream_caps) {
1917 if (!gst_caps_is_empty (upstream_caps)
1918 && !gst_caps_is_any (upstream_caps)) {
1919 GST_LOG_OBJECT (self, "upstream caps: %" GST_PTR_FORMAT, upstream_caps);
1920 /* fixate to avoid ambiguity with lists when parsing */
1921 upstream_caps = gst_caps_fixate (upstream_caps);
1922 self->in_align = gst_av1_parse_alignment_from_caps (upstream_caps);
1923 }
1924
1925 gst_caps_unref (upstream_caps);
1926
1927 gst_av1_parser_reset (self->parser,
1928 self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B);
1929 }
1930
1931 if (self->in_align != GST_AV1_PARSE_ALIGN_NONE)
1932 GST_LOG_OBJECT (self, "Query the upstream get the alignment %d",
1933 self->in_align);
1934 }
1935
1936 if (self->in_align == GST_AV1_PARSE_ALIGN_NONE) {
1937 guint32 consumed = 0;
1938
1939 /* Only happend at the first time of handle_frame, and the
1940 alignment in the sink caps is unset. Try the default and
1941 if error, try the annex B. */
1942 ret = gst_av1_parse_detect_alignment (parse, frame, skipsize, &consumed);
1943 if (ret == GST_FLOW_OK && self->in_align != GST_AV1_PARSE_ALIGN_NONE) {
1944 GST_INFO_OBJECT (self, "Detect the input alignment %d", self->in_align);
1945 } else {
1946 *skipsize = consumed > 0 ? consumed : gst_buffer_get_size (frame->buffer);
1947 GST_WARNING_OBJECT (self, "Fail to detect the alignment, skip %d",
1948 *skipsize);
1949 return GST_FLOW_OK;
1950 }
1951 }
1952
1953 /* We may in pull mode and no caps is set */
1954 if (self->align == GST_AV1_PARSE_ALIGN_NONE)
1955 gst_av1_parse_negotiate (self, NULL);
1956
1957 in_level = self->in_align;
1958 if (self->in_align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
1959 in_level = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
1960 out_level = self->align;
1961 if (self->align == GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT_ANNEX_B)
1962 out_level = GST_AV1_PARSE_ALIGN_TEMPORAL_UNIT;
1963
1964 if (self->in_align <= GST_AV1_PARSE_ALIGN_OBU
1965 && self->align == GST_AV1_PARSE_ALIGN_OBU) {
1966 ret = gst_av1_parse_handle_obu_to_obu (parse, frame, skipsize);
1967 } else if (in_level < out_level) {
1968 ret = gst_av1_parse_handle_to_big_align (parse, frame, skipsize);
1969 } else {
1970 ret = gst_av1_parse_handle_to_small_and_equal_align (parse,
1971 frame, skipsize);
1972 }
1973
1974 return ret;
1975 }
1976