1 /* GStreamer H.264 Parser
2 * Copyright (C) <2010> Collabora ltd
3 * Copyright (C) <2010> Nokia Corporation
4 * Copyright (C) <2011> Intel Corporation
5 *
6 * Copyright (C) <2010> Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>
7 * Copyright (C) <2011> Thibault Saunier <thibault.saunier@collabora.com>
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Library General Public
11 * License as published by the Free Software Foundation; either
12 * version 2 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Library General Public License for more details.
18 *
19 * You should have received a copy of the GNU Library General Public
20 * License along with this library; if not, write to the
21 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
22 * Boston, MA 02110-1301, USA.
23 */
24
25 #ifdef HAVE_CONFIG_H
26 # include "config.h"
27 #endif
28
29 #include <gst/base/base.h>
30 #include <gst/pbutils/pbutils.h>
31 #include <gst/video/video.h>
32 #include "gstvideoparserselements.h"
33 #include "gsth264parse.h"
34
35 #include <string.h>
36
37 GST_DEBUG_CATEGORY (h264_parse_debug);
38 #define GST_CAT_DEFAULT h264_parse_debug
39
40 #define DEFAULT_CONFIG_INTERVAL (0)
41 #define DEFAULT_UPDATE_TIMECODE FALSE
42
43 enum
44 {
45 PROP_0,
46 PROP_CONFIG_INTERVAL,
47 PROP_UPDATE_TIMECODE,
48 };
49
50 enum
51 {
52 GST_H264_PARSE_FORMAT_NONE,
53 GST_H264_PARSE_FORMAT_AVC,
54 GST_H264_PARSE_FORMAT_BYTE,
55 GST_H264_PARSE_FORMAT_AVC3
56 };
57
58 enum
59 {
60 GST_H264_PARSE_ALIGN_NONE = 0,
61 GST_H264_PARSE_ALIGN_NAL,
62 GST_H264_PARSE_ALIGN_AU
63 };
64
65 enum
66 {
67 GST_H264_PARSE_STATE_GOT_SPS = 1 << 0,
68 GST_H264_PARSE_STATE_GOT_PPS = 1 << 1,
69 GST_H264_PARSE_STATE_GOT_SLICE = 1 << 2,
70
71 GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS = (GST_H264_PARSE_STATE_GOT_SPS |
72 GST_H264_PARSE_STATE_GOT_PPS),
73 GST_H264_PARSE_STATE_VALID_PICTURE =
74 (GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS |
75 GST_H264_PARSE_STATE_GOT_SLICE)
76 };
77
78 enum
79 {
80 GST_H264_PARSE_SEI_EXPIRED = 0,
81 GST_H264_PARSE_SEI_ACTIVE = 1,
82 GST_H264_PARSE_SEI_PARSED = 2,
83 };
84
85 #define GST_H264_PARSE_STATE_VALID(parse, expected_state) \
86 (((parse)->state & (expected_state)) == (expected_state))
87
88 static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
89 GST_PAD_SINK,
90 GST_PAD_ALWAYS,
91 GST_STATIC_CAPS ("video/x-h264"));
92
93 static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
94 GST_PAD_SRC,
95 GST_PAD_ALWAYS,
96 GST_STATIC_CAPS ("video/x-h264, parsed = (boolean) true, "
97 "stream-format=(string) { avc, avc3, byte-stream }, "
98 "alignment=(string) { au, nal }"));
99
100 #define parent_class gst_h264_parse_parent_class
101 G_DEFINE_TYPE (GstH264Parse, gst_h264_parse, GST_TYPE_BASE_PARSE);
102 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (h264parse, "h264parse",
103 GST_RANK_PRIMARY + 1, GST_TYPE_H264_PARSE,
104 videoparsers_element_init (plugin));
105
106 static void gst_h264_parse_finalize (GObject * object);
107
108 static gboolean gst_h264_parse_start (GstBaseParse * parse);
109 static gboolean gst_h264_parse_stop (GstBaseParse * parse);
110 static GstFlowReturn gst_h264_parse_handle_frame (GstBaseParse * parse,
111 GstBaseParseFrame * frame, gint * skipsize);
112 static GstFlowReturn gst_h264_parse_parse_frame (GstBaseParse * parse,
113 GstBaseParseFrame * frame);
114 static GstFlowReturn gst_h264_parse_pre_push_frame (GstBaseParse * parse,
115 GstBaseParseFrame * frame);
116
117 static void gst_h264_parse_set_property (GObject * object, guint prop_id,
118 const GValue * value, GParamSpec * pspec);
119 static void gst_h264_parse_get_property (GObject * object, guint prop_id,
120 GValue * value, GParamSpec * pspec);
121
122 static gboolean gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps);
123 static GstCaps *gst_h264_parse_get_caps (GstBaseParse * parse,
124 GstCaps * filter);
125 static gboolean gst_h264_parse_event (GstBaseParse * parse, GstEvent * event);
126 static gboolean gst_h264_parse_src_event (GstBaseParse * parse,
127 GstEvent * event);
128 static void gst_h264_parse_update_src_caps (GstH264Parse * h264parse,
129 GstCaps * caps);
130
131 static void
gst_h264_parse_class_init(GstH264ParseClass * klass)132 gst_h264_parse_class_init (GstH264ParseClass * klass)
133 {
134 GObjectClass *gobject_class = (GObjectClass *) klass;
135 GstBaseParseClass *parse_class = GST_BASE_PARSE_CLASS (klass);
136 GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);
137
138 GST_DEBUG_CATEGORY_INIT (h264_parse_debug, "h264parse", 0, "h264 parser");
139
140 gobject_class->finalize = gst_h264_parse_finalize;
141 gobject_class->set_property = gst_h264_parse_set_property;
142 gobject_class->get_property = gst_h264_parse_get_property;
143
144 g_object_class_install_property (gobject_class, PROP_CONFIG_INTERVAL,
145 g_param_spec_int ("config-interval",
146 "SPS PPS Send Interval",
147 "Send SPS and PPS Insertion Interval in seconds (sprop parameter sets "
148 "will be multiplexed in the data stream when detected.) "
149 "(0 = disabled, -1 = send with every IDR frame)",
150 -1, 3600, DEFAULT_CONFIG_INTERVAL,
151 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
152
153 /**
154 * GstH264Parse:update-timecode:
155 *
156 * If the stream contains Picture Timing SEI, update their timecode values
157 * using upstream GstVideoTimeCodeMeta. However, if there are no Picture
158 * Timing SEI in bitstream, this property will not insert the SEI into the
159 * bitstream - it only modifies existing ones.
160 * Moreover, even if both GstVideoTimeCodeMeta and Picture Timing SEI
161 * are present, if pic_struct_present_flag of VUI is equal to zero,
162 * timecode values will not updated as there is not enough information
163 * in the stream to do so.
164 *
165 * Since: 1.18
166 */
167 g_object_class_install_property (gobject_class, PROP_UPDATE_TIMECODE,
168 g_param_spec_boolean ("update-timecode",
169 "Update Timecode",
170 "Update time code values in Picture Timing SEI if GstVideoTimeCodeMeta "
171 "is attached to incoming buffer and also Picture Timing SEI exists "
172 "in the bitstream. To make this property work, SPS must contain "
173 "VUI and pic_struct_present_flag of VUI must be non-zero",
174 DEFAULT_UPDATE_TIMECODE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
175
176 /* Override BaseParse vfuncs */
177 parse_class->start = GST_DEBUG_FUNCPTR (gst_h264_parse_start);
178 parse_class->stop = GST_DEBUG_FUNCPTR (gst_h264_parse_stop);
179 parse_class->handle_frame = GST_DEBUG_FUNCPTR (gst_h264_parse_handle_frame);
180 parse_class->pre_push_frame =
181 GST_DEBUG_FUNCPTR (gst_h264_parse_pre_push_frame);
182 parse_class->set_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_set_caps);
183 parse_class->get_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_get_caps);
184 parse_class->sink_event = GST_DEBUG_FUNCPTR (gst_h264_parse_event);
185 parse_class->src_event = GST_DEBUG_FUNCPTR (gst_h264_parse_src_event);
186
187 gst_element_class_add_static_pad_template (gstelement_class, &srctemplate);
188 gst_element_class_add_static_pad_template (gstelement_class, &sinktemplate);
189
190 gst_element_class_set_static_metadata (gstelement_class, "H.264 parser",
191 "Codec/Parser/Converter/Video",
192 "Parses H.264 streams",
193 "Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>");
194 }
195
196 static void
gst_h264_parse_init(GstH264Parse * h264parse)197 gst_h264_parse_init (GstH264Parse * h264parse)
198 {
199 h264parse->frame_out = gst_adapter_new ();
200 gst_base_parse_set_pts_interpolation (GST_BASE_PARSE (h264parse), FALSE);
201 gst_base_parse_set_infer_ts (GST_BASE_PARSE (h264parse), FALSE);
202 GST_PAD_SET_ACCEPT_INTERSECT (GST_BASE_PARSE_SINK_PAD (h264parse));
203 GST_PAD_SET_ACCEPT_TEMPLATE (GST_BASE_PARSE_SINK_PAD (h264parse));
204 #ifdef OHOS_OPT_COMPAT
205 // ohos.ext.func.0031
206 h264parse->aud_needed = FALSE;
207 h264parse->aud_insert = FALSE;
208 #else
209 h264parse->aud_needed = TRUE;
210 h264parse->aud_insert = TRUE;
211 #endif
212 h264parse->update_timecode = DEFAULT_UPDATE_TIMECODE;
213 }
214
215 static void
gst_h264_parse_finalize(GObject * object)216 gst_h264_parse_finalize (GObject * object)
217 {
218 GstH264Parse *h264parse = GST_H264_PARSE (object);
219
220 g_object_unref (h264parse->frame_out);
221
222 G_OBJECT_CLASS (parent_class)->finalize (object);
223 }
224
225 static void
gst_h264_parse_reset_frame(GstH264Parse * h264parse)226 gst_h264_parse_reset_frame (GstH264Parse * h264parse)
227 {
228 GST_DEBUG_OBJECT (h264parse, "reset frame");
229
230 /* done parsing; reset state */
231 h264parse->current_off = -1;
232
233 h264parse->update_caps = FALSE;
234 h264parse->idr_pos = -1;
235 h264parse->sei_pos = -1;
236 h264parse->pic_timing_sei_pos = -1;
237 h264parse->pic_timing_sei_size = -1;
238 h264parse->keyframe = FALSE;
239 h264parse->predicted = FALSE;
240 h264parse->bidirectional = FALSE;
241 h264parse->header = FALSE;
242 h264parse->frame_start = FALSE;
243 #ifdef OHOS_OPT_COMPAT
244 // ohos.ext.func.0031
245 h264parse->aud_insert = FALSE;
246 #else
247 h264parse->aud_insert = TRUE;
248 #endif
249 h264parse->have_sps_in_frame = FALSE;
250 h264parse->have_pps_in_frame = FALSE;
251 h264parse->have_aud_in_frame = FALSE;
252 gst_adapter_clear (h264parse->frame_out);
253 }
254
255 static void
gst_h264_parse_reset_stream_info(GstH264Parse * h264parse)256 gst_h264_parse_reset_stream_info (GstH264Parse * h264parse)
257 {
258 gint i;
259
260 h264parse->width = 0;
261 h264parse->height = 0;
262 h264parse->fps_num = 0;
263 h264parse->fps_den = 0;
264 h264parse->upstream_par_n = -1;
265 h264parse->upstream_par_d = -1;
266 h264parse->parsed_par_n = 0;
267 h264parse->parsed_par_d = 0;
268 h264parse->parsed_colorimetry.range = GST_VIDEO_COLOR_RANGE_UNKNOWN;
269 h264parse->parsed_colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_UNKNOWN;
270 h264parse->parsed_colorimetry.transfer = GST_VIDEO_TRANSFER_UNKNOWN;
271 h264parse->parsed_colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_UNKNOWN;
272
273 h264parse->have_pps = FALSE;
274 h264parse->have_sps = FALSE;
275
276 h264parse->multiview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
277 h264parse->multiview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
278 h264parse->first_in_bundle = TRUE;
279
280 h264parse->align = GST_H264_PARSE_ALIGN_NONE;
281 h264parse->format = GST_H264_PARSE_FORMAT_NONE;
282
283 h264parse->transform = FALSE;
284 h264parse->nal_length_size = 4;
285 h264parse->packetized = FALSE;
286 h264parse->push_codec = FALSE;
287 h264parse->first_frame = TRUE;
288
289 gst_buffer_replace (&h264parse->codec_data, NULL);
290 gst_buffer_replace (&h264parse->codec_data_in, NULL);
291
292 gst_h264_parse_reset_frame (h264parse);
293
294 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++)
295 gst_buffer_replace (&h264parse->sps_nals[i], NULL);
296 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++)
297 gst_buffer_replace (&h264parse->pps_nals[i], NULL);
298
299 gst_video_mastering_display_info_init (&h264parse->mastering_display_info);
300 h264parse->mastering_display_info_state = GST_H264_PARSE_SEI_EXPIRED;
301
302 gst_video_content_light_level_init (&h264parse->content_light_level);
303 h264parse->content_light_level_state = GST_H264_PARSE_SEI_EXPIRED;
304 }
305
306 static void
gst_h264_parse_reset(GstH264Parse * h264parse)307 gst_h264_parse_reset (GstH264Parse * h264parse)
308 {
309 h264parse->last_report = GST_CLOCK_TIME_NONE;
310
311 h264parse->dts = GST_CLOCK_TIME_NONE;
312 h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
313 h264parse->do_ts = TRUE;
314
315 h264parse->pending_key_unit_ts = GST_CLOCK_TIME_NONE;
316 gst_event_replace (&h264parse->force_key_unit_event, NULL);
317
318 h264parse->discont = FALSE;
319 h264parse->discard_bidirectional = FALSE;
320 h264parse->marker = FALSE;
321
322 gst_h264_parse_reset_stream_info (h264parse);
323 }
324
325 static gboolean
gst_h264_parse_start(GstBaseParse * parse)326 gst_h264_parse_start (GstBaseParse * parse)
327 {
328 GstH264Parse *h264parse = GST_H264_PARSE (parse);
329
330 GST_DEBUG_OBJECT (parse, "start");
331 gst_h264_parse_reset (h264parse);
332
333 h264parse->nalparser = gst_h264_nal_parser_new ();
334
335 h264parse->state = 0;
336 h264parse->dts = GST_CLOCK_TIME_NONE;
337 h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
338 h264parse->sei_pic_struct_pres_flag = FALSE;
339 h264parse->sei_pic_struct = 0;
340 h264parse->field_pic_flag = 0;
341 h264parse->aud_needed = TRUE;
342 h264parse->aud_insert = FALSE;
343
344 gst_base_parse_set_min_frame_size (parse, 4);
345
346 return TRUE;
347 }
348
349 static gboolean
gst_h264_parse_stop(GstBaseParse * parse)350 gst_h264_parse_stop (GstBaseParse * parse)
351 {
352 GstH264Parse *h264parse = GST_H264_PARSE (parse);
353
354 GST_DEBUG_OBJECT (parse, "stop");
355 gst_h264_parse_reset (h264parse);
356
357 gst_h264_nal_parser_free (h264parse->nalparser);
358
359 return TRUE;
360 }
361
362 static const gchar *
gst_h264_parse_get_string(GstH264Parse * parse,gboolean format,gint code)363 gst_h264_parse_get_string (GstH264Parse * parse, gboolean format, gint code)
364 {
365 if (format) {
366 switch (code) {
367 case GST_H264_PARSE_FORMAT_AVC:
368 return "avc";
369 case GST_H264_PARSE_FORMAT_BYTE:
370 return "byte-stream";
371 case GST_H264_PARSE_FORMAT_AVC3:
372 return "avc3";
373 default:
374 return "none";
375 }
376 } else {
377 switch (code) {
378 case GST_H264_PARSE_ALIGN_NAL:
379 return "nal";
380 case GST_H264_PARSE_ALIGN_AU:
381 return "au";
382 default:
383 return "none";
384 }
385 }
386 }
387
388 static void
gst_h264_parse_format_from_caps(GstCaps * caps,guint * format,guint * align)389 gst_h264_parse_format_from_caps (GstCaps * caps, guint * format, guint * align)
390 {
391 if (format)
392 *format = GST_H264_PARSE_FORMAT_NONE;
393
394 if (align)
395 *align = GST_H264_PARSE_ALIGN_NONE;
396
397 g_return_if_fail (gst_caps_is_fixed (caps));
398
399 GST_DEBUG ("parsing caps: %" GST_PTR_FORMAT, caps);
400
401 if (caps && gst_caps_get_size (caps) > 0) {
402 GstStructure *s = gst_caps_get_structure (caps, 0);
403 const gchar *str = NULL;
404
405 if (format) {
406 if ((str = gst_structure_get_string (s, "stream-format"))) {
407 if (strcmp (str, "avc") == 0)
408 *format = GST_H264_PARSE_FORMAT_AVC;
409 else if (strcmp (str, "byte-stream") == 0)
410 *format = GST_H264_PARSE_FORMAT_BYTE;
411 else if (strcmp (str, "avc3") == 0)
412 *format = GST_H264_PARSE_FORMAT_AVC3;
413 }
414 }
415
416 if (align) {
417 if ((str = gst_structure_get_string (s, "alignment"))) {
418 if (strcmp (str, "au") == 0)
419 *align = GST_H264_PARSE_ALIGN_AU;
420 else if (strcmp (str, "nal") == 0)
421 *align = GST_H264_PARSE_ALIGN_NAL;
422 }
423 }
424 }
425 }
426
427 /* check downstream caps to configure format and alignment */
428 static void
gst_h264_parse_negotiate(GstH264Parse * h264parse,gint in_format,GstCaps * in_caps)429 gst_h264_parse_negotiate (GstH264Parse * h264parse, gint in_format,
430 GstCaps * in_caps)
431 {
432 GstCaps *caps;
433 guint format = h264parse->format;
434 guint align = h264parse->align;
435
436 g_return_if_fail ((in_caps == NULL) || gst_caps_is_fixed (in_caps));
437
438 caps = gst_pad_get_allowed_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
439 GST_DEBUG_OBJECT (h264parse, "allowed caps: %" GST_PTR_FORMAT, caps);
440
441 /* concentrate on leading structure, since decodebin parser
442 * capsfilter always includes parser template caps */
443 if (caps) {
444 caps = gst_caps_truncate (caps);
445 GST_DEBUG_OBJECT (h264parse, "negotiating with caps: %" GST_PTR_FORMAT,
446 caps);
447 }
448
449 h264parse->can_passthrough = FALSE;
450
451 if (in_caps && caps) {
452 if (gst_caps_can_intersect (in_caps, caps)) {
453 GST_DEBUG_OBJECT (h264parse, "downstream accepts upstream caps");
454 gst_h264_parse_format_from_caps (in_caps, &format, &align);
455 gst_caps_unref (caps);
456 caps = NULL;
457 h264parse->can_passthrough = TRUE;
458 }
459 }
460
461 /* FIXME We could fail the negotiation immediately if caps are empty */
462 if (caps && !gst_caps_is_empty (caps)) {
463 /* fixate to avoid ambiguity with lists when parsing */
464 caps = gst_caps_fixate (caps);
465 gst_h264_parse_format_from_caps (caps, &format, &align);
466 }
467
468 /* default */
469 if (!format)
470 format = GST_H264_PARSE_FORMAT_BYTE;
471 if (!align)
472 align = GST_H264_PARSE_ALIGN_AU;
473
474 GST_DEBUG_OBJECT (h264parse, "selected format %s, alignment %s",
475 gst_h264_parse_get_string (h264parse, TRUE, format),
476 gst_h264_parse_get_string (h264parse, FALSE, align));
477
478 h264parse->format = format;
479 h264parse->align = align;
480
481 h264parse->transform = in_format != h264parse->format ||
482 align == GST_H264_PARSE_ALIGN_AU;
483
484 if (caps)
485 gst_caps_unref (caps);
486 }
487
488 static GstBuffer *
gst_h264_parse_wrap_nal(GstH264Parse * h264parse,guint format,guint8 * data,guint size)489 gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data,
490 guint size)
491 {
492 GstBuffer *buf;
493 guint nl = h264parse->nal_length_size;
494 guint32 tmp;
495
496 GST_DEBUG_OBJECT (h264parse, "nal length %d", size);
497
498 buf = gst_buffer_new_allocate (NULL, 4 + size, NULL);
499 if (format == GST_H264_PARSE_FORMAT_AVC
500 || format == GST_H264_PARSE_FORMAT_AVC3) {
501 tmp = GUINT32_TO_BE (size << (32 - 8 * nl));
502 } else {
503 /* HACK: nl should always be 4 here, otherwise this won't work.
504 * There are legit cases where nl in avc stream is 2, but byte-stream
505 * SC is still always 4 bytes. */
506 nl = 4;
507 tmp = GUINT32_TO_BE (1);
508 }
509
510 gst_buffer_fill (buf, 0, &tmp, sizeof (guint32));
511 gst_buffer_fill (buf, nl, data, size);
512 gst_buffer_set_size (buf, size + nl);
513
514 return buf;
515 }
516
517 static void
gst_h264_parser_store_nal(GstH264Parse * h264parse,guint id,GstH264NalUnitType naltype,GstH264NalUnit * nalu)518 gst_h264_parser_store_nal (GstH264Parse * h264parse, guint id,
519 GstH264NalUnitType naltype, GstH264NalUnit * nalu)
520 {
521 GstBuffer *buf, **store;
522 guint size = nalu->size, store_size;
523
524 if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_SUBSET_SPS) {
525 store_size = GST_H264_MAX_SPS_COUNT;
526 store = h264parse->sps_nals;
527 GST_DEBUG_OBJECT (h264parse, "storing sps %u", id);
528 } else if (naltype == GST_H264_NAL_PPS) {
529 store_size = GST_H264_MAX_PPS_COUNT;
530 store = h264parse->pps_nals;
531 GST_DEBUG_OBJECT (h264parse, "storing pps %u", id);
532 } else
533 return;
534
535 if (id >= store_size) {
536 GST_DEBUG_OBJECT (h264parse, "unable to store nal, id out-of-range %d", id);
537 return;
538 }
539
540 buf = gst_buffer_new_allocate (NULL, size, NULL);
541 gst_buffer_fill (buf, 0, nalu->data + nalu->offset, size);
542
543 /* Indicate that buffer contain a header needed for decoding */
544 if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_PPS)
545 GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_HEADER);
546
547 if (store[id])
548 gst_buffer_unref (store[id]);
549
550 store[id] = buf;
551 }
552
553 #ifndef GST_DISABLE_GST_DEBUG
554 static const gchar *nal_names[] = {
555 "Unknown",
556 "Slice",
557 "Slice DPA",
558 "Slice DPB",
559 "Slice DPC",
560 "Slice IDR",
561 "SEI",
562 "SPS",
563 "PPS",
564 "AU delimiter",
565 "Sequence End",
566 "Stream End",
567 "Filler Data",
568 "SPS extension",
569 "Prefix",
570 "SPS Subset",
571 "Depth Parameter Set",
572 "Reserved", "Reserved",
573 "Slice Aux Unpartitioned",
574 "Slice Extension",
575 "Slice Depth/3D-AVC Extension"
576 };
577
578 static const gchar *
_nal_name(GstH264NalUnitType nal_type)579 _nal_name (GstH264NalUnitType nal_type)
580 {
581 if (nal_type <= GST_H264_NAL_SLICE_DEPTH)
582 return nal_names[nal_type];
583 return "Invalid";
584 }
585 #endif
586
587 static void
gst_h264_parse_process_sei_user_data(GstH264Parse * h264parse,GstH264RegisteredUserData * rud)588 gst_h264_parse_process_sei_user_data (GstH264Parse * h264parse,
589 GstH264RegisteredUserData * rud)
590 {
591 guint16 provider_code;
592 GstByteReader br;
593 GstVideoParseUtilsField field = GST_VIDEO_PARSE_UTILS_FIELD_1;
594
595 /* only US country code is currently supported */
596 switch (rud->country_code) {
597 case ITU_T_T35_COUNTRY_CODE_US:
598 break;
599 default:
600 GST_LOG_OBJECT (h264parse, "Unsupported country code %d",
601 rud->country_code);
602 return;
603 }
604
605 if (rud->data == NULL || rud->size < 2)
606 return;
607
608 gst_byte_reader_init (&br, rud->data, rud->size);
609
610 provider_code = gst_byte_reader_get_uint16_be_unchecked (&br);
611
612 if (h264parse->sei_pic_struct ==
613 (guint8) GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD)
614 field = GST_VIDEO_PARSE_UTILS_FIELD_2;
615 gst_video_parse_user_data ((GstElement *) h264parse, &h264parse->user_data,
616 &br, field, provider_code);
617
618 }
619
620 static void
gst_h264_parse_process_sei(GstH264Parse * h264parse,GstH264NalUnit * nalu)621 gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
622 {
623 GstH264SEIMessage sei;
624 GstH264NalParser *nalparser = h264parse->nalparser;
625 GstH264ParserResult pres;
626 GArray *messages;
627 guint i;
628
629 pres = gst_h264_parser_parse_sei (nalparser, nalu, &messages);
630 if (pres != GST_H264_PARSER_OK)
631 GST_WARNING_OBJECT (h264parse, "failed to parse one or more SEI message");
632
633 /* Even if pres != GST_H264_PARSER_OK, some message could have been parsed and
634 * stored in messages.
635 */
636 for (i = 0; i < messages->len; i++) {
637 sei = g_array_index (messages, GstH264SEIMessage, i);
638 switch (sei.payloadType) {
639 case GST_H264_SEI_PIC_TIMING:
640 {
641 guint j;
642 h264parse->sei_pic_struct_pres_flag =
643 sei.payload.pic_timing.pic_struct_present_flag;
644 h264parse->sei_cpb_removal_delay =
645 sei.payload.pic_timing.cpb_removal_delay;
646 if (h264parse->sei_pic_struct_pres_flag) {
647 h264parse->sei_pic_struct = sei.payload.pic_timing.pic_struct;
648 }
649
650 h264parse->num_clock_timestamp = 0;
651 memcpy (&h264parse->pic_timing_sei, &sei.payload.pic_timing,
652 sizeof (GstH264PicTiming));
653
654 for (j = 0; j < 3; j++) {
655 if (sei.payload.pic_timing.clock_timestamp_flag[j]) {
656 h264parse->num_clock_timestamp++;
657 }
658 }
659
660 if (h264parse->sei_pic_struct_pres_flag && h264parse->update_timecode) {
661 /* FIXME: add support multiple messages in a SEI nalu.
662 * Updating only this SEI message and preserving the others
663 * is a bit complicated */
664 if (messages->len == 1) {
665 h264parse->pic_timing_sei_pos = nalu->sc_offset;
666 h264parse->pic_timing_sei_size =
667 nalu->size + (nalu->offset - nalu->sc_offset);
668 }
669 }
670
671 GST_LOG_OBJECT (h264parse, "pic timing updated");
672 break;
673 }
674 case GST_H264_SEI_REGISTERED_USER_DATA:
675 gst_h264_parse_process_sei_user_data (h264parse,
676 &sei.payload.registered_user_data);
677 break;
678 case GST_H264_SEI_BUF_PERIOD:
679 if (h264parse->ts_trn_nb == GST_CLOCK_TIME_NONE ||
680 h264parse->dts == GST_CLOCK_TIME_NONE)
681 h264parse->ts_trn_nb = 0;
682 else
683 h264parse->ts_trn_nb = h264parse->dts;
684
685 GST_LOG_OBJECT (h264parse,
686 "new buffering period; ts_trn_nb updated: %" GST_TIME_FORMAT,
687 GST_TIME_ARGS (h264parse->ts_trn_nb));
688 break;
689
690 /* Additional messages that are not innerly useful to the
691 * element but for debugging purposes */
692 case GST_H264_SEI_RECOVERY_POINT:
693 GST_LOG_OBJECT (h264parse, "recovery point found: %u %u %u %u",
694 sei.payload.recovery_point.recovery_frame_cnt,
695 sei.payload.recovery_point.exact_match_flag,
696 sei.payload.recovery_point.broken_link_flag,
697 sei.payload.recovery_point.changing_slice_group_idc);
698 h264parse->keyframe = TRUE;
699 break;
700
701 /* Additional messages that are not innerly useful to the
702 * element but for debugging purposes */
703 case GST_H264_SEI_STEREO_VIDEO_INFO:{
704 GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
705 GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
706
707 GST_LOG_OBJECT (h264parse, "Stereo video information %u %u %u %u %u %u",
708 sei.payload.stereo_video_info.field_views_flag,
709 sei.payload.stereo_video_info.top_field_is_left_view_flag,
710 sei.payload.stereo_video_info.current_frame_is_left_view_flag,
711 sei.payload.stereo_video_info.next_frame_is_second_view_flag,
712 sei.payload.stereo_video_info.left_view_self_contained_flag,
713 sei.payload.stereo_video_info.right_view_self_contained_flag);
714
715 if (sei.payload.stereo_video_info.field_views_flag) {
716 mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
717 if (!sei.payload.stereo_video_info.top_field_is_left_view_flag)
718 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
719 } else {
720 mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
721 if (sei.payload.stereo_video_info.next_frame_is_second_view_flag) {
722 /* Mark current frame as first in bundle */
723 h264parse->first_in_bundle = TRUE;
724 if (!sei.payload.stereo_video_info.current_frame_is_left_view_flag)
725 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
726 }
727 }
728 if (mview_mode != h264parse->multiview_mode ||
729 mview_flags != h264parse->multiview_flags) {
730 h264parse->multiview_mode = mview_mode;
731 h264parse->multiview_flags = mview_flags;
732 /* output caps need to be changed */
733 gst_h264_parse_update_src_caps (h264parse, NULL);
734 }
735 break;
736 }
737 case GST_H264_SEI_FRAME_PACKING:{
738 GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
739 GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
740
741 GST_LOG_OBJECT (h264parse,
742 "frame packing arrangement message: id %u cancelled %u "
743 "type %u quincunx %u content_interpretation %d flip %u "
744 "right_first %u field_views %u is_frame0 %u "
745 "frame0_self_contained %u frame1_self_contained %u "
746 "frame0_grid (%u, %u) frame1_grid (%u, %u) "
747 "repetition_period %" G_GUINT16_FORMAT,
748 sei.payload.frame_packing.frame_packing_id,
749 sei.payload.frame_packing.frame_packing_cancel_flag,
750 sei.payload.frame_packing.frame_packing_type,
751 sei.payload.frame_packing.quincunx_sampling_flag,
752 sei.payload.frame_packing.content_interpretation_type,
753 sei.payload.frame_packing.spatial_flipping_flag,
754 sei.payload.frame_packing.frame0_flipped_flag,
755 sei.payload.frame_packing.field_views_flag,
756 sei.payload.frame_packing.current_frame_is_frame0_flag,
757 sei.payload.frame_packing.frame0_self_contained_flag,
758 sei.payload.frame_packing.frame1_self_contained_flag,
759 sei.payload.frame_packing.frame0_grid_position_x,
760 sei.payload.frame_packing.frame0_grid_position_y,
761 sei.payload.frame_packing.frame1_grid_position_x,
762 sei.payload.frame_packing.frame1_grid_position_y,
763 sei.payload.frame_packing.frame_packing_repetition_period);
764
765 /* Only IDs from 0->255 and 512->2^31-1 are valid. Ignore others */
766 if ((sei.payload.frame_packing.frame_packing_id >= 256 &&
767 sei.payload.frame_packing.frame_packing_id < 512) ||
768 (sei.payload.frame_packing.frame_packing_id >= (1U << 31)))
769 break; /* ignore */
770
771 if (!sei.payload.frame_packing.frame_packing_cancel_flag) {
772 /* Cancel flag sets things back to no-info */
773
774 if (sei.payload.frame_packing.content_interpretation_type == 2)
775 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
776
777 switch (sei.payload.frame_packing.frame_packing_type) {
778 case 0:
779 mview_mode = GST_VIDEO_MULTIVIEW_MODE_CHECKERBOARD;
780 break;
781 case 1:
782 mview_mode = GST_VIDEO_MULTIVIEW_MODE_COLUMN_INTERLEAVED;
783 break;
784 case 2:
785 mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
786 break;
787 case 3:
788 if (sei.payload.frame_packing.quincunx_sampling_flag)
789 mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX;
790 else
791 mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
792 if (sei.payload.frame_packing.spatial_flipping_flag) {
793 /* One of the views is flopped. */
794 if (sei.payload.frame_packing.frame0_flipped_flag !=
795 ! !(mview_flags &
796 GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
797 /* the left view is flopped */
798 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED;
799 else
800 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED;
801 }
802 break;
803 case 4:
804 mview_mode = GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM;
805 if (sei.payload.frame_packing.spatial_flipping_flag) {
806 /* One of the views is flipped, */
807 if (sei.payload.frame_packing.frame0_flipped_flag !=
808 ! !(mview_flags &
809 GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
810 /* the left view is flipped */
811 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED;
812 else
813 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED;
814 }
815 break;
816 case 5:
817 if (sei.payload.frame_packing.content_interpretation_type == 0)
818 mview_mode = GST_VIDEO_MULTIVIEW_MODE_MULTIVIEW_FRAME_BY_FRAME;
819 else
820 mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
821 break;
822 default:
823 GST_DEBUG_OBJECT (h264parse, "Invalid frame packing type %u",
824 sei.payload.frame_packing.frame_packing_type);
825 break;
826 }
827 }
828
829 if (mview_mode != h264parse->multiview_mode ||
830 mview_flags != h264parse->multiview_flags) {
831 h264parse->multiview_mode = mview_mode;
832 h264parse->multiview_flags = mview_flags;
833 /* output caps need to be changed */
834 gst_h264_parse_update_src_caps (h264parse, NULL);
835 }
836 break;
837 }
838 case GST_H264_SEI_MASTERING_DISPLAY_COLOUR_VOLUME:
839 {
840 /* Precision defined by spec.
841 * See D.2.29 Mastering display colour volume SEI message semantics */
842 GstVideoMasteringDisplayInfo minfo;
843 gint j, k;
844
845 /* GstVideoMasteringDisplayInfo::display_primaries is rgb order but
846 * AVC uses gbr order.
847 * See spec D.2.29 display_primaries_x and display_primaries_y
848 */
849 for (j = 0, k = 2; j < G_N_ELEMENTS (minfo.display_primaries); j++, k++) {
850 minfo.display_primaries[j].x =
851 sei.payload.
852 mastering_display_colour_volume.display_primaries_x[k % 3];
853 minfo.display_primaries[j].y =
854 sei.payload.
855 mastering_display_colour_volume.display_primaries_y[k % 3];
856 }
857
858 minfo.white_point.x =
859 sei.payload.mastering_display_colour_volume.white_point_x;
860 minfo.white_point.y =
861 sei.payload.mastering_display_colour_volume.white_point_y;
862 minfo.max_display_mastering_luminance =
863 sei.payload.mastering_display_colour_volume.
864 max_display_mastering_luminance;
865 minfo.min_display_mastering_luminance =
866 sei.payload.mastering_display_colour_volume.
867 min_display_mastering_luminance;
868
869 GST_LOG_OBJECT (h264parse, "mastering display info found: "
870 "Red(%u, %u) "
871 "Green(%u, %u) "
872 "Blue(%u, %u) "
873 "White(%u, %u) "
874 "max_luminance(%u) "
875 "min_luminance(%u) ",
876 minfo.display_primaries[0].x, minfo.display_primaries[0].y,
877 minfo.display_primaries[1].x, minfo.display_primaries[1].y,
878 minfo.display_primaries[2].x, minfo.display_primaries[2].y,
879 minfo.white_point.x, minfo.white_point.y,
880 minfo.max_display_mastering_luminance,
881 minfo.min_display_mastering_luminance);
882
883 if (h264parse->mastering_display_info_state ==
884 GST_H264_PARSE_SEI_EXPIRED) {
885 h264parse->update_caps = TRUE;
886 } else if (!gst_video_mastering_display_info_is_equal
887 (&h264parse->mastering_display_info, &minfo)) {
888 h264parse->update_caps = TRUE;
889 }
890
891 h264parse->mastering_display_info_state = GST_H264_PARSE_SEI_PARSED;
892 h264parse->mastering_display_info = minfo;
893
894 break;
895 }
896 case GST_H264_SEI_CONTENT_LIGHT_LEVEL:
897 {
898 GstVideoContentLightLevel cll;
899
900 cll.max_content_light_level =
901 sei.payload.content_light_level.max_content_light_level;
902 cll.max_frame_average_light_level =
903 sei.payload.content_light_level.max_pic_average_light_level;
904
905 GST_LOG_OBJECT (h264parse, "content light level found: "
906 "maxCLL:(%u), maxFALL:(%u)", cll.max_content_light_level,
907 cll.max_frame_average_light_level);
908
909 if (h264parse->content_light_level_state == GST_H264_PARSE_SEI_EXPIRED) {
910 h264parse->update_caps = TRUE;
911 } else if (cll.max_content_light_level !=
912 h264parse->content_light_level.max_content_light_level ||
913 cll.max_frame_average_light_level !=
914 h264parse->content_light_level.max_frame_average_light_level) {
915 h264parse->update_caps = TRUE;
916 }
917
918 h264parse->content_light_level_state = GST_H264_PARSE_SEI_PARSED;
919 h264parse->content_light_level = cll;
920
921 break;
922 }
923 default:{
924 gint payload_type = sei.payloadType;
925
926 if (payload_type == GST_H264_SEI_UNHANDLED_PAYLOAD) {
927 GstH264SEIUnhandledPayload *unhandled =
928 &sei.payload.unhandled_payload;
929 payload_type = unhandled->payloadType;
930 }
931
932 GST_LOG_OBJECT (h264parse, "Unsupported payload type %d", payload_type);
933 break;
934 }
935 }
936 }
937 g_array_free (messages, TRUE);
938 }
939
940 /* caller guarantees 2 bytes of nal payload */
941 static gboolean
gst_h264_parse_process_nal(GstH264Parse * h264parse,GstH264NalUnit * nalu)942 gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
943 {
944 guint nal_type;
945 GstH264PPS pps = { 0, };
946 GstH264SPS sps = { 0, };
947 GstH264NalParser *nalparser = h264parse->nalparser;
948 GstH264ParserResult pres;
949 GstH264SliceHdr slice;
950
951 /* nothing to do for broken input */
952 if (G_UNLIKELY (nalu->size < 2)) {
953 GST_DEBUG_OBJECT (h264parse, "not processing nal size %u", nalu->size);
954 return TRUE;
955 }
956
957 /* we have a peek as well */
958 nal_type = nalu->type;
959
960 GST_DEBUG_OBJECT (h264parse, "processing nal of type %u %s, size %u",
961 nal_type, _nal_name (nal_type), nalu->size);
962
963 switch (nal_type) {
964 case GST_H264_NAL_SUBSET_SPS:
965 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
966 return FALSE;
967 pres = gst_h264_parser_parse_subset_sps (nalparser, nalu, &sps);
968 goto process_sps;
969
970 case GST_H264_NAL_SPS:
971 /* reset state, everything else is obsolete */
972 h264parse->state &= GST_H264_PARSE_STATE_GOT_PPS;
973 pres = gst_h264_parser_parse_sps (nalparser, nalu, &sps);
974
975 process_sps:
976 /* arranged for a fallback sps.id, so use that one and only warn */
977 if (pres != GST_H264_PARSER_OK) {
978 GST_WARNING_OBJECT (h264parse, "failed to parse SPS:");
979 h264parse->state |= GST_H264_PARSE_STATE_GOT_SPS;
980 h264parse->header = TRUE;
981 return FALSE;
982 }
983
984 GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
985 h264parse->update_caps = TRUE;
986 h264parse->have_sps = TRUE;
987 h264parse->have_sps_in_frame = TRUE;
988 if (h264parse->push_codec && h264parse->have_pps) {
989 /* SPS and PPS found in stream before the first pre_push_frame, no need
990 * to forcibly push at start */
991 GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
992 h264parse->push_codec = FALSE;
993 h264parse->have_sps = FALSE;
994 h264parse->have_pps = FALSE;
995 }
996
997 gst_h264_parser_store_nal (h264parse, sps.id, nal_type, nalu);
998 gst_h264_sps_clear (&sps);
999 h264parse->state |= GST_H264_PARSE_STATE_GOT_SPS;
1000 h264parse->header = TRUE;
1001 break;
1002 case GST_H264_NAL_PPS:
1003 /* expected state: got-sps */
1004 h264parse->state &= GST_H264_PARSE_STATE_GOT_SPS;
1005 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
1006 return FALSE;
1007
1008 pres = gst_h264_parser_parse_pps (nalparser, nalu, &pps);
1009 /* arranged for a fallback pps.id, so use that one and only warn */
1010 if (pres != GST_H264_PARSER_OK) {
1011 GST_WARNING_OBJECT (h264parse, "failed to parse PPS:");
1012 if (pres != GST_H264_PARSER_BROKEN_LINK)
1013 return FALSE;
1014 }
1015
1016 /* parameters might have changed, force caps check */
1017 if (!h264parse->have_pps) {
1018 GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
1019 h264parse->update_caps = TRUE;
1020 }
1021 h264parse->have_pps = TRUE;
1022 h264parse->have_pps_in_frame = TRUE;
1023 if (h264parse->push_codec && h264parse->have_sps) {
1024 /* SPS and PPS found in stream before the first pre_push_frame, no need
1025 * to forcibly push at start */
1026 GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
1027 h264parse->push_codec = FALSE;
1028 h264parse->have_sps = FALSE;
1029 h264parse->have_pps = FALSE;
1030 }
1031
1032 gst_h264_parser_store_nal (h264parse, pps.id, nal_type, nalu);
1033 gst_h264_pps_clear (&pps);
1034 h264parse->state |= GST_H264_PARSE_STATE_GOT_PPS;
1035 h264parse->header = TRUE;
1036 break;
1037 case GST_H264_NAL_SEI:
1038 /* expected state: got-sps */
1039 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
1040 return FALSE;
1041
1042 h264parse->header = TRUE;
1043 gst_h264_parse_process_sei (h264parse, nalu);
1044 /* mark SEI pos */
1045 if (h264parse->sei_pos == -1) {
1046 if (h264parse->transform)
1047 h264parse->sei_pos = gst_adapter_available (h264parse->frame_out);
1048 else
1049 h264parse->sei_pos = nalu->sc_offset;
1050 GST_DEBUG_OBJECT (h264parse, "marking SEI in frame at offset %d",
1051 h264parse->sei_pos);
1052 }
1053 break;
1054
1055 case GST_H264_NAL_SLICE:
1056 case GST_H264_NAL_SLICE_DPA:
1057 case GST_H264_NAL_SLICE_DPB:
1058 case GST_H264_NAL_SLICE_DPC:
1059 case GST_H264_NAL_SLICE_IDR:
1060 case GST_H264_NAL_SLICE_EXT:
1061 /* expected state: got-sps|got-pps (valid picture headers) */
1062 h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
1063 if (!GST_H264_PARSE_STATE_VALID (h264parse,
1064 GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS))
1065 return FALSE;
1066
1067 /* This is similar to the GOT_SLICE state, but is only reset when the
1068 * AU is complete. This is used to keep track of AU */
1069 h264parse->picture_start = TRUE;
1070
1071 /* don't need to parse the whole slice (header) here */
1072 if (*(nalu->data + nalu->offset + nalu->header_bytes) & 0x80) {
1073 /* means first_mb_in_slice == 0 */
1074 /* real frame data */
1075 GST_DEBUG_OBJECT (h264parse, "first_mb_in_slice = 0");
1076 h264parse->frame_start = TRUE;
1077 }
1078 GST_DEBUG_OBJECT (h264parse, "frame start: %i", h264parse->frame_start);
1079 if (nal_type == GST_H264_NAL_SLICE_EXT && !GST_H264_IS_MVC_NALU (nalu))
1080 break;
1081
1082 pres = gst_h264_parser_parse_slice_hdr (nalparser, nalu, &slice,
1083 FALSE, FALSE);
1084 GST_DEBUG_OBJECT (h264parse,
1085 "parse result %d, first MB: %u, slice type: %u",
1086 pres, slice.first_mb_in_slice, slice.type);
1087 if (pres == GST_H264_PARSER_OK) {
1088 if (GST_H264_IS_I_SLICE (&slice) || GST_H264_IS_SI_SLICE (&slice))
1089 h264parse->keyframe = TRUE;
1090 else if (GST_H264_IS_P_SLICE (&slice)
1091 || GST_H264_IS_SP_SLICE (&slice))
1092 h264parse->predicted = TRUE;
1093 else if (GST_H264_IS_B_SLICE (&slice))
1094 h264parse->bidirectional = TRUE;
1095
1096 h264parse->state |= GST_H264_PARSE_STATE_GOT_SLICE;
1097 h264parse->field_pic_flag = slice.field_pic_flag;
1098 }
1099
1100 if (G_LIKELY (nal_type != GST_H264_NAL_SLICE_IDR &&
1101 !h264parse->push_codec))
1102 break;
1103
1104 /* if we need to sneak codec NALs into the stream,
1105 * this is a good place, so fake it as IDR
1106 * (which should be at start anyway) */
1107 /* mark where config needs to go if interval expired */
1108 /* mind replacement buffer if applicable */
1109 if (h264parse->idr_pos == -1) {
1110 if (h264parse->transform)
1111 h264parse->idr_pos = gst_adapter_available (h264parse->frame_out);
1112 else
1113 h264parse->idr_pos = nalu->sc_offset;
1114 GST_DEBUG_OBJECT (h264parse, "marking IDR in frame at offset %d",
1115 h264parse->idr_pos);
1116 }
1117 /* if SEI precedes (faked) IDR, then we have to insert config there */
1118 if (h264parse->sei_pos >= 0 && h264parse->idr_pos > h264parse->sei_pos) {
1119 h264parse->idr_pos = h264parse->sei_pos;
1120 GST_DEBUG_OBJECT (h264parse, "moved IDR mark to SEI position %d",
1121 h264parse->idr_pos);
1122 }
1123 /* Reset state only on first IDR slice of CVS D.2.29 */
1124 if (slice.first_mb_in_slice == 0) {
1125 if (h264parse->mastering_display_info_state ==
1126 GST_H264_PARSE_SEI_PARSED)
1127 h264parse->mastering_display_info_state = GST_H264_PARSE_SEI_ACTIVE;
1128 else if (h264parse->mastering_display_info_state ==
1129 GST_H264_PARSE_SEI_ACTIVE)
1130 h264parse->mastering_display_info_state = GST_H264_PARSE_SEI_EXPIRED;
1131
1132 if (h264parse->content_light_level_state == GST_H264_PARSE_SEI_PARSED)
1133 h264parse->content_light_level_state = GST_H264_PARSE_SEI_ACTIVE;
1134 else if (h264parse->content_light_level_state ==
1135 GST_H264_PARSE_SEI_ACTIVE)
1136 h264parse->content_light_level_state = GST_H264_PARSE_SEI_EXPIRED;
1137 }
1138 break;
1139 case GST_H264_NAL_AU_DELIMITER:
1140 /* Just accumulate AU Delimiter, whether it's before SPS or not */
1141 pres = gst_h264_parser_parse_nal (nalparser, nalu);
1142 if (pres != GST_H264_PARSER_OK)
1143 return FALSE;
1144 h264parse->aud_needed = FALSE;
1145 h264parse->have_aud_in_frame = TRUE;
1146 break;
1147 default:
1148 /* drop anything before the initial SPS */
1149 if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
1150 return FALSE;
1151
1152 pres = gst_h264_parser_parse_nal (nalparser, nalu);
1153 if (pres != GST_H264_PARSER_OK)
1154 return FALSE;
1155 break;
1156 }
1157
1158 /* if AVC output needed, collect properly prefixed nal in adapter,
1159 * and use that to replace outgoing buffer data later on */
1160 if (h264parse->transform) {
1161 GstBuffer *buf;
1162
1163 GST_LOG_OBJECT (h264parse, "collecting NAL in AVC frame");
1164 buf = gst_h264_parse_wrap_nal (h264parse, h264parse->format,
1165 nalu->data + nalu->offset, nalu->size);
1166 gst_adapter_push (h264parse->frame_out, buf);
1167 }
1168 return TRUE;
1169 }
1170
1171 /* caller guarantees at least 2 bytes of nal payload for each nal
1172 * returns TRUE if next_nal indicates that nal terminates an AU */
1173 static inline gboolean
gst_h264_parse_collect_nal(GstH264Parse * h264parse,GstH264NalUnit * nalu)1174 gst_h264_parse_collect_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
1175 {
1176 GstH264NalUnitType nal_type = nalu->type;
1177 gboolean complete;
1178
1179 /* determine if AU complete */
1180 GST_LOG_OBJECT (h264parse, "next nal type: %d %s (picture started %i)",
1181 nal_type, _nal_name (nal_type), h264parse->picture_start);
1182
1183 /* consider a coded slices (IDR or not) to start a picture,
1184 * (so ending the previous one) if first_mb_in_slice == 0
1185 * (non-0 is part of previous one) */
1186 /* NOTE this is not entirely according to Access Unit specs in 7.4.1.2.4,
1187 * but in practice it works in sane cases, needs not much parsing,
1188 * and also works with broken frame_num in NAL
1189 * (where spec-wise would fail) */
1190 complete = h264parse->picture_start && ((nal_type >= GST_H264_NAL_SEI &&
1191 nal_type <= GST_H264_NAL_AU_DELIMITER) ||
1192 (nal_type >= 14 && nal_type <= 18));
1193
1194 /* first_mb_in_slice == 0 considered start of frame */
1195 if (nalu->size > nalu->header_bytes)
1196 complete |= h264parse->picture_start && (nal_type == GST_H264_NAL_SLICE
1197 || nal_type == GST_H264_NAL_SLICE_DPA
1198 || nal_type == GST_H264_NAL_SLICE_IDR) &&
1199 (nalu->data[nalu->offset + nalu->header_bytes] & 0x80);
1200
1201 GST_LOG_OBJECT (h264parse, "au complete: %d", complete);
1202
1203 if (complete)
1204 h264parse->picture_start = FALSE;
1205
1206 return complete;
1207 }
1208
1209 static guint8 au_delim[6] = {
1210 0x00, 0x00, 0x00, 0x01, /* nal prefix */
1211 0x09, /* nal unit type = access unit delimiter */
1212 0xf0 /* allow any slice type */
1213 };
1214
1215 static GstFlowReturn
gst_h264_parse_handle_frame_packetized(GstBaseParse * parse,GstBaseParseFrame * frame)1216 gst_h264_parse_handle_frame_packetized (GstBaseParse * parse,
1217 GstBaseParseFrame * frame)
1218 {
1219 GstH264Parse *h264parse = GST_H264_PARSE (parse);
1220 GstBuffer *buffer = frame->buffer;
1221 GstFlowReturn ret = GST_FLOW_OK;
1222 GstH264ParserResult parse_res;
1223 GstH264NalUnit nalu;
1224 const guint nl = h264parse->nal_length_size;
1225 GstMapInfo map;
1226 gint left;
1227
1228 if (nl < 1 || nl > 4) {
1229 GST_DEBUG_OBJECT (h264parse, "insufficient data to split input");
1230 return GST_FLOW_NOT_NEGOTIATED;
1231 }
1232
1233 /* need to save buffer from invalidation upon _finish_frame */
1234 if (h264parse->split_packetized)
1235 buffer = gst_buffer_copy (frame->buffer);
1236
1237 gst_buffer_map (buffer, &map, GST_MAP_READ);
1238
1239 left = map.size;
1240
1241 GST_LOG_OBJECT (h264parse,
1242 "processing packet buffer of size %" G_GSIZE_FORMAT, map.size);
1243
1244 parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
1245 map.data, 0, map.size, nl, &nalu);
1246
1247 /* Always enable AUD insertion per frame here. The pre_push function
1248 * will only add it once, and will only add it for byte-stream output
1249 * if AUD doesn't exist in the current frame */
1250 h264parse->aud_insert = TRUE;
1251
1252 while (parse_res == GST_H264_PARSER_OK) {
1253 GST_DEBUG_OBJECT (h264parse, "AVC nal offset %d", nalu.offset + nalu.size);
1254
1255 /* either way, have a look at it */
1256 gst_h264_parse_process_nal (h264parse, &nalu);
1257
1258 /* dispatch per NALU if needed */
1259 if (h264parse->split_packetized) {
1260 GstBaseParseFrame tmp_frame;
1261
1262 gst_base_parse_frame_init (&tmp_frame);
1263 tmp_frame.flags |= frame->flags;
1264 tmp_frame.offset = frame->offset;
1265 tmp_frame.overhead = frame->overhead;
1266 tmp_frame.buffer = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
1267 nalu.offset, nalu.size);
1268
1269 /* Set marker on last packet */
1270 if (nl + nalu.size == left) {
1271 if (GST_BUFFER_FLAG_IS_SET (frame->buffer, GST_BUFFER_FLAG_MARKER))
1272 h264parse->marker = TRUE;
1273 }
1274
1275 /* note we don't need to come up with a sub-buffer, since
1276 * subsequent code only considers input buffer's metadata.
1277 * Real data is either taken from input by baseclass or
1278 * a replacement output buffer is provided anyway. */
1279 gst_h264_parse_parse_frame (parse, &tmp_frame);
1280 ret = gst_base_parse_finish_frame (parse, &tmp_frame, nl + nalu.size);
1281 left -= nl + nalu.size;
1282 }
1283
1284 parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
1285 map.data, nalu.offset + nalu.size, map.size, nl, &nalu);
1286 }
1287
1288 gst_buffer_unmap (buffer, &map);
1289
1290 if (!h264parse->split_packetized) {
1291 h264parse->marker = TRUE;
1292 gst_h264_parse_parse_frame (parse, frame);
1293 ret = gst_base_parse_finish_frame (parse, frame, map.size);
1294 } else {
1295 gst_buffer_unref (buffer);
1296 if (G_UNLIKELY (left)) {
1297 /* should not be happening for nice AVC */
1298 GST_WARNING_OBJECT (parse, "skipping leftover AVC data %d", left);
1299 frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
1300 ret = gst_base_parse_finish_frame (parse, frame, map.size);
1301 }
1302 }
1303
1304 if (parse_res == GST_H264_PARSER_NO_NAL_END ||
1305 parse_res == GST_H264_PARSER_BROKEN_DATA) {
1306
1307 if (h264parse->split_packetized) {
1308 #ifdef OHOS_OPT_COMPAT
1309 /**
1310 * ohos.opt.compat.0052
1311 * When parsing the header of AVC NAL unit fails, gstremaer will post error and playback will be stopped.
1312 * Try to drop this frame and handle next frame.
1313 */
1314 if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_DROP) {
1315 GST_DEBUG_OBJECT (h264parse, "invalid AVC input data, just drop it");
1316 return GST_FLOW_OK;
1317 }
1318 #endif
1319 GST_ELEMENT_ERROR (h264parse, STREAM, FAILED, (NULL),
1320 ("invalid AVC input data"));
1321
1322 return GST_FLOW_ERROR;
1323 } else {
1324 /* do not meddle to much in this case */
1325 GST_DEBUG_OBJECT (h264parse, "parsing packet failed");
1326 }
1327 }
1328
1329 return ret;
1330 }
1331
1332 static GstFlowReturn
gst_h264_parse_handle_frame(GstBaseParse * parse,GstBaseParseFrame * frame,gint * skipsize)1333 gst_h264_parse_handle_frame (GstBaseParse * parse,
1334 GstBaseParseFrame * frame, gint * skipsize)
1335 {
1336 GstH264Parse *h264parse = GST_H264_PARSE (parse);
1337 GstBuffer *buffer = frame->buffer;
1338 GstMapInfo map;
1339 guint8 *data;
1340 gsize size;
1341 gint current_off = 0;
1342 gboolean drain, nonext;
1343 GstH264NalParser *nalparser = h264parse->nalparser;
1344 GstH264NalUnit nalu;
1345 GstH264ParserResult pres;
1346 gint framesize;
1347
1348 if (G_UNLIKELY (GST_BUFFER_FLAG_IS_SET (frame->buffer,
1349 GST_BUFFER_FLAG_DISCONT))) {
1350 h264parse->discont = TRUE;
1351 }
1352
1353 /* delegate in packetized case, no skipping should be needed */
1354 if (h264parse->packetized)
1355 return gst_h264_parse_handle_frame_packetized (parse, frame);
1356
1357 gst_buffer_map (buffer, &map, GST_MAP_READ);
1358 data = map.data;
1359 size = map.size;
1360
1361 /* expect at least 3 bytes start_code, and 1 bytes NALU header.
1362 * the length of the NALU payload can be zero.
1363 * (e.g. EOS/EOB placed at the end of an AU.) */
1364 if (G_UNLIKELY (size < 4)) {
1365 gst_buffer_unmap (buffer, &map);
1366 *skipsize = 1;
1367 return GST_FLOW_OK;
1368 }
1369
1370 /* need to configure aggregation */
1371 if (G_UNLIKELY (h264parse->format == GST_H264_PARSE_FORMAT_NONE))
1372 gst_h264_parse_negotiate (h264parse, GST_H264_PARSE_FORMAT_BYTE, NULL);
1373
1374 /* avoid stale cached parsing state */
1375 if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME) {
1376 GST_LOG_OBJECT (h264parse, "parsing new frame");
1377 gst_h264_parse_reset_frame (h264parse);
1378 } else {
1379 GST_LOG_OBJECT (h264parse, "resuming frame parsing");
1380 }
1381
1382 /* Always consume the entire input buffer when in_align == ALIGN_AU */
1383 drain = GST_BASE_PARSE_DRAINING (parse)
1384 || h264parse->in_align == GST_H264_PARSE_ALIGN_AU;
1385 nonext = FALSE;
1386
1387 current_off = h264parse->current_off;
1388 if (current_off < 0)
1389 current_off = 0;
1390
1391 /* The parser is being drain, but no new data was added, just prentend this
1392 * AU is complete */
1393 if (drain && current_off == size) {
1394 GST_DEBUG_OBJECT (h264parse, "draining with no new data");
1395 nalu.size = 0;
1396 nalu.offset = current_off;
1397 goto end;
1398 }
1399
1400 g_assert (current_off < size);
1401 GST_DEBUG_OBJECT (h264parse, "last parse position %d", current_off);
1402
1403 /* check for initial skip */
1404 if (h264parse->current_off == -1) {
1405 pres =
1406 gst_h264_parser_identify_nalu_unchecked (nalparser, data, current_off,
1407 size, &nalu);
1408 switch (pres) {
1409 case GST_H264_PARSER_OK:
1410 if (nalu.sc_offset > 0) {
1411 *skipsize = nalu.sc_offset;
1412 goto skip;
1413 }
1414 break;
1415 case GST_H264_PARSER_NO_NAL:
1416 /* Start code may have up to 4 bytes */
1417 *skipsize = size - 4;
1418 goto skip;
1419 break;
1420 default:
1421 /* should not really occur either */
1422 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1423 ("Error parsing H.264 stream"), ("Invalid H.264 stream"));
1424 goto invalid_stream;
1425 }
1426
1427 /* Ensure we use the TS of the first NAL. This avoids broken timestamp in
1428 * the case of a miss-placed filler byte. */
1429 gst_base_parse_set_ts_at_offset (parse, nalu.offset);
1430 }
1431
1432 while (TRUE) {
1433 pres =
1434 gst_h264_parser_identify_nalu (nalparser, data, current_off, size,
1435 &nalu);
1436
1437 switch (pres) {
1438 case GST_H264_PARSER_OK:
1439 GST_DEBUG_OBJECT (h264parse, "complete nal (offset, size): (%u, %u) ",
1440 nalu.offset, nalu.size);
1441 break;
1442 case GST_H264_PARSER_NO_NAL_END:
1443 /* In NAL alignment, assume the NAL is complete */
1444 if (h264parse->in_align == GST_H264_PARSE_ALIGN_NAL ||
1445 h264parse->in_align == GST_H264_PARSE_ALIGN_AU) {
1446 nonext = TRUE;
1447 nalu.size = size - nalu.offset;
1448 break;
1449 }
1450 GST_DEBUG_OBJECT (h264parse, "not a complete nal found at offset %u",
1451 nalu.offset);
1452 /* if draining, accept it as complete nal */
1453 if (drain) {
1454 nonext = TRUE;
1455 nalu.size = size - nalu.offset;
1456 GST_DEBUG_OBJECT (h264parse, "draining, accepting with size %u",
1457 nalu.size);
1458 /* if it's not too short at least */
1459 if (nalu.size < 2)
1460 goto broken;
1461 break;
1462 }
1463 /* otherwise need more */
1464 goto more;
1465 case GST_H264_PARSER_BROKEN_LINK:
1466 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1467 ("Error parsing H.264 stream"),
1468 ("The link to structure needed for the parsing couldn't be found"));
1469 goto invalid_stream;
1470 case GST_H264_PARSER_ERROR:
1471 /* should not really occur either */
1472 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1473 ("Error parsing H.264 stream"), ("Invalid H.264 stream"));
1474 goto invalid_stream;
1475 case GST_H264_PARSER_NO_NAL:
1476 GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
1477 ("Error parsing H.264 stream"), ("No H.264 NAL unit found"));
1478 goto invalid_stream;
1479 case GST_H264_PARSER_BROKEN_DATA:
1480 GST_WARNING_OBJECT (h264parse, "input stream is corrupt; "
1481 "it contains a NAL unit of length %u", nalu.size);
1482 broken:
1483 /* broken nal at start -> arrange to skip it,
1484 * otherwise have it terminate current au
1485 * (and so it will be skipped on next frame round) */
1486 GST_ELEMENT_WARNING (h264parse, STREAM, DECODE,
1487 (NULL), ("Broken bit stream"));
1488 if (current_off == 0) {
1489 GST_DEBUG_OBJECT (h264parse, "skipping broken nal");
1490 *skipsize = nalu.offset;
1491 goto skip;
1492 } else {
1493 GST_DEBUG_OBJECT (h264parse, "terminating au");
1494 nalu.size = 0;
1495 nalu.offset = nalu.sc_offset;
1496 goto end;
1497 }
1498 break;
1499 default:
1500 g_assert_not_reached ();
1501 break;
1502 }
1503
1504 GST_DEBUG_OBJECT (h264parse, "%p complete nal found. Off: %u, Size: %u",
1505 data, nalu.offset, nalu.size);
1506
1507 if (gst_h264_parse_collect_nal (h264parse, &nalu)) {
1508 h264parse->aud_needed = TRUE;
1509 /* complete current frame, if it exist */
1510 if (current_off > 0) {
1511 nalu.size = 0;
1512 nalu.offset = nalu.sc_offset;
1513 h264parse->marker = TRUE;
1514 break;
1515 }
1516 }
1517
1518 if (!gst_h264_parse_process_nal (h264parse, &nalu)) {
1519 GST_WARNING_OBJECT (h264parse,
1520 "broken/invalid nal Type: %d %s, Size: %u will be dropped",
1521 nalu.type, _nal_name (nalu.type), nalu.size);
1522 *skipsize = nalu.size;
1523 goto skip;
1524 }
1525
1526 /* Make sure the next buffer will contain an AUD */
1527 if (h264parse->aud_needed) {
1528 h264parse->aud_insert = TRUE;
1529 h264parse->aud_needed = FALSE;
1530 }
1531
1532 /* Do not push immediately if we don't have all headers. This ensure that
1533 * our caps are complete, avoiding a renegotiation */
1534 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL &&
1535 !GST_H264_PARSE_STATE_VALID (h264parse,
1536 GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS))
1537 frame->flags |= GST_BASE_PARSE_FRAME_FLAG_QUEUE;
1538
1539 /* if no next nal, we reached the end of this buffer */
1540 if (nonext) {
1541 /* If there is a marker flag, or input is AU, we know this is complete */
1542 if (GST_BUFFER_FLAG_IS_SET (frame->buffer, GST_BUFFER_FLAG_MARKER) ||
1543 h264parse->in_align == GST_H264_PARSE_ALIGN_AU) {
1544 h264parse->marker = TRUE;
1545 break;
1546 }
1547
1548 /* or if we are draining */
1549 if (drain || h264parse->align == GST_H264_PARSE_ALIGN_NAL)
1550 break;
1551
1552 current_off = nalu.offset + nalu.size;
1553 goto more;
1554 }
1555
1556 /* If the output is NAL, we are done */
1557 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL)
1558 break;
1559
1560 GST_DEBUG_OBJECT (h264parse, "Looking for more");
1561 current_off = nalu.offset + nalu.size;
1562
1563 /* expect at least 3 bytes start_code, and 1 bytes NALU header.
1564 * the length of the NALU payload can be zero.
1565 * (e.g. EOS/EOB placed at the end of an AU.) */
1566 if (size - current_off < 4) {
1567 /* Finish the frame if there is no more data in the stream */
1568 if (drain)
1569 break;
1570
1571 goto more;
1572 }
1573 }
1574
1575 end:
1576 framesize = nalu.offset + nalu.size;
1577
1578 gst_buffer_unmap (buffer, &map);
1579
1580 gst_h264_parse_parse_frame (parse, frame);
1581
1582 return gst_base_parse_finish_frame (parse, frame, framesize);
1583
1584 more:
1585 *skipsize = 0;
1586
1587 /* Restart parsing from here next time */
1588 if (current_off > 0)
1589 h264parse->current_off = current_off;
1590
1591 /* Fall-through. */
1592 out:
1593 gst_buffer_unmap (buffer, &map);
1594 return GST_FLOW_OK;
1595
1596 skip:
1597 GST_DEBUG_OBJECT (h264parse, "skipping %d", *skipsize);
1598 /* If we are collecting access units, we need to preserve the initial
1599 * config headers (SPS, PPS et al.) and only reset the frame if another
1600 * slice NAL was received. This means that broken pictures are discarded */
1601 if (h264parse->align != GST_H264_PARSE_ALIGN_AU ||
1602 !(h264parse->state & GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS) ||
1603 (h264parse->state & GST_H264_PARSE_STATE_GOT_SLICE))
1604 gst_h264_parse_reset_frame (h264parse);
1605 goto out;
1606
1607 invalid_stream:
1608 gst_buffer_unmap (buffer, &map);
1609 return GST_FLOW_ERROR;
1610 }
1611
1612 /* byte together avc codec data based on collected pps and sps so far */
1613 static GstBuffer *
gst_h264_parse_make_codec_data(GstH264Parse * h264parse)1614 gst_h264_parse_make_codec_data (GstH264Parse * h264parse)
1615 {
1616 GstBuffer *buf, *nal;
1617 gint i, sps_size = 0, pps_size = 0, num_sps = 0, num_pps = 0;
1618 guint8 profile_idc = 0, profile_comp = 0, level_idc = 0;
1619 gboolean found = FALSE;
1620 GstMapInfo map;
1621 guint8 *data;
1622 gint nl;
1623
1624 /* only nal payload in stored nals */
1625
1626 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
1627 if ((nal = h264parse->sps_nals[i])) {
1628 gsize size = gst_buffer_get_size (nal);
1629 num_sps++;
1630 /* size bytes also count */
1631 sps_size += size + 2;
1632 if (size >= 4) {
1633 guint8 tmp[3];
1634 found = TRUE;
1635 gst_buffer_extract (nal, 1, tmp, 3);
1636 profile_idc = tmp[0];
1637 profile_comp = tmp[1];
1638 level_idc = tmp[2];
1639 }
1640 }
1641 }
1642 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
1643 if ((nal = h264parse->pps_nals[i])) {
1644 num_pps++;
1645 /* size bytes also count */
1646 pps_size += gst_buffer_get_size (nal) + 2;
1647 }
1648 }
1649
1650 /* AVC3 has SPS/PPS inside the stream, not in the codec_data */
1651 if (h264parse->format == GST_H264_PARSE_FORMAT_AVC3) {
1652 num_sps = sps_size = 0;
1653 num_pps = pps_size = 0;
1654 }
1655
1656 GST_DEBUG_OBJECT (h264parse,
1657 "constructing codec_data: num_sps=%d, num_pps=%d", num_sps, num_pps);
1658
1659 if (!found || (0 == num_pps
1660 && GST_H264_PARSE_FORMAT_AVC3 != h264parse->format))
1661 return NULL;
1662
1663 buf = gst_buffer_new_allocate (NULL, 5 + 1 + sps_size + 1 + pps_size, NULL);
1664 gst_buffer_map (buf, &map, GST_MAP_WRITE);
1665 data = map.data;
1666 nl = h264parse->nal_length_size;
1667
1668 data[0] = 1; /* AVC Decoder Configuration Record ver. 1 */
1669 data[1] = profile_idc; /* profile_idc */
1670 data[2] = profile_comp; /* profile_compability */
1671 data[3] = level_idc; /* level_idc */
1672 data[4] = 0xfc | (nl - 1); /* nal_length_size_minus1 */
1673 data[5] = 0xe0 | num_sps; /* number of SPSs */
1674
1675 data += 6;
1676 if (h264parse->format != GST_H264_PARSE_FORMAT_AVC3) {
1677 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
1678 if ((nal = h264parse->sps_nals[i])) {
1679 gsize nal_size = gst_buffer_get_size (nal);
1680 GST_WRITE_UINT16_BE (data, nal_size);
1681 gst_buffer_extract (nal, 0, data + 2, nal_size);
1682 data += 2 + nal_size;
1683 }
1684 }
1685 }
1686
1687 data[0] = num_pps;
1688 data++;
1689 if (h264parse->format != GST_H264_PARSE_FORMAT_AVC3) {
1690 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
1691 if ((nal = h264parse->pps_nals[i])) {
1692 gsize nal_size = gst_buffer_get_size (nal);
1693 GST_WRITE_UINT16_BE (data, nal_size);
1694 gst_buffer_extract (nal, 0, data + 2, nal_size);
1695 data += 2 + nal_size;
1696 }
1697 }
1698 }
1699
1700 gst_buffer_unmap (buf, &map);
1701
1702 return buf;
1703 }
1704
1705 static void
gst_h264_parse_get_par(GstH264Parse * h264parse,gint * num,gint * den)1706 gst_h264_parse_get_par (GstH264Parse * h264parse, gint * num, gint * den)
1707 {
1708 if (h264parse->upstream_par_n != -1 && h264parse->upstream_par_d != -1) {
1709 *num = h264parse->upstream_par_n;
1710 *den = h264parse->upstream_par_d;
1711 } else {
1712 *num = h264parse->parsed_par_n;
1713 *den = h264parse->parsed_par_d;
1714 }
1715 }
1716
1717 static GstCaps *
get_compatible_profile_caps(GstH264SPS * sps)1718 get_compatible_profile_caps (GstH264SPS * sps)
1719 {
1720 GstCaps *caps = NULL;
1721 const gchar **profiles = NULL;
1722 gint i;
1723 GValue compat_profiles = G_VALUE_INIT;
1724 g_value_init (&compat_profiles, GST_TYPE_LIST);
1725
1726 switch (sps->profile_idc) {
1727 case GST_H264_PROFILE_EXTENDED:
1728 if (sps->constraint_set0_flag) { /* A.2.1 */
1729 if (sps->constraint_set1_flag) {
1730 static const gchar *profile_array[] =
1731 { "constrained-baseline", "baseline", "main", "high",
1732 "high-10", "high-4:2:2", "high-4:4:4", NULL
1733 };
1734 profiles = profile_array;
1735 } else {
1736 static const gchar *profile_array[] = { "baseline", NULL };
1737 profiles = profile_array;
1738 }
1739 } else if (sps->constraint_set1_flag) { /* A.2.2 */
1740 static const gchar *profile_array[] =
1741 { "main", "high", "high-10", "high-4:2:2", "high-4:4:4", NULL };
1742 profiles = profile_array;
1743 }
1744 break;
1745 case GST_H264_PROFILE_BASELINE:
1746 if (sps->constraint_set1_flag) { /* A.2.1 */
1747 static const gchar *profile_array[] =
1748 { "baseline", "main", "high", "high-10", "high-4:2:2",
1749 "high-4:4:4", NULL
1750 };
1751 profiles = profile_array;
1752 } else {
1753 static const gchar *profile_array[] = { "extended", NULL };
1754 profiles = profile_array;
1755 }
1756 break;
1757 case GST_H264_PROFILE_MAIN:
1758 {
1759 static const gchar *profile_array[] =
1760 { "high", "high-10", "high-4:2:2", "high-4:4:4", NULL };
1761 profiles = profile_array;
1762 }
1763 break;
1764 case GST_H264_PROFILE_HIGH:
1765 if (sps->constraint_set1_flag) {
1766 static const gchar *profile_array[] =
1767 { "main", "high-10", "high-4:2:2", "high-4:4:4", NULL };
1768 profiles = profile_array;
1769 } else {
1770 static const gchar *profile_array[] =
1771 { "high-10", "high-4:2:2", "high-4:4:4", NULL };
1772 profiles = profile_array;
1773 }
1774 break;
1775 case GST_H264_PROFILE_HIGH10:
1776 if (sps->constraint_set1_flag) {
1777 static const gchar *profile_array[] =
1778 { "main", "high", "high-4:2:2", "high-4:4:4", NULL };
1779 profiles = profile_array;
1780 } else {
1781 if (sps->constraint_set3_flag) { /* A.2.8 */
1782 static const gchar *profile_array[] =
1783 { "high-10", "high-4:2:2", "high-4:4:4", "high-4:2:2-intra",
1784 "high-4:4:4-intra", NULL
1785 };
1786 profiles = profile_array;
1787 } else {
1788 static const gchar *profile_array[] =
1789 { "high-4:2:2", "high-4:4:4", NULL };
1790 profiles = profile_array;
1791 }
1792 }
1793 break;
1794 case GST_H264_PROFILE_HIGH_422:
1795 if (sps->constraint_set1_flag) {
1796 static const gchar *profile_array[] =
1797 { "main", "high", "high-10", "high-4:4:4", NULL };
1798 profiles = profile_array;
1799 } else {
1800 if (sps->constraint_set3_flag) { /* A.2.9 */
1801 static const gchar *profile_array[] =
1802 { "high-4:2:2", "high-4:4:4", "high-4:2:2-intra",
1803 "high-4:4:4-intra", NULL
1804 };
1805 profiles = profile_array;
1806 } else {
1807 static const gchar *profile_array[] =
1808 { "high-4:2:2", "high-4:4:4", NULL };
1809 profiles = profile_array;
1810 }
1811 }
1812 break;
1813 case GST_H264_PROFILE_HIGH_444:
1814 if (sps->constraint_set1_flag) {
1815 static const gchar *profile_array[] =
1816 { "main", "high", "high-10", "high-4:2:2", NULL };
1817 profiles = profile_array;
1818 } else if (sps->constraint_set3_flag) { /* A.2.10 */
1819 static const gchar *profile_array[] = { "high-4:4:4", NULL };
1820 profiles = profile_array;
1821 }
1822 break;
1823 case GST_H264_PROFILE_MULTIVIEW_HIGH:
1824 if (sps->extension_type == GST_H264_NAL_EXTENSION_MVC
1825 && sps->extension.mvc.num_views_minus1 == 1) {
1826 static const gchar *profile_array[] =
1827 { "stereo-high", "multiview-high", NULL };
1828 profiles = profile_array;
1829 } else {
1830 static const gchar *profile_array[] = { "multiview-high", NULL };
1831 profiles = profile_array;
1832 }
1833 break;
1834 default:
1835 break;
1836 }
1837
1838 if (profiles) {
1839 GValue value = G_VALUE_INIT;
1840 caps = gst_caps_new_empty_simple ("video/x-h264");
1841 for (i = 0; profiles[i]; i++) {
1842 g_value_init (&value, G_TYPE_STRING);
1843 g_value_set_string (&value, profiles[i]);
1844 gst_value_list_append_value (&compat_profiles, &value);
1845 g_value_unset (&value);
1846 }
1847 gst_caps_set_value (caps, "profile", &compat_profiles);
1848 g_value_unset (&compat_profiles);
1849 }
1850
1851 return caps;
1852 }
1853
1854 /* if downstream didn't support the exact profile indicated in sps header,
1855 * check for the compatible profiles also */
1856 static void
ensure_caps_profile(GstH264Parse * h264parse,GstCaps * caps,GstH264SPS * sps)1857 ensure_caps_profile (GstH264Parse * h264parse, GstCaps * caps, GstH264SPS * sps)
1858 {
1859 GstCaps *peer_caps, *compat_caps;
1860
1861 peer_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
1862 if (!peer_caps || !gst_caps_can_intersect (caps, peer_caps)) {
1863 GstCaps *filter_caps = gst_caps_new_empty_simple ("video/x-h264");
1864
1865 if (peer_caps)
1866 gst_caps_unref (peer_caps);
1867 peer_caps =
1868 gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (h264parse),
1869 filter_caps);
1870
1871 gst_caps_unref (filter_caps);
1872 }
1873
1874 if (peer_caps && !gst_caps_can_intersect (caps, peer_caps)) {
1875 GstStructure *structure;
1876
1877 compat_caps = get_compatible_profile_caps (sps);
1878 if (compat_caps != NULL) {
1879 GstCaps *res_caps = NULL;
1880
1881 res_caps = gst_caps_intersect (peer_caps, compat_caps);
1882
1883 if (res_caps && !gst_caps_is_empty (res_caps)) {
1884 const gchar *profile_str = NULL;
1885
1886 res_caps = gst_caps_fixate (res_caps);
1887 structure = gst_caps_get_structure (res_caps, 0);
1888 profile_str = gst_structure_get_string (structure, "profile");
1889 if (profile_str) {
1890 gst_caps_set_simple (caps, "profile", G_TYPE_STRING, profile_str,
1891 NULL);
1892 GST_DEBUG_OBJECT (h264parse,
1893 "Setting compatible profile %s to the caps", profile_str);
1894 }
1895 }
1896 if (res_caps)
1897 gst_caps_unref (res_caps);
1898 gst_caps_unref (compat_caps);
1899 }
1900 }
1901 if (peer_caps)
1902 gst_caps_unref (peer_caps);
1903 }
1904
1905 static const gchar *
digit_to_string(guint digit)1906 digit_to_string (guint digit)
1907 {
1908 static const char itoa[][2] = {
1909 "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"
1910 };
1911
1912 if (G_LIKELY (digit < 10))
1913 return itoa[digit];
1914 else
1915 return NULL;
1916 }
1917
1918 static const gchar *
get_profile_string(GstH264SPS * sps)1919 get_profile_string (GstH264SPS * sps)
1920 {
1921 const gchar *profile = NULL;
1922
1923 switch (sps->profile_idc) {
1924 case 66:
1925 if (sps->constraint_set1_flag)
1926 profile = "constrained-baseline";
1927 else
1928 profile = "baseline";
1929 break;
1930 case 77:
1931 profile = "main";
1932 break;
1933 case 88:
1934 profile = "extended";
1935 break;
1936 case 100:
1937 if (sps->constraint_set4_flag) {
1938 if (sps->constraint_set5_flag)
1939 profile = "constrained-high";
1940 else
1941 profile = "progressive-high";
1942 } else
1943 profile = "high";
1944 break;
1945 case 110:
1946 if (sps->constraint_set3_flag)
1947 profile = "high-10-intra";
1948 else if (sps->constraint_set4_flag)
1949 profile = "progressive-high-10";
1950 else
1951 profile = "high-10";
1952 break;
1953 case 122:
1954 if (sps->constraint_set3_flag)
1955 profile = "high-4:2:2-intra";
1956 else
1957 profile = "high-4:2:2";
1958 break;
1959 case 244:
1960 if (sps->constraint_set3_flag)
1961 profile = "high-4:4:4-intra";
1962 else
1963 profile = "high-4:4:4";
1964 break;
1965 case 44:
1966 profile = "cavlc-4:4:4-intra";
1967 break;
1968 case 118:
1969 profile = "multiview-high";
1970 break;
1971 case 128:
1972 profile = "stereo-high";
1973 break;
1974 case 83:
1975 if (sps->constraint_set5_flag)
1976 profile = "scalable-constrained-baseline";
1977 else
1978 profile = "scalable-baseline";
1979 break;
1980 case 86:
1981 if (sps->constraint_set3_flag)
1982 profile = "scalable-high-intra";
1983 else if (sps->constraint_set5_flag)
1984 profile = "scalable-constrained-high";
1985 else
1986 profile = "scalable-high";
1987 break;
1988 default:
1989 return NULL;
1990 }
1991
1992 return profile;
1993 }
1994
1995 static const gchar *
get_level_string(GstH264SPS * sps)1996 get_level_string (GstH264SPS * sps)
1997 {
1998 if (sps->level_idc == 0)
1999 return NULL;
2000 else if ((sps->level_idc == 11 && sps->constraint_set3_flag)
2001 || sps->level_idc == 9)
2002 return "1b";
2003 else if (sps->level_idc % 10 == 0)
2004 return digit_to_string (sps->level_idc / 10);
2005 else {
2006 switch (sps->level_idc) {
2007 case 11:
2008 return "1.1";
2009 case 12:
2010 return "1.2";
2011 case 13:
2012 return "1.3";
2013 case 21:
2014 return "2.1";
2015 case 22:
2016 return "2.2";
2017 case 31:
2018 return "3.1";
2019 case 32:
2020 return "3.2";
2021 case 41:
2022 return "4.1";
2023 case 42:
2024 return "4.2";
2025 case 51:
2026 return "5.1";
2027 case 52:
2028 return "5.2";
2029 case 61:
2030 return "6.1";
2031 case 62:
2032 return "6.2";
2033 default:
2034 return NULL;
2035 }
2036 }
2037 }
2038
2039 static void
gst_h264_parse_update_src_caps(GstH264Parse * h264parse,GstCaps * caps)2040 gst_h264_parse_update_src_caps (GstH264Parse * h264parse, GstCaps * caps)
2041 {
2042 GstH264SPS *sps;
2043 GstCaps *sink_caps, *src_caps;
2044 gboolean modified = FALSE;
2045 GstBuffer *buf = NULL;
2046 GstStructure *s = NULL;
2047
2048 if (G_UNLIKELY (!gst_pad_has_current_caps (GST_BASE_PARSE_SRC_PAD
2049 (h264parse))))
2050 modified = TRUE;
2051 else if (G_UNLIKELY (!h264parse->update_caps))
2052 return;
2053
2054 /* if this is being called from the first _setcaps call, caps on the sinkpad
2055 * aren't set yet and so they need to be passed as an argument */
2056 if (caps)
2057 sink_caps = gst_caps_ref (caps);
2058 else
2059 sink_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SINK_PAD (h264parse));
2060
2061 /* carry over input caps as much as possible; override with our own stuff */
2062 if (!sink_caps)
2063 sink_caps = gst_caps_new_empty_simple ("video/x-h264");
2064 else
2065 s = gst_caps_get_structure (sink_caps, 0);
2066
2067 sps = h264parse->nalparser->last_sps;
2068 GST_DEBUG_OBJECT (h264parse, "sps: %p", sps);
2069
2070 /* only codec-data for nice-and-clean au aligned packetized avc format */
2071 if ((h264parse->format == GST_H264_PARSE_FORMAT_AVC
2072 || h264parse->format == GST_H264_PARSE_FORMAT_AVC3)
2073 && h264parse->align == GST_H264_PARSE_ALIGN_AU) {
2074 buf = gst_h264_parse_make_codec_data (h264parse);
2075 if (buf && h264parse->codec_data) {
2076 GstMapInfo map;
2077
2078 gst_buffer_map (buf, &map, GST_MAP_READ);
2079 if (map.size != gst_buffer_get_size (h264parse->codec_data) ||
2080 gst_buffer_memcmp (h264parse->codec_data, 0, map.data, map.size))
2081 modified = TRUE;
2082
2083 gst_buffer_unmap (buf, &map);
2084 } else {
2085 if (!buf && h264parse->codec_data_in)
2086 buf = gst_buffer_ref (h264parse->codec_data_in);
2087 modified = TRUE;
2088 }
2089 }
2090
2091 caps = NULL;
2092 if (G_UNLIKELY (!sps)) {
2093 caps = gst_caps_copy (sink_caps);
2094 } else {
2095 gint crop_width, crop_height;
2096 gint fps_num, fps_den;
2097 gint par_n, par_d;
2098 GstH264VUIParams *vui = &sps->vui_parameters;
2099 gchar *colorimetry = NULL;
2100
2101 if (sps->frame_cropping_flag) {
2102 crop_width = sps->crop_rect_width;
2103 crop_height = sps->crop_rect_height;
2104 } else {
2105 crop_width = sps->width;
2106 crop_height = sps->height;
2107 }
2108
2109 if (G_UNLIKELY (h264parse->width != crop_width ||
2110 h264parse->height != crop_height)) {
2111 GST_INFO_OBJECT (h264parse, "resolution changed %dx%d",
2112 crop_width, crop_height);
2113 h264parse->width = crop_width;
2114 h264parse->height = crop_height;
2115 modified = TRUE;
2116 }
2117
2118 /* 0/1 is set as the default in the codec parser, we will set
2119 * it in case we have no info */
2120 gst_h264_video_calculate_framerate (sps, h264parse->field_pic_flag,
2121 h264parse->sei_pic_struct, &fps_num, &fps_den);
2122 if (G_UNLIKELY (h264parse->fps_num != fps_num
2123 || h264parse->fps_den != fps_den)) {
2124 GST_DEBUG_OBJECT (h264parse, "framerate changed %d/%d", fps_num, fps_den);
2125 h264parse->fps_num = fps_num;
2126 h264parse->fps_den = fps_den;
2127 modified = TRUE;
2128 }
2129
2130 if (vui->aspect_ratio_info_present_flag) {
2131 if (G_UNLIKELY ((h264parse->parsed_par_n != vui->par_n)
2132 || (h264parse->parsed_par_d != vui->par_d))) {
2133 h264parse->parsed_par_n = vui->par_n;
2134 h264parse->parsed_par_d = vui->par_d;
2135 GST_INFO_OBJECT (h264parse, "pixel aspect ratio has been changed %d/%d",
2136 h264parse->parsed_par_n, h264parse->parsed_par_d);
2137 modified = TRUE;
2138 }
2139 }
2140
2141 if (vui->video_signal_type_present_flag &&
2142 vui->colour_description_present_flag) {
2143 GstVideoColorimetry ci = { 0, };
2144 gchar *old_colorimetry = NULL;
2145
2146 if (vui->video_full_range_flag)
2147 ci.range = GST_VIDEO_COLOR_RANGE_0_255;
2148 else
2149 ci.range = GST_VIDEO_COLOR_RANGE_16_235;
2150
2151 ci.matrix = gst_video_color_matrix_from_iso (vui->matrix_coefficients);
2152 ci.transfer =
2153 gst_video_transfer_function_from_iso (vui->transfer_characteristics);
2154 ci.primaries = gst_video_color_primaries_from_iso (vui->colour_primaries);
2155
2156 old_colorimetry =
2157 gst_video_colorimetry_to_string (&h264parse->parsed_colorimetry);
2158 colorimetry = gst_video_colorimetry_to_string (&ci);
2159
2160 if (colorimetry && g_strcmp0 (old_colorimetry, colorimetry)) {
2161 GST_INFO_OBJECT (h264parse,
2162 "colorimetry has been changed from %s to %s",
2163 GST_STR_NULL (old_colorimetry), colorimetry);
2164 h264parse->parsed_colorimetry = ci;
2165 modified = TRUE;
2166 }
2167
2168 g_free (old_colorimetry);
2169 }
2170
2171 if (G_UNLIKELY (modified || h264parse->update_caps)) {
2172 gint width, height;
2173 GstClockTime latency = 0;
2174
2175 const gchar *caps_mview_mode = NULL;
2176 GstVideoMultiviewMode mview_mode = h264parse->multiview_mode;
2177 GstVideoMultiviewFlags mview_flags = h264parse->multiview_flags;
2178 const gchar *chroma_format = NULL;
2179 guint bit_depth_chroma;
2180
2181 fps_num = h264parse->fps_num;
2182 fps_den = h264parse->fps_den;
2183
2184 caps = gst_caps_copy (sink_caps);
2185
2186 /* sps should give this but upstream overrides */
2187 if (s && gst_structure_has_field (s, "width"))
2188 gst_structure_get_int (s, "width", &width);
2189 else
2190 width = h264parse->width;
2191
2192 if (s && gst_structure_has_field (s, "height"))
2193 gst_structure_get_int (s, "height", &height);
2194 else
2195 height = h264parse->height;
2196
2197 if (s == NULL ||
2198 !gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n,
2199 &par_d)) {
2200 gst_h264_parse_get_par (h264parse, &par_n, &par_d);
2201 if (par_n != 0 && par_d != 0) {
2202 GST_INFO_OBJECT (h264parse, "PAR %d/%d", par_n, par_d);
2203 gst_caps_set_simple (caps, "pixel-aspect-ratio", GST_TYPE_FRACTION,
2204 par_n, par_d, NULL);
2205 } else {
2206 /* Assume par_n/par_d of 1/1 for calcs below, but don't set into caps */
2207 par_n = par_d = 1;
2208 }
2209 }
2210
2211 /* Pass through or set output stereo/multiview config */
2212 if (s && gst_structure_has_field (s, "multiview-mode")) {
2213 caps_mview_mode = gst_structure_get_string (s, "multiview-mode");
2214 gst_structure_get_flagset (s, "multiview-flags",
2215 (guint *) & mview_flags, NULL);
2216 } else if (mview_mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
2217 if (gst_video_multiview_guess_half_aspect (mview_mode,
2218 width, height, par_n, par_d)) {
2219 mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT;
2220 }
2221
2222 caps_mview_mode = gst_video_multiview_mode_to_caps_string (mview_mode);
2223 gst_caps_set_simple (caps, "multiview-mode", G_TYPE_STRING,
2224 caps_mview_mode, "multiview-flags",
2225 GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags,
2226 GST_FLAG_SET_MASK_EXACT, NULL);
2227 }
2228
2229 gst_caps_set_simple (caps, "width", G_TYPE_INT, width,
2230 "height", G_TYPE_INT, height, NULL);
2231
2232 /* upstream overrides */
2233 if (s && gst_structure_has_field (s, "framerate")) {
2234 gst_structure_get_fraction (s, "framerate", &fps_num, &fps_den);
2235 }
2236
2237 /* but not necessarily or reliably this */
2238 if (fps_den > 0) {
2239 GstStructure *s2;
2240 gst_caps_set_simple (caps, "framerate",
2241 GST_TYPE_FRACTION, fps_num, fps_den, NULL);
2242 s2 = gst_caps_get_structure (caps, 0);
2243 gst_structure_get_fraction (s2, "framerate", &h264parse->parsed_fps_n,
2244 &h264parse->parsed_fps_d);
2245
2246 /* If we know the frame duration, and if we are not in one of the zero
2247 * latency pattern, add one frame of latency */
2248 if (fps_num > 0 && h264parse->in_align != GST_H264_PARSE_ALIGN_AU &&
2249 !(h264parse->in_align == GST_H264_PARSE_ALIGN_NAL &&
2250 h264parse->align == GST_H264_PARSE_ALIGN_NAL)) {
2251 latency = gst_util_uint64_scale (GST_SECOND, fps_den, fps_num);
2252 }
2253
2254 gst_base_parse_set_latency (GST_BASE_PARSE (h264parse), latency,
2255 latency);
2256 }
2257
2258 bit_depth_chroma = sps->bit_depth_chroma_minus8 + 8;
2259
2260 switch (sps->chroma_format_idc) {
2261 case 0:
2262 chroma_format = "4:0:0";
2263 bit_depth_chroma = 0;
2264 break;
2265 case 1:
2266 chroma_format = "4:2:0";
2267 break;
2268 case 2:
2269 chroma_format = "4:2:2";
2270 break;
2271 case 3:
2272 chroma_format = "4:4:4";
2273 break;
2274 default:
2275 break;
2276 }
2277
2278 if (chroma_format)
2279 gst_caps_set_simple (caps,
2280 "chroma-format", G_TYPE_STRING, chroma_format,
2281 "bit-depth-luma", G_TYPE_UINT, sps->bit_depth_luma_minus8 + 8,
2282 "bit-depth-chroma", G_TYPE_UINT, bit_depth_chroma, NULL);
2283
2284 if (colorimetry && (!s || !gst_structure_has_field (s, "colorimetry"))) {
2285 gst_caps_set_simple (caps, "colorimetry", G_TYPE_STRING, colorimetry,
2286 NULL);
2287 }
2288 }
2289
2290 g_free (colorimetry);
2291 }
2292
2293 if (caps) {
2294 const gchar *mdi_str = NULL;
2295 const gchar *cll_str = NULL;
2296 gboolean codec_data_modified = FALSE;
2297
2298 gst_caps_set_simple (caps, "parsed", G_TYPE_BOOLEAN, TRUE,
2299 "stream-format", G_TYPE_STRING,
2300 gst_h264_parse_get_string (h264parse, TRUE, h264parse->format),
2301 "alignment", G_TYPE_STRING,
2302 gst_h264_parse_get_string (h264parse, FALSE, h264parse->align), NULL);
2303
2304 /* set profile and level in caps */
2305 if (sps) {
2306 const gchar *profile, *level;
2307
2308 profile = get_profile_string (sps);
2309 if (profile != NULL)
2310 gst_caps_set_simple (caps, "profile", G_TYPE_STRING, profile, NULL);
2311
2312 level = get_level_string (sps);
2313 if (level != NULL)
2314 gst_caps_set_simple (caps, "level", G_TYPE_STRING, level, NULL);
2315
2316 /* relax the profile constraint to find a suitable decoder */
2317 ensure_caps_profile (h264parse, caps, sps);
2318 }
2319
2320 if (s)
2321 mdi_str = gst_structure_get_string (s, "mastering-display-info");
2322 if (mdi_str) {
2323 gst_caps_set_simple (caps, "mastering-display-info", G_TYPE_STRING,
2324 mdi_str, NULL);
2325 } else if (h264parse->mastering_display_info_state !=
2326 GST_H264_PARSE_SEI_EXPIRED &&
2327 !gst_video_mastering_display_info_add_to_caps
2328 (&h264parse->mastering_display_info, caps)) {
2329 GST_WARNING_OBJECT (h264parse,
2330 "Couldn't set mastering display info to caps");
2331 }
2332
2333 if (s)
2334 cll_str = gst_structure_get_string (s, "content-light-level");
2335 if (cll_str) {
2336 gst_caps_set_simple (caps, "content-light-level", G_TYPE_STRING, cll_str,
2337 NULL);
2338 } else if (h264parse->content_light_level_state !=
2339 GST_H264_PARSE_SEI_EXPIRED &&
2340 !gst_video_content_light_level_add_to_caps
2341 (&h264parse->content_light_level, caps)) {
2342 GST_WARNING_OBJECT (h264parse,
2343 "Couldn't set content light level to caps");
2344 }
2345
2346 src_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
2347
2348 if (src_caps) {
2349 GstStructure *src_caps_str = gst_caps_get_structure (src_caps, 0);
2350
2351 /* use codec data from old caps for comparison if we have pushed frame for now.
2352 * we don't want to resend caps if everything is same except codec data.
2353 * However, if the updated sps/pps is not in bitstream, we should put
2354 * it on bitstream */
2355 if (gst_structure_has_field (src_caps_str, "codec_data")) {
2356 const GValue *codec_data_value =
2357 gst_structure_get_value (src_caps_str, "codec_data");
2358
2359 if (!GST_VALUE_HOLDS_BUFFER (codec_data_value)) {
2360 GST_WARNING_OBJECT (h264parse, "codec_data does not hold buffer");
2361 } else if (!h264parse->first_frame) {
2362 /* If there is no pushed frame before, we can update caps without worry.
2363 * But updating codec_data in the middle of frames
2364 * (especially on non-keyframe) might make downstream be confused.
2365 * Therefore we are setting old codec data
2366 * (i.e., was pushed to downstream previously) to new caps candidate
2367 * here for gst_caps_is_strictly_equal() to be returned TRUE if only
2368 * the codec_data is different, and to avoid re-sending caps it
2369 * that case.
2370 */
2371 gst_caps_set_value (caps, "codec_data", codec_data_value);
2372
2373 /* check for codec_data update to re-send sps/pps inband data if
2374 * current frame has no sps/pps but upstream codec_data was updated */
2375 if ((!h264parse->have_sps_in_frame || !h264parse->have_pps_in_frame)
2376 && buf) {
2377 GstBuffer *codec_data_buf = gst_value_get_buffer (codec_data_value);
2378 GstMapInfo map;
2379
2380 gst_buffer_map (buf, &map, GST_MAP_READ);
2381 if (map.size != gst_buffer_get_size (codec_data_buf) ||
2382 gst_buffer_memcmp (codec_data_buf, 0, map.data, map.size)) {
2383 codec_data_modified = TRUE;
2384 }
2385
2386 gst_buffer_unmap (buf, &map);
2387 }
2388 }
2389 } else if (!buf) {
2390 GstStructure *s;
2391 /* remove any left-over codec-data hanging around */
2392 s = gst_caps_get_structure (caps, 0);
2393 gst_structure_remove_field (s, "codec_data");
2394 }
2395 }
2396
2397 if (!(src_caps && gst_caps_is_strictly_equal (src_caps, caps))) {
2398 /* update codec data to new value */
2399 if (buf) {
2400 gst_caps_set_simple (caps, "codec_data", GST_TYPE_BUFFER, buf, NULL);
2401 gst_buffer_replace (&h264parse->codec_data, buf);
2402 gst_buffer_unref (buf);
2403 buf = NULL;
2404 } else {
2405 GstStructure *s;
2406 /* remove any left-over codec-data hanging around */
2407 s = gst_caps_get_structure (caps, 0);
2408 gst_structure_remove_field (s, "codec_data");
2409 gst_buffer_replace (&h264parse->codec_data, NULL);
2410 }
2411
2412 gst_pad_set_caps (GST_BASE_PARSE_SRC_PAD (h264parse), caps);
2413 } else if (codec_data_modified) {
2414 GST_DEBUG_OBJECT (h264parse,
2415 "Only codec_data is different, need inband sps/pps update");
2416
2417 /* this will insert updated codec_data with next idr */
2418 h264parse->push_codec = TRUE;
2419 }
2420
2421 if (src_caps)
2422 gst_caps_unref (src_caps);
2423 gst_caps_unref (caps);
2424 }
2425
2426 gst_caps_unref (sink_caps);
2427 if (buf)
2428 gst_buffer_unref (buf);
2429 }
2430
2431 static GstClockTime
gst_h264_parse_get_duration(GstH264Parse * h264parse,gboolean frame)2432 gst_h264_parse_get_duration (GstH264Parse * h264parse, gboolean frame)
2433 {
2434 GstClockTime ret = GST_CLOCK_TIME_NONE;
2435 GstH264SPS *sps = h264parse->nalparser->last_sps;
2436 gint duration = 1;
2437
2438 if (!frame) {
2439 GST_LOG_OBJECT (h264parse, "no frame data -> 0 duration");
2440 ret = 0;
2441 goto done;
2442 }
2443
2444 if (!sps) {
2445 GST_DEBUG_OBJECT (h264parse, "referred SPS invalid");
2446 goto fps_duration;
2447 } else if (!sps->vui_parameters_present_flag) {
2448 GST_DEBUG_OBJECT (h264parse, "unable to compute duration: VUI not present");
2449 goto fps_duration;
2450 } else if (!sps->vui_parameters.timing_info_present_flag) {
2451 GST_DEBUG_OBJECT (h264parse,
2452 "unable to compute duration: timing info not present");
2453 goto fps_duration;
2454 } else if (sps->vui_parameters.time_scale == 0) {
2455 GST_DEBUG_OBJECT (h264parse,
2456 "unable to compute duration: time_scale = 0 "
2457 "(this is forbidden in spec; bitstream probably contains error)");
2458 goto fps_duration;
2459 }
2460
2461 if (h264parse->sei_pic_struct_pres_flag &&
2462 h264parse->sei_pic_struct != (guint8) - 1) {
2463 /* Note that when h264parse->sei_pic_struct == -1 (unspecified), there
2464 * are ways to infer its value. This is related to computing the
2465 * TopFieldOrderCnt and BottomFieldOrderCnt, which looks
2466 * complicated and thus not implemented for the time being. Yet
2467 * the value we have here is correct for many applications
2468 */
2469 switch (h264parse->sei_pic_struct) {
2470 case GST_H264_SEI_PIC_STRUCT_TOP_FIELD:
2471 case GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD:
2472 duration = 1;
2473 break;
2474 case GST_H264_SEI_PIC_STRUCT_FRAME:
2475 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM:
2476 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP:
2477 duration = 2;
2478 break;
2479 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP:
2480 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM:
2481 duration = 3;
2482 break;
2483 case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING:
2484 duration = 4;
2485 break;
2486 case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING:
2487 duration = 6;
2488 break;
2489 default:
2490 GST_DEBUG_OBJECT (h264parse,
2491 "h264parse->sei_pic_struct of unknown value %d. Not parsed",
2492 h264parse->sei_pic_struct);
2493 break;
2494 }
2495 } else {
2496 duration = h264parse->field_pic_flag ? 1 : 2;
2497 }
2498
2499 GST_LOG_OBJECT (h264parse, "frame tick duration %d", duration);
2500
2501 ret = gst_util_uint64_scale (duration * GST_SECOND,
2502 sps->vui_parameters.num_units_in_tick, sps->vui_parameters.time_scale);
2503 /* sanity check */
2504 if (ret < GST_MSECOND) {
2505 GST_DEBUG_OBJECT (h264parse, "discarding dur %" GST_TIME_FORMAT,
2506 GST_TIME_ARGS (ret));
2507 goto fps_duration;
2508 }
2509
2510 done:
2511 return ret;
2512
2513 fps_duration:
2514 if (h264parse->parsed_fps_d > 0 && h264parse->parsed_fps_n > 0)
2515 ret =
2516 gst_util_uint64_scale (GST_SECOND, h264parse->parsed_fps_d,
2517 h264parse->parsed_fps_n);
2518 goto done;
2519 }
2520
2521 static void
gst_h264_parse_get_timestamp(GstH264Parse * h264parse,GstClockTime * out_ts,GstClockTime * out_dur,gboolean frame)2522 gst_h264_parse_get_timestamp (GstH264Parse * h264parse,
2523 GstClockTime * out_ts, GstClockTime * out_dur, gboolean frame)
2524 {
2525 GstH264SPS *sps = h264parse->nalparser->last_sps;
2526 GstClockTime upstream;
2527 gint duration = 1;
2528
2529 g_return_if_fail (out_dur != NULL);
2530 g_return_if_fail (out_ts != NULL);
2531
2532 upstream = *out_ts;
2533 GST_LOG_OBJECT (h264parse, "Upstream ts %" GST_TIME_FORMAT,
2534 GST_TIME_ARGS (upstream));
2535
2536 if (!frame) {
2537 GST_LOG_OBJECT (h264parse, "no frame data -> 0 duration");
2538 *out_dur = 0;
2539 goto exit;
2540 }
2541
2542 if (!sps) {
2543 GST_DEBUG_OBJECT (h264parse, "referred SPS invalid");
2544 goto exit;
2545 } else if (!sps->vui_parameters_present_flag) {
2546 GST_DEBUG_OBJECT (h264parse,
2547 "unable to compute timestamp: VUI not present");
2548 goto exit;
2549 } else if (!sps->vui_parameters.timing_info_present_flag) {
2550 GST_DEBUG_OBJECT (h264parse,
2551 "unable to compute timestamp: timing info not present");
2552 goto exit;
2553 } else if (sps->vui_parameters.time_scale == 0) {
2554 GST_DEBUG_OBJECT (h264parse,
2555 "unable to compute timestamp: time_scale = 0 "
2556 "(this is forbidden in spec; bitstream probably contains error)");
2557 goto exit;
2558 }
2559
2560 if (h264parse->sei_pic_struct_pres_flag &&
2561 h264parse->sei_pic_struct != (guint8) - 1) {
2562 /* Note that when h264parse->sei_pic_struct == -1 (unspecified), there
2563 * are ways to infer its value. This is related to computing the
2564 * TopFieldOrderCnt and BottomFieldOrderCnt, which looks
2565 * complicated and thus not implemented for the time being. Yet
2566 * the value we have here is correct for many applications
2567 */
2568 switch (h264parse->sei_pic_struct) {
2569 case GST_H264_SEI_PIC_STRUCT_TOP_FIELD:
2570 case GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD:
2571 duration = 1;
2572 break;
2573 case GST_H264_SEI_PIC_STRUCT_FRAME:
2574 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM:
2575 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP:
2576 duration = 2;
2577 break;
2578 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP:
2579 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM:
2580 duration = 3;
2581 break;
2582 case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING:
2583 duration = 4;
2584 break;
2585 case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING:
2586 duration = 6;
2587 break;
2588 default:
2589 GST_DEBUG_OBJECT (h264parse,
2590 "h264parse->sei_pic_struct of unknown value %d. Not parsed",
2591 h264parse->sei_pic_struct);
2592 break;
2593 }
2594 } else {
2595 duration = h264parse->field_pic_flag ? 1 : 2;
2596 }
2597
2598 GST_LOG_OBJECT (h264parse, "frame tick duration %d", duration);
2599
2600 /*
2601 * h264parse.264 C.1.2 Timing of coded picture removal (equivalent to DTS):
2602 * Tr,n(0) = initial_cpb_removal_delay[ SchedSelIdx ] / 90000
2603 * Tr,n(n) = Tr,n(nb) + Tc * cpb_removal_delay(n)
2604 * where
2605 * Tc = num_units_in_tick / time_scale
2606 */
2607
2608 if (h264parse->ts_trn_nb != GST_CLOCK_TIME_NONE) {
2609 GST_LOG_OBJECT (h264parse, "buffering based ts");
2610 /* buffering period is present */
2611 if (upstream != GST_CLOCK_TIME_NONE) {
2612 /* If upstream timestamp is valid, we respect it and adjust current
2613 * reference point */
2614 h264parse->ts_trn_nb = upstream -
2615 (GstClockTime) gst_util_uint64_scale
2616 (h264parse->sei_cpb_removal_delay * GST_SECOND,
2617 sps->vui_parameters.num_units_in_tick,
2618 sps->vui_parameters.time_scale);
2619 } else {
2620 /* If no upstream timestamp is given, we write in new timestamp */
2621 upstream = h264parse->dts = h264parse->ts_trn_nb +
2622 (GstClockTime) gst_util_uint64_scale
2623 (h264parse->sei_cpb_removal_delay * GST_SECOND,
2624 sps->vui_parameters.num_units_in_tick,
2625 sps->vui_parameters.time_scale);
2626 }
2627 } else {
2628 GstClockTime dur;
2629
2630 GST_LOG_OBJECT (h264parse, "duration based ts");
2631 /* naive method: no removal delay specified
2632 * track upstream timestamp and provide best guess frame duration */
2633 dur = gst_util_uint64_scale (duration * GST_SECOND,
2634 sps->vui_parameters.num_units_in_tick, sps->vui_parameters.time_scale);
2635 /* sanity check */
2636 if (dur < GST_MSECOND) {
2637 GST_DEBUG_OBJECT (h264parse, "discarding dur %" GST_TIME_FORMAT,
2638 GST_TIME_ARGS (dur));
2639 } else {
2640 *out_dur = dur;
2641 }
2642 }
2643
2644 exit:
2645 if (GST_CLOCK_TIME_IS_VALID (upstream))
2646 *out_ts = h264parse->dts = upstream;
2647
2648 if (GST_CLOCK_TIME_IS_VALID (*out_dur) &&
2649 GST_CLOCK_TIME_IS_VALID (h264parse->dts))
2650 h264parse->dts += *out_dur;
2651 }
2652
2653 static GstFlowReturn
gst_h264_parse_parse_frame(GstBaseParse * parse,GstBaseParseFrame * frame)2654 gst_h264_parse_parse_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
2655 {
2656 GstH264Parse *h264parse;
2657 GstBuffer *buffer;
2658 guint av;
2659
2660 h264parse = GST_H264_PARSE (parse);
2661 buffer = frame->buffer;
2662
2663 gst_h264_parse_update_src_caps (h264parse, NULL);
2664
2665 /* don't mess with timestamps if provided by upstream,
2666 * particularly since our ts not that good they handle seeking etc */
2667 if (h264parse->do_ts) {
2668 gst_h264_parse_get_timestamp (h264parse,
2669 &GST_BUFFER_DTS (buffer), &GST_BUFFER_DURATION (buffer),
2670 h264parse->frame_start);
2671 }
2672
2673 /* We don't want to let baseparse select a duration itself based
2674 * solely on the framerate, as we have more per-frame information
2675 * available */
2676 if (!GST_CLOCK_TIME_IS_VALID (GST_BUFFER_DURATION (buffer))) {
2677 GST_BUFFER_DURATION (buffer) =
2678 gst_h264_parse_get_duration (h264parse, h264parse->frame_start);
2679 }
2680
2681 if (h264parse->keyframe)
2682 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
2683 else
2684 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
2685
2686 if (h264parse->discard_bidirectional && h264parse->bidirectional)
2687 goto discard;
2688
2689 if (h264parse->header)
2690 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_HEADER);
2691 else
2692 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_HEADER);
2693
2694 if (h264parse->discont) {
2695 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DISCONT);
2696 h264parse->discont = FALSE;
2697 }
2698
2699 if (h264parse->marker) {
2700 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_MARKER);
2701 h264parse->marker = FALSE;
2702 } else {
2703 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_MARKER);
2704 }
2705
2706 /* replace with transformed AVC output if applicable */
2707 av = gst_adapter_available (h264parse->frame_out);
2708 if (av) {
2709 GstBuffer *buf;
2710
2711 buf = gst_adapter_take_buffer (h264parse->frame_out, av);
2712 gst_buffer_copy_into (buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
2713 gst_buffer_replace (&frame->out_buffer, buf);
2714 gst_buffer_unref (buf);
2715 }
2716
2717 done:
2718 return GST_FLOW_OK;
2719
2720 discard:
2721 GST_DEBUG_OBJECT (h264parse, "Discarding bidirectional frame");
2722 frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
2723 gst_h264_parse_reset_frame (h264parse);
2724 goto done;
2725 }
2726
2727 /* sends a codec NAL downstream, decorating and transforming as needed.
2728 * No ownership is taken of @nal */
2729 static GstFlowReturn
gst_h264_parse_push_codec_buffer(GstH264Parse * h264parse,GstBuffer * nal,GstBuffer * buffer)2730 gst_h264_parse_push_codec_buffer (GstH264Parse * h264parse,
2731 GstBuffer * nal, GstBuffer * buffer)
2732 {
2733 GstMapInfo map;
2734 GstBuffer *wrapped_nal;
2735
2736 gst_buffer_map (nal, &map, GST_MAP_READ);
2737 wrapped_nal = gst_h264_parse_wrap_nal (h264parse, h264parse->format,
2738 map.data, map.size);
2739 gst_buffer_unmap (nal, &map);
2740
2741 GST_BUFFER_PTS (wrapped_nal) = GST_BUFFER_PTS (buffer);
2742 GST_BUFFER_DTS (wrapped_nal) = GST_BUFFER_DTS (buffer);
2743 GST_BUFFER_DURATION (wrapped_nal) = 0;
2744
2745 return gst_pad_push (GST_BASE_PARSE_SRC_PAD (h264parse), wrapped_nal);
2746 }
2747
2748 static GstEvent *
check_pending_key_unit_event(GstEvent * pending_event,GstSegment * segment,GstClockTime timestamp,guint flags,GstClockTime pending_key_unit_ts)2749 check_pending_key_unit_event (GstEvent * pending_event,
2750 GstSegment * segment, GstClockTime timestamp, guint flags,
2751 GstClockTime pending_key_unit_ts)
2752 {
2753 GstClockTime running_time, stream_time;
2754 gboolean all_headers;
2755 guint count;
2756 GstEvent *event = NULL;
2757
2758 g_return_val_if_fail (segment != NULL, NULL);
2759
2760 if (pending_event == NULL)
2761 goto out;
2762
2763 if (GST_CLOCK_TIME_IS_VALID (pending_key_unit_ts) &&
2764 timestamp == GST_CLOCK_TIME_NONE)
2765 goto out;
2766
2767 running_time = gst_segment_to_running_time (segment,
2768 GST_FORMAT_TIME, timestamp);
2769
2770 GST_INFO ("now %" GST_TIME_FORMAT " wanted %" GST_TIME_FORMAT,
2771 GST_TIME_ARGS (running_time), GST_TIME_ARGS (pending_key_unit_ts));
2772 if (GST_CLOCK_TIME_IS_VALID (pending_key_unit_ts) &&
2773 running_time < pending_key_unit_ts)
2774 goto out;
2775
2776 if (flags & GST_BUFFER_FLAG_DELTA_UNIT) {
2777 GST_DEBUG ("pending force key unit, waiting for keyframe");
2778 goto out;
2779 }
2780
2781 stream_time = gst_segment_to_stream_time (segment,
2782 GST_FORMAT_TIME, timestamp);
2783
2784 if (!gst_video_event_parse_upstream_force_key_unit (pending_event,
2785 NULL, &all_headers, &count)) {
2786 gst_video_event_parse_downstream_force_key_unit (pending_event, NULL,
2787 NULL, NULL, &all_headers, &count);
2788 }
2789
2790 event =
2791 gst_video_event_new_downstream_force_key_unit (timestamp, stream_time,
2792 running_time, all_headers, count);
2793 gst_event_set_seqnum (event, gst_event_get_seqnum (pending_event));
2794
2795 out:
2796 return event;
2797 }
2798
2799 static void
gst_h264_parse_prepare_key_unit(GstH264Parse * parse,GstEvent * event)2800 gst_h264_parse_prepare_key_unit (GstH264Parse * parse, GstEvent * event)
2801 {
2802 GstClockTime running_time;
2803 guint count;
2804 #ifndef GST_DISABLE_GST_DEBUG
2805 gboolean have_sps, have_pps;
2806 gint i;
2807 #endif
2808
2809 parse->pending_key_unit_ts = GST_CLOCK_TIME_NONE;
2810 gst_event_replace (&parse->force_key_unit_event, NULL);
2811
2812 gst_video_event_parse_downstream_force_key_unit (event,
2813 NULL, NULL, &running_time, NULL, &count);
2814
2815 GST_INFO_OBJECT (parse, "pushing downstream force-key-unit event %d "
2816 "%" GST_TIME_FORMAT " count %d", gst_event_get_seqnum (event),
2817 GST_TIME_ARGS (running_time), count);
2818 gst_pad_push_event (GST_BASE_PARSE_SRC_PAD (parse), event);
2819
2820 #ifndef GST_DISABLE_GST_DEBUG
2821 have_sps = have_pps = FALSE;
2822 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
2823 if (parse->sps_nals[i] != NULL) {
2824 have_sps = TRUE;
2825 break;
2826 }
2827 }
2828 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
2829 if (parse->pps_nals[i] != NULL) {
2830 have_pps = TRUE;
2831 break;
2832 }
2833 }
2834
2835 GST_INFO_OBJECT (parse, "preparing key unit, have sps %d have pps %d",
2836 have_sps, have_pps);
2837 #endif
2838
2839 /* set push_codec to TRUE so that pre_push_frame sends SPS/PPS again */
2840 parse->push_codec = TRUE;
2841 }
2842
2843 static gboolean
gst_h264_parse_handle_sps_pps_nals(GstH264Parse * h264parse,GstBuffer * buffer,GstBaseParseFrame * frame)2844 gst_h264_parse_handle_sps_pps_nals (GstH264Parse * h264parse,
2845 GstBuffer * buffer, GstBaseParseFrame * frame)
2846 {
2847 GstBuffer *codec_nal;
2848 gint i;
2849 gboolean send_done = FALSE;
2850
2851 if (h264parse->have_sps_in_frame && h264parse->have_pps_in_frame) {
2852 GST_DEBUG_OBJECT (h264parse, "SPS/PPS exist in frame, will not insert");
2853 return TRUE;
2854 }
2855
2856 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL) {
2857 /* send separate config NAL buffers */
2858 GST_DEBUG_OBJECT (h264parse, "- sending SPS/PPS");
2859 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
2860 if ((codec_nal = h264parse->sps_nals[i])) {
2861 GST_DEBUG_OBJECT (h264parse, "sending SPS nal");
2862 gst_h264_parse_push_codec_buffer (h264parse, codec_nal, buffer);
2863 send_done = TRUE;
2864 }
2865 }
2866 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
2867 if ((codec_nal = h264parse->pps_nals[i])) {
2868 GST_DEBUG_OBJECT (h264parse, "sending PPS nal");
2869 gst_h264_parse_push_codec_buffer (h264parse, codec_nal, buffer);
2870 send_done = TRUE;
2871 }
2872 }
2873 } else {
2874 /* insert config NALs into AU */
2875 GstByteWriter bw;
2876 GstBuffer *new_buf;
2877 const gboolean bs = h264parse->format == GST_H264_PARSE_FORMAT_BYTE;
2878 const gint nls = 4 - h264parse->nal_length_size;
2879 gboolean ok;
2880
2881 gst_byte_writer_init_with_size (&bw, gst_buffer_get_size (buffer), FALSE);
2882 ok = gst_byte_writer_put_buffer (&bw, buffer, 0, h264parse->idr_pos);
2883 GST_DEBUG_OBJECT (h264parse, "- inserting SPS/PPS");
2884 for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++) {
2885 if ((codec_nal = h264parse->sps_nals[i])) {
2886 gsize nal_size = gst_buffer_get_size (codec_nal);
2887 GST_DEBUG_OBJECT (h264parse, "inserting SPS nal");
2888 if (bs) {
2889 ok &= gst_byte_writer_put_uint32_be (&bw, 1);
2890 } else {
2891 ok &= gst_byte_writer_put_uint32_be (&bw, (nal_size << (nls * 8)));
2892 ok &= gst_byte_writer_set_pos (&bw,
2893 gst_byte_writer_get_pos (&bw) - nls);
2894 }
2895
2896 ok &= gst_byte_writer_put_buffer (&bw, codec_nal, 0, nal_size);
2897 send_done = TRUE;
2898 }
2899 }
2900 for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++) {
2901 if ((codec_nal = h264parse->pps_nals[i])) {
2902 gsize nal_size = gst_buffer_get_size (codec_nal);
2903 GST_DEBUG_OBJECT (h264parse, "inserting PPS nal");
2904 if (bs) {
2905 ok &= gst_byte_writer_put_uint32_be (&bw, 1);
2906 } else {
2907 ok &= gst_byte_writer_put_uint32_be (&bw, (nal_size << (nls * 8)));
2908 ok &= gst_byte_writer_set_pos (&bw,
2909 gst_byte_writer_get_pos (&bw) - nls);
2910 }
2911 ok &= gst_byte_writer_put_buffer (&bw, codec_nal, 0, nal_size);
2912 send_done = TRUE;
2913 }
2914 }
2915 ok &= gst_byte_writer_put_buffer (&bw, buffer, h264parse->idr_pos, -1);
2916 /* collect result and push */
2917 new_buf = gst_byte_writer_reset_and_get_buffer (&bw);
2918 gst_buffer_copy_into (new_buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
2919 /* should already be keyframe/IDR, but it may not have been,
2920 * so mark it as such to avoid being discarded by picky decoder */
2921 GST_BUFFER_FLAG_UNSET (new_buf, GST_BUFFER_FLAG_DELTA_UNIT);
2922 gst_buffer_replace (&frame->out_buffer, new_buf);
2923 gst_buffer_unref (new_buf);
2924 /* some result checking seems to make some compilers happy */
2925 if (G_UNLIKELY (!ok)) {
2926 GST_ERROR_OBJECT (h264parse, "failed to insert SPS/PPS");
2927 }
2928 }
2929
2930 return send_done;
2931 }
2932
2933 static GstBuffer *
gst_h264_parse_create_pic_timing_sei(GstH264Parse * h264parse,GstBuffer * buffer)2934 gst_h264_parse_create_pic_timing_sei (GstH264Parse * h264parse,
2935 GstBuffer * buffer)
2936 {
2937 guint num_meta;
2938 const guint8 num_clock_ts_table[9] = {
2939 1, 1, 1, 2, 2, 3, 3, 2, 3
2940 };
2941 guint num_clock_ts;
2942 GstBuffer *out_buf = NULL;
2943 GstMemory *sei_mem;
2944 GArray *msg_array;
2945 gint i, j;
2946 GstH264SEIMessage sei;
2947 GstH264PicTiming *pic_timing;
2948 GstVideoTimeCodeMeta *tc_meta;
2949 gpointer iter = NULL;
2950 guint8 ct_type = GST_H264_CT_TYPE_PROGRESSIVE;
2951
2952 if (!h264parse->update_timecode)
2953 return NULL;
2954
2955 num_meta = gst_buffer_get_n_meta (buffer, GST_VIDEO_TIME_CODE_META_API_TYPE);
2956 if (num_meta == 0)
2957 return NULL;
2958
2959 if (!h264parse->sei_pic_struct_pres_flag || h264parse->pic_timing_sei_pos < 0) {
2960 GST_ELEMENT_WARNING (h264parse, STREAM, NOT_IMPLEMENTED, (NULL),
2961 ("timecode update was requested but VUI doesn't support timecode"));
2962 return NULL;
2963 }
2964
2965 g_assert (h264parse->sei_pic_struct <=
2966 GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING);
2967
2968 num_clock_ts = num_clock_ts_table[h264parse->sei_pic_struct];
2969
2970 if (num_meta > num_clock_ts) {
2971 GST_LOG_OBJECT (h264parse,
2972 "The number of timecode meta %d is superior to required %d",
2973 num_meta, num_clock_ts);
2974
2975 return NULL;
2976 }
2977
2978 GST_LOG_OBJECT (h264parse,
2979 "The number of timecode meta %d is compatible", num_meta);
2980
2981 memset (&sei, 0, sizeof (GstH264SEIMessage));
2982 sei.payloadType = GST_H264_SEI_PIC_TIMING;
2983 memcpy (&sei.payload.pic_timing,
2984 &h264parse->pic_timing_sei, sizeof (GstH264PicTiming));
2985
2986 pic_timing = &sei.payload.pic_timing;
2987
2988 switch (h264parse->sei_pic_struct) {
2989 case GST_H264_SEI_PIC_STRUCT_FRAME:
2990 case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING:
2991 case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING:
2992 ct_type = GST_H264_CT_TYPE_PROGRESSIVE;
2993 break;
2994 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM:
2995 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP:
2996 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP:
2997 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM:
2998 ct_type = GST_H264_CT_TYPE_INTERLACED;
2999 break;
3000 default:
3001 ct_type = GST_H264_CT_TYPE_UNKNOWN;
3002 break;
3003 }
3004
3005 i = 0;
3006 while ((tc_meta =
3007 (GstVideoTimeCodeMeta *) gst_buffer_iterate_meta_filtered (buffer,
3008 &iter, GST_VIDEO_TIME_CODE_META_API_TYPE))) {
3009 GstH264ClockTimestamp *tim = &pic_timing->clock_timestamp[i];
3010 GstVideoTimeCode *tc = &tc_meta->tc;
3011
3012 pic_timing->clock_timestamp_flag[i] = 1;
3013 tim->ct_type = ct_type;
3014 tim->nuit_field_based_flag = 1;
3015 tim->counting_type = 0;
3016
3017 if ((tc->config.flags & GST_VIDEO_TIME_CODE_FLAGS_DROP_FRAME)
3018 == GST_VIDEO_TIME_CODE_FLAGS_DROP_FRAME)
3019 tim->counting_type = 4;
3020
3021 tim->discontinuity_flag = 0;
3022 tim->cnt_dropped_flag = 0;
3023 tim->n_frames = tc->frames;
3024
3025 tim->hours_value = tc->hours;
3026 tim->minutes_value = tc->minutes;
3027 tim->seconds_value = tc->seconds;
3028
3029 tim->full_timestamp_flag =
3030 tim->seconds_flag = tim->minutes_flag = tim->hours_flag = 0;
3031
3032 if (tc->hours > 0)
3033 tim->full_timestamp_flag = 1;
3034 else if (tc->minutes > 0)
3035 tim->seconds_flag = tim->minutes_flag = 1;
3036 else if (tc->seconds > 0)
3037 tim->seconds_flag = 1;
3038
3039 GST_LOG_OBJECT (h264parse,
3040 "New time code value %02u:%02u:%02u:%02u",
3041 tim->hours_value, tim->minutes_value, tim->seconds_value,
3042 tim->n_frames);
3043
3044 i++;
3045 }
3046
3047 for (j = i; j < 3; j++)
3048 pic_timing->clock_timestamp_flag[j] = 0;
3049
3050 msg_array = g_array_new (FALSE, FALSE, sizeof (GstH264SEIMessage));
3051 g_array_set_clear_func (msg_array, (GDestroyNotify) gst_h264_sei_clear);
3052
3053 g_array_append_val (msg_array, sei);
3054 if (h264parse->format == GST_H264_PARSE_FORMAT_BYTE) {
3055 sei_mem = gst_h264_create_sei_memory (3, msg_array);
3056 } else {
3057 sei_mem = gst_h264_create_sei_memory_avc (h264parse->nal_length_size,
3058 msg_array);
3059 }
3060 g_array_unref (msg_array);
3061
3062 if (!sei_mem) {
3063 GST_WARNING_OBJECT (h264parse, "Cannot create Picture Timing SEI memory");
3064 return NULL;
3065 }
3066
3067 out_buf = gst_buffer_new ();
3068 gst_buffer_copy_into (out_buf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
3069
3070 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL) {
3071 gst_buffer_append_memory (out_buf, sei_mem);
3072 } else {
3073 gsize mem_size;
3074
3075 mem_size = gst_memory_get_sizes (sei_mem, NULL, NULL);
3076
3077 /* copy every data except for the SEI */
3078 if (h264parse->pic_timing_sei_pos > 0) {
3079 gst_buffer_copy_into (out_buf, buffer, GST_BUFFER_COPY_MEMORY, 0,
3080 h264parse->pic_timing_sei_pos);
3081 }
3082
3083 /* insert new SEI */
3084 gst_buffer_append_memory (out_buf, sei_mem);
3085
3086 if (gst_buffer_get_size (buffer) >
3087 h264parse->pic_timing_sei_pos + h264parse->pic_timing_sei_size) {
3088 gst_buffer_copy_into (out_buf, buffer, GST_BUFFER_COPY_MEMORY,
3089 h264parse->pic_timing_sei_pos + h264parse->pic_timing_sei_size, -1);
3090 }
3091
3092 if (h264parse->idr_pos >= 0) {
3093 h264parse->idr_pos += mem_size;
3094 h264parse->idr_pos -= h264parse->pic_timing_sei_size;
3095 }
3096 }
3097
3098 return out_buf;
3099 }
3100
3101 static GstFlowReturn
gst_h264_parse_pre_push_frame(GstBaseParse * parse,GstBaseParseFrame * frame)3102 gst_h264_parse_pre_push_frame (GstBaseParse * parse, GstBaseParseFrame * frame)
3103 {
3104 GstH264Parse *h264parse;
3105 GstBuffer *buffer;
3106 GstBuffer *new_buf;
3107 GstEvent *event;
3108 GstBuffer *parse_buffer = NULL;
3109 gboolean is_interlaced = FALSE;
3110
3111 h264parse = GST_H264_PARSE (parse);
3112
3113 if (h264parse->first_frame) {
3114 GstTagList *taglist;
3115 GstCaps *caps;
3116
3117 /* codec tag */
3118 caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (parse));
3119 if (caps == NULL) {
3120 if (GST_PAD_IS_FLUSHING (GST_BASE_PARSE_SRC_PAD (h264parse))) {
3121 GST_INFO_OBJECT (h264parse, "Src pad is flushing");
3122 return GST_FLOW_FLUSHING;
3123 } else {
3124 GST_INFO_OBJECT (h264parse, "Src pad is not negotiated!");
3125 return GST_FLOW_NOT_NEGOTIATED;
3126 }
3127 }
3128
3129 taglist = gst_tag_list_new_empty ();
3130 gst_pb_utils_add_codec_description_to_tag_list (taglist,
3131 GST_TAG_VIDEO_CODEC, caps);
3132 gst_caps_unref (caps);
3133
3134 gst_base_parse_merge_tags (parse, taglist, GST_TAG_MERGE_REPLACE);
3135 gst_tag_list_unref (taglist);
3136
3137 /* also signals the end of first-frame processing */
3138 h264parse->first_frame = FALSE;
3139 }
3140
3141 #ifdef OHOS_OPT_COMPAT
3142 /**
3143 * ohos.ext.compat.0034
3144 * Set not to insert the aud header,
3145 * otherwise it will affect the subsequent recognition of multiple slices
3146 */
3147 h264parse->aud_insert = FALSE;
3148 #endif
3149
3150 /* In case of byte-stream, insert au delimiter by default
3151 * if it doesn't exist */
3152 if (h264parse->aud_insert && !h264parse->have_aud_in_frame &&
3153 h264parse->format == GST_H264_PARSE_FORMAT_BYTE) {
3154 GST_DEBUG_OBJECT (h264parse, "Inserting AUD into the stream.");
3155 if (h264parse->align == GST_H264_PARSE_ALIGN_AU) {
3156 GstMemory *mem =
3157 gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, (guint8 *) au_delim,
3158 sizeof (au_delim), 0, sizeof (au_delim), NULL, NULL);
3159
3160 frame->out_buffer = gst_buffer_copy (frame->buffer);
3161 gst_buffer_prepend_memory (frame->out_buffer, mem);
3162 if (h264parse->idr_pos >= 0)
3163 h264parse->idr_pos += sizeof (au_delim);
3164
3165 buffer = frame->out_buffer;
3166 } else {
3167 GstBuffer *aud_buffer = gst_buffer_new_allocate (NULL, 2, NULL);
3168 gst_buffer_fill (aud_buffer, 0, (guint8 *) (au_delim + 4), 2);
3169
3170 buffer = frame->buffer;
3171 gst_h264_parse_push_codec_buffer (h264parse, aud_buffer, buffer);
3172 gst_buffer_unref (aud_buffer);
3173 }
3174 } else {
3175 buffer = frame->buffer;
3176 }
3177 h264parse->aud_insert = FALSE;
3178
3179 if ((event = check_pending_key_unit_event (h264parse->force_key_unit_event,
3180 &parse->segment, GST_BUFFER_TIMESTAMP (buffer),
3181 GST_BUFFER_FLAGS (buffer), h264parse->pending_key_unit_ts))) {
3182 gst_h264_parse_prepare_key_unit (h264parse, event);
3183 }
3184
3185 /* handle timecode */
3186 new_buf = gst_h264_parse_create_pic_timing_sei (h264parse, buffer);
3187 if (new_buf) {
3188 if (frame->out_buffer)
3189 gst_buffer_unref (frame->out_buffer);
3190
3191 buffer = frame->out_buffer = new_buf;
3192 }
3193
3194 /* periodic SPS/PPS sending */
3195 if (h264parse->interval > 0 || h264parse->push_codec) {
3196 GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer);
3197 guint64 diff;
3198 gboolean initial_frame = FALSE;
3199
3200 /* init */
3201 if (!GST_CLOCK_TIME_IS_VALID (h264parse->last_report)) {
3202 h264parse->last_report = timestamp;
3203 initial_frame = TRUE;
3204 }
3205
3206 if (h264parse->idr_pos >= 0) {
3207 GST_LOG_OBJECT (h264parse, "IDR nal at offset %d", h264parse->idr_pos);
3208
3209 if (timestamp > h264parse->last_report)
3210 diff = timestamp - h264parse->last_report;
3211 else
3212 diff = 0;
3213
3214 GST_LOG_OBJECT (h264parse,
3215 "now %" GST_TIME_FORMAT ", last SPS/PPS %" GST_TIME_FORMAT,
3216 GST_TIME_ARGS (timestamp), GST_TIME_ARGS (h264parse->last_report));
3217
3218 GST_DEBUG_OBJECT (h264parse,
3219 "interval since last SPS/PPS %" GST_TIME_FORMAT,
3220 GST_TIME_ARGS (diff));
3221
3222 if (GST_TIME_AS_SECONDS (diff) >= h264parse->interval ||
3223 initial_frame || h264parse->push_codec) {
3224 GstClockTime new_ts;
3225
3226 /* avoid overwriting a perfectly fine timestamp */
3227 new_ts = GST_CLOCK_TIME_IS_VALID (timestamp) ? timestamp :
3228 h264parse->last_report;
3229
3230 if (gst_h264_parse_handle_sps_pps_nals (h264parse, buffer, frame)) {
3231 h264parse->last_report = new_ts;
3232 }
3233 }
3234 /* we pushed whatever we had */
3235 h264parse->push_codec = FALSE;
3236 h264parse->have_sps = FALSE;
3237 h264parse->have_pps = FALSE;
3238 h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
3239 }
3240 } else if (h264parse->interval == -1) {
3241 if (h264parse->idr_pos >= 0) {
3242 GST_LOG_OBJECT (h264parse, "IDR nal at offset %d", h264parse->idr_pos);
3243
3244 gst_h264_parse_handle_sps_pps_nals (h264parse, buffer, frame);
3245
3246 /* we pushed whatever we had */
3247 h264parse->push_codec = FALSE;
3248 h264parse->have_sps = FALSE;
3249 h264parse->have_pps = FALSE;
3250 h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
3251 }
3252 }
3253
3254 /* Fixme: setting passthrough mode causing multiple issues:
3255 * For nal aligned multiresoluton streams, passthrough mode make h264parse
3256 * unable to advertise the new resolutions. Also causing issues while
3257 * parsing MVC streams when it has two layers.
3258 * Disabing passthourgh mode for now */
3259 #if 0
3260 /* If SPS/PPS and a keyframe have been parsed, and we're not converting,
3261 * we might switch to passthrough mode now on the basis that we've seen
3262 * the SEI packets and know optional caps params (such as multiview).
3263 * This is an efficiency optimisation that relies on stream properties
3264 * remaining uniform in practice. */
3265 if (h264parse->can_passthrough) {
3266 if (h264parse->keyframe && h264parse->have_sps && h264parse->have_pps) {
3267 GST_LOG_OBJECT (parse, "Switching to passthrough mode");
3268 gst_base_parse_set_passthrough (parse, TRUE);
3269 }
3270 }
3271 #endif
3272
3273 if (frame->out_buffer) {
3274 parse_buffer = frame->out_buffer =
3275 gst_buffer_make_writable (frame->out_buffer);
3276 } else {
3277 parse_buffer = frame->buffer = gst_buffer_make_writable (frame->buffer);
3278 }
3279
3280 if (!gst_buffer_get_video_time_code_meta (parse_buffer)) {
3281 guint i = 0;
3282
3283 for (i = 0; i < 3 && h264parse->num_clock_timestamp; i++) {
3284 GstH264ClockTimestamp *tim =
3285 &h264parse->pic_timing_sei.clock_timestamp[i];
3286 gint field_count = -1;
3287 guint n_frames;
3288 GstVideoTimeCodeFlags flags = 0;
3289
3290 if (!h264parse->pic_timing_sei.clock_timestamp_flag[i])
3291 continue;
3292
3293 /* Table D-1 */
3294 switch (h264parse->sei_pic_struct) {
3295 case GST_H264_SEI_PIC_STRUCT_FRAME:
3296 case GST_H264_SEI_PIC_STRUCT_TOP_FIELD:
3297 case GST_H264_SEI_PIC_STRUCT_BOTTOM_FIELD:
3298 field_count = h264parse->sei_pic_struct;
3299 break;
3300 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM:
3301 field_count = i + 1;
3302 break;
3303 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP:
3304 field_count = 2 - i;
3305 break;
3306 case GST_H264_SEI_PIC_STRUCT_TOP_BOTTOM_TOP:
3307 field_count = i % 2 ? 2 : 1;
3308 break;
3309 case GST_H264_SEI_PIC_STRUCT_BOTTOM_TOP_BOTTOM:
3310 field_count = i % 2 ? 1 : 2;
3311 break;
3312 case GST_H264_SEI_PIC_STRUCT_FRAME_DOUBLING:
3313 case GST_H264_SEI_PIC_STRUCT_FRAME_TRIPLING:
3314 field_count = 0;
3315 break;
3316 }
3317
3318 if (field_count == -1) {
3319 GST_WARNING_OBJECT (parse,
3320 "failed to determine field count for timecode");
3321 field_count = 0;
3322 }
3323
3324 /* dropping of the two lowest (value 0 and 1) n_frames
3325 * counts when seconds_value is equal to 0 and
3326 * minutes_value is not an integer multiple of 10 */
3327 if (tim->counting_type == 4)
3328 flags |= GST_VIDEO_TIME_CODE_FLAGS_DROP_FRAME;
3329
3330 if (tim->ct_type == GST_H264_CT_TYPE_INTERLACED) {
3331 flags |= GST_VIDEO_TIME_CODE_FLAGS_INTERLACED;
3332 is_interlaced = TRUE;
3333 }
3334
3335 n_frames =
3336 gst_util_uint64_scale_int (tim->n_frames, 1,
3337 2 - tim->nuit_field_based_flag);
3338
3339 GST_LOG_OBJECT (h264parse,
3340 "Add time code meta %02u:%02u:%02u:%02u",
3341 tim->hours_value, tim->minutes_value, tim->seconds_value, n_frames);
3342
3343 gst_buffer_add_video_time_code_meta_full (parse_buffer,
3344 h264parse->parsed_fps_n,
3345 h264parse->parsed_fps_d,
3346 NULL,
3347 flags,
3348 tim->hours_flag ? tim->hours_value : 0,
3349 tim->minutes_flag ? tim->minutes_value : 0,
3350 tim->seconds_flag ? tim->seconds_value : 0, n_frames, field_count);
3351 }
3352
3353 h264parse->num_clock_timestamp = 0;
3354 }
3355
3356 if (is_interlaced) {
3357 GST_BUFFER_FLAG_SET (parse_buffer, GST_VIDEO_BUFFER_FLAG_INTERLACED);
3358 if (h264parse->sei_pic_struct == GST_H264_SEI_PIC_STRUCT_TOP_FIELD)
3359 GST_BUFFER_FLAG_SET (parse_buffer, GST_VIDEO_BUFFER_FLAG_TFF);
3360 }
3361
3362 gst_video_push_user_data ((GstElement *) h264parse, &h264parse->user_data,
3363 parse_buffer);
3364
3365 gst_h264_parse_reset_frame (h264parse);
3366
3367 return GST_FLOW_OK;
3368 }
3369
3370 static gboolean
gst_h264_parse_set_caps(GstBaseParse * parse,GstCaps * caps)3371 gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps)
3372 {
3373 GstH264Parse *h264parse;
3374 GstStructure *str;
3375 const GValue *codec_data_value;
3376 GstBuffer *codec_data = NULL;
3377 gsize size;
3378 guint format, align, off;
3379 GstH264NalUnit nalu;
3380 GstH264ParserResult parseres;
3381 GstCaps *old_caps;
3382
3383 h264parse = GST_H264_PARSE (parse);
3384
3385 /* reset */
3386 h264parse->push_codec = FALSE;
3387
3388 old_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SINK_PAD (parse));
3389 if (old_caps) {
3390 if (!gst_caps_is_equal (old_caps, caps))
3391 gst_h264_parse_reset_stream_info (h264parse);
3392 gst_caps_unref (old_caps);
3393 }
3394
3395 str = gst_caps_get_structure (caps, 0);
3396
3397 /* accept upstream info if provided */
3398 gst_structure_get_int (str, "width", &h264parse->width);
3399 gst_structure_get_int (str, "height", &h264parse->height);
3400 gst_structure_get_fraction (str, "framerate", &h264parse->fps_num,
3401 &h264parse->fps_den);
3402 gst_structure_get_fraction (str, "pixel-aspect-ratio",
3403 &h264parse->upstream_par_n, &h264parse->upstream_par_d);
3404
3405 /* get upstream format and align from caps */
3406 gst_h264_parse_format_from_caps (caps, &format, &align);
3407
3408 codec_data_value = gst_structure_get_value (str, "codec_data");
3409
3410 /* fix up caps without stream-format for max. backwards compatibility */
3411 if (format == GST_H264_PARSE_FORMAT_NONE) {
3412 /* codec_data implies avc */
3413 if (codec_data_value != NULL) {
3414 GST_ERROR ("video/x-h264 caps with codec_data but no stream-format=avc");
3415 format = GST_H264_PARSE_FORMAT_AVC;
3416 } else {
3417 /* otherwise assume bytestream input */
3418 GST_ERROR ("video/x-h264 caps without codec_data or stream-format");
3419 format = GST_H264_PARSE_FORMAT_BYTE;
3420 }
3421 }
3422
3423 /* avc caps sanity checks */
3424 if (format == GST_H264_PARSE_FORMAT_AVC) {
3425 /* AVC requires codec_data, AVC3 might have one and/or SPS/PPS inline */
3426 if (codec_data_value == NULL)
3427 goto avc_caps_codec_data_missing;
3428
3429 /* AVC implies alignment=au, everything else is not allowed */
3430 if (align == GST_H264_PARSE_ALIGN_NONE)
3431 align = GST_H264_PARSE_ALIGN_AU;
3432 else if (align != GST_H264_PARSE_ALIGN_AU)
3433 goto avc_caps_wrong_alignment;
3434 }
3435
3436 /* bytestream caps sanity checks */
3437 if (format == GST_H264_PARSE_FORMAT_BYTE) {
3438 /* should have SPS/PSS in-band (and/or oob in streamheader field) */
3439 if (codec_data_value != NULL)
3440 goto bytestream_caps_with_codec_data;
3441 }
3442
3443 /* packetized video has codec_data (required for AVC, optional for AVC3) */
3444 if (codec_data_value != NULL) {
3445 GstMapInfo map;
3446 guint8 *data;
3447 guint num_sps, num_pps;
3448 #ifndef GST_DISABLE_GST_DEBUG
3449 guint profile;
3450 #endif
3451 gint i;
3452
3453 GST_DEBUG_OBJECT (h264parse, "have packetized h264");
3454 /* make note for optional split processing */
3455 h264parse->packetized = TRUE;
3456
3457 /* codec_data field should hold a buffer */
3458 if (!GST_VALUE_HOLDS_BUFFER (codec_data_value))
3459 goto avc_caps_codec_data_wrong_type;
3460
3461 codec_data = gst_value_get_buffer (codec_data_value);
3462 if (!codec_data)
3463 goto avc_caps_codec_data_missing;
3464 gst_buffer_map (codec_data, &map, GST_MAP_READ);
3465 data = map.data;
3466 size = map.size;
3467
3468 /* parse the avcC data */
3469 if (size < 7) { /* when numSPS==0 and numPPS==0, length is 7 bytes */
3470 gst_buffer_unmap (codec_data, &map);
3471 goto avcc_too_small;
3472 }
3473 /* parse the version, this must be 1 */
3474 if (data[0] != 1) {
3475 gst_buffer_unmap (codec_data, &map);
3476 goto wrong_version;
3477 }
3478 #ifndef GST_DISABLE_GST_DEBUG
3479 /* AVCProfileIndication */
3480 /* profile_compat */
3481 /* AVCLevelIndication */
3482 profile = (data[1] << 16) | (data[2] << 8) | data[3];
3483 GST_DEBUG_OBJECT (h264parse, "profile %06x", profile);
3484 #endif
3485
3486 /* 6 bits reserved | 2 bits lengthSizeMinusOne */
3487 /* this is the number of bytes in front of the NAL units to mark their
3488 * length */
3489 h264parse->nal_length_size = (data[4] & 0x03) + 1;
3490 GST_DEBUG_OBJECT (h264parse, "nal length size %u",
3491 h264parse->nal_length_size);
3492
3493 num_sps = data[5] & 0x1f;
3494 off = 6;
3495 for (i = 0; i < num_sps; i++) {
3496 parseres = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
3497 data, off, size, 2, &nalu);
3498 if (parseres != GST_H264_PARSER_OK) {
3499 gst_buffer_unmap (codec_data, &map);
3500 goto avcc_too_small;
3501 }
3502
3503 gst_h264_parse_process_nal (h264parse, &nalu);
3504 off = nalu.offset + nalu.size;
3505 }
3506
3507 if (off >= size) {
3508 gst_buffer_unmap (codec_data, &map);
3509 goto avcc_too_small;
3510 }
3511 num_pps = data[off];
3512 off++;
3513
3514 for (i = 0; i < num_pps; i++) {
3515 parseres = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
3516 data, off, size, 2, &nalu);
3517 if (parseres != GST_H264_PARSER_OK) {
3518 gst_buffer_unmap (codec_data, &map);
3519 goto avcc_too_small;
3520 }
3521
3522 gst_h264_parse_process_nal (h264parse, &nalu);
3523 off = nalu.offset + nalu.size;
3524 }
3525
3526 gst_buffer_unmap (codec_data, &map);
3527
3528 gst_buffer_replace (&h264parse->codec_data_in, codec_data);
3529
3530 /* don't confuse codec_data with inband sps/pps */
3531 h264parse->have_sps_in_frame = FALSE;
3532 h264parse->have_pps_in_frame = FALSE;
3533 } else if (format == GST_H264_PARSE_FORMAT_BYTE) {
3534 GST_DEBUG_OBJECT (h264parse, "have bytestream h264");
3535 /* nothing to pre-process */
3536 h264parse->packetized = FALSE;
3537 /* we have 4 sync bytes */
3538 h264parse->nal_length_size = 4;
3539 } else {
3540 /* probably AVC3 without codec_data field, anything to do here? */
3541 }
3542
3543 {
3544 GstCaps *in_caps;
3545
3546 /* prefer input type determined above */
3547 in_caps = gst_caps_new_simple ("video/x-h264",
3548 "parsed", G_TYPE_BOOLEAN, TRUE,
3549 "stream-format", G_TYPE_STRING,
3550 gst_h264_parse_get_string (h264parse, TRUE, format),
3551 "alignment", G_TYPE_STRING,
3552 gst_h264_parse_get_string (h264parse, FALSE, align), NULL);
3553 /* negotiate with downstream, sets ->format and ->align */
3554 gst_h264_parse_negotiate (h264parse, format, in_caps);
3555 gst_caps_unref (in_caps);
3556 }
3557
3558 if (format == h264parse->format && align == h264parse->align) {
3559 /* we did parse codec-data and might supplement src caps */
3560 gst_h264_parse_update_src_caps (h264parse, caps);
3561 } else if (format == GST_H264_PARSE_FORMAT_AVC
3562 || format == GST_H264_PARSE_FORMAT_AVC3) {
3563 /* if input != output, and input is avc, must split before anything else */
3564 /* arrange to insert codec-data in-stream if needed.
3565 * src caps are only arranged for later on */
3566 h264parse->push_codec = TRUE;
3567 h264parse->have_sps = FALSE;
3568 h264parse->have_pps = FALSE;
3569 if (h264parse->align == GST_H264_PARSE_ALIGN_NAL)
3570 h264parse->split_packetized = TRUE;
3571 h264parse->packetized = TRUE;
3572 }
3573
3574 h264parse->in_align = align;
3575
3576 return TRUE;
3577
3578 /* ERRORS */
3579 avc_caps_codec_data_wrong_type:
3580 {
3581 GST_WARNING_OBJECT (parse, "H.264 AVC caps, codec_data field not a buffer");
3582 goto refuse_caps;
3583 }
3584 avc_caps_codec_data_missing:
3585 {
3586 GST_WARNING_OBJECT (parse, "H.264 AVC caps, but no codec_data");
3587 goto refuse_caps;
3588 }
3589 avc_caps_wrong_alignment:
3590 {
3591 GST_WARNING_OBJECT (parse, "H.264 AVC caps with NAL alignment, must be AU");
3592 goto refuse_caps;
3593 }
3594 bytestream_caps_with_codec_data:
3595 {
3596 GST_WARNING_OBJECT (parse, "H.264 bytestream caps with codec_data is not "
3597 "expected, send SPS/PPS in-band with data or in streamheader field");
3598 goto refuse_caps;
3599 }
3600 avcc_too_small:
3601 {
3602 GST_DEBUG_OBJECT (h264parse, "avcC size %" G_GSIZE_FORMAT " < 8", size);
3603 goto refuse_caps;
3604 }
3605 wrong_version:
3606 {
3607 GST_DEBUG_OBJECT (h264parse, "wrong avcC version");
3608 goto refuse_caps;
3609 }
3610 refuse_caps:
3611 {
3612 GST_WARNING_OBJECT (h264parse, "refused caps %" GST_PTR_FORMAT, caps);
3613 return FALSE;
3614 }
3615 }
3616
3617 static void
remove_fields(GstCaps * caps,gboolean all)3618 remove_fields (GstCaps * caps, gboolean all)
3619 {
3620 guint i, n;
3621
3622 n = gst_caps_get_size (caps);
3623 for (i = 0; i < n; i++) {
3624 GstStructure *s = gst_caps_get_structure (caps, i);
3625
3626 if (all) {
3627 gst_structure_remove_field (s, "alignment");
3628 gst_structure_remove_field (s, "stream-format");
3629 }
3630 gst_structure_remove_field (s, "parsed");
3631 }
3632 }
3633
3634 static GstCaps *
gst_h264_parse_get_caps(GstBaseParse * parse,GstCaps * filter)3635 gst_h264_parse_get_caps (GstBaseParse * parse, GstCaps * filter)
3636 {
3637 GstCaps *peercaps, *templ;
3638 GstCaps *res, *tmp, *pcopy;
3639
3640 templ = gst_pad_get_pad_template_caps (GST_BASE_PARSE_SINK_PAD (parse));
3641 if (filter) {
3642 GstCaps *fcopy = gst_caps_copy (filter);
3643 /* Remove the fields we convert */
3644 remove_fields (fcopy, TRUE);
3645 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), fcopy);
3646 gst_caps_unref (fcopy);
3647 } else
3648 peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (parse), NULL);
3649
3650 pcopy = gst_caps_copy (peercaps);
3651 remove_fields (pcopy, TRUE);
3652
3653 res = gst_caps_intersect_full (pcopy, templ, GST_CAPS_INTERSECT_FIRST);
3654 gst_caps_unref (pcopy);
3655 gst_caps_unref (templ);
3656
3657 if (filter) {
3658 GstCaps *tmp = gst_caps_intersect_full (res, filter,
3659 GST_CAPS_INTERSECT_FIRST);
3660 gst_caps_unref (res);
3661 res = tmp;
3662 }
3663
3664 /* Try if we can put the downstream caps first */
3665 pcopy = gst_caps_copy (peercaps);
3666 remove_fields (pcopy, FALSE);
3667 tmp = gst_caps_intersect_full (pcopy, res, GST_CAPS_INTERSECT_FIRST);
3668 gst_caps_unref (pcopy);
3669 if (!gst_caps_is_empty (tmp))
3670 res = gst_caps_merge (tmp, res);
3671 else
3672 gst_caps_unref (tmp);
3673
3674 gst_caps_unref (peercaps);
3675 return res;
3676 }
3677
3678 static gboolean
gst_h264_parse_event(GstBaseParse * parse,GstEvent * event)3679 gst_h264_parse_event (GstBaseParse * parse, GstEvent * event)
3680 {
3681 gboolean res;
3682 GstH264Parse *h264parse = GST_H264_PARSE (parse);
3683
3684 switch (GST_EVENT_TYPE (event)) {
3685 case GST_EVENT_CUSTOM_DOWNSTREAM:
3686 {
3687 GstClockTime timestamp, stream_time, running_time;
3688 gboolean all_headers;
3689 guint count;
3690
3691 if (gst_video_event_is_force_key_unit (event)) {
3692 gst_video_event_parse_downstream_force_key_unit (event,
3693 ×tamp, &stream_time, &running_time, &all_headers, &count);
3694
3695 GST_INFO_OBJECT (h264parse,
3696 "received downstream force key unit event, "
3697 "seqnum %d running_time %" GST_TIME_FORMAT
3698 " all_headers %d count %d", gst_event_get_seqnum (event),
3699 GST_TIME_ARGS (running_time), all_headers, count);
3700 if (h264parse->force_key_unit_event) {
3701 GST_INFO_OBJECT (h264parse, "ignoring force key unit event "
3702 "as one is already queued");
3703 } else {
3704 h264parse->pending_key_unit_ts = running_time;
3705 gst_event_replace (&h264parse->force_key_unit_event, event);
3706 }
3707 gst_event_unref (event);
3708 res = TRUE;
3709 } else {
3710 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3711 break;
3712 }
3713 break;
3714 }
3715 case GST_EVENT_FLUSH_STOP:
3716 case GST_EVENT_SEGMENT_DONE:
3717 h264parse->dts = GST_CLOCK_TIME_NONE;
3718 h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
3719 h264parse->push_codec = TRUE;
3720
3721 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3722 break;
3723 case GST_EVENT_SEGMENT:
3724 {
3725 const GstSegment *segment;
3726
3727 gst_event_parse_segment (event, &segment);
3728 /* don't try to mess with more subtle cases (e.g. seek) */
3729 if (segment->format == GST_FORMAT_TIME &&
3730 (segment->start != 0 || segment->rate != 1.0
3731 || segment->applied_rate != 1.0))
3732 h264parse->do_ts = FALSE;
3733
3734 if (segment->flags & GST_SEEK_FLAG_TRICKMODE_FORWARD_PREDICTED) {
3735 GST_DEBUG_OBJECT (h264parse, "Will discard bidirectional frames");
3736 h264parse->discard_bidirectional = TRUE;
3737 }
3738
3739
3740 h264parse->last_report = GST_CLOCK_TIME_NONE;
3741
3742 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3743 break;
3744 }
3745 default:
3746 res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event);
3747 break;
3748 }
3749 return res;
3750 }
3751
3752 static gboolean
gst_h264_parse_src_event(GstBaseParse * parse,GstEvent * event)3753 gst_h264_parse_src_event (GstBaseParse * parse, GstEvent * event)
3754 {
3755 gboolean res;
3756 GstH264Parse *h264parse = GST_H264_PARSE (parse);
3757
3758 switch (GST_EVENT_TYPE (event)) {
3759 case GST_EVENT_CUSTOM_UPSTREAM:
3760 {
3761 GstClockTime running_time;
3762 gboolean all_headers;
3763 guint count;
3764
3765 if (gst_video_event_is_force_key_unit (event)) {
3766 gst_video_event_parse_upstream_force_key_unit (event,
3767 &running_time, &all_headers, &count);
3768
3769 GST_INFO_OBJECT (h264parse, "received upstream force-key-unit event, "
3770 "seqnum %d running_time %" GST_TIME_FORMAT
3771 " all_headers %d count %d", gst_event_get_seqnum (event),
3772 GST_TIME_ARGS (running_time), all_headers, count);
3773
3774 if (all_headers) {
3775 h264parse->pending_key_unit_ts = running_time;
3776 gst_event_replace (&h264parse->force_key_unit_event, event);
3777 }
3778 }
3779 res = GST_BASE_PARSE_CLASS (parent_class)->src_event (parse, event);
3780 break;
3781 }
3782 default:
3783 res = GST_BASE_PARSE_CLASS (parent_class)->src_event (parse, event);
3784 break;
3785 }
3786
3787 return res;
3788 }
3789
3790 static void
gst_h264_parse_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)3791 gst_h264_parse_set_property (GObject * object, guint prop_id,
3792 const GValue * value, GParamSpec * pspec)
3793 {
3794 GstH264Parse *parse;
3795
3796 parse = GST_H264_PARSE (object);
3797
3798 switch (prop_id) {
3799 case PROP_CONFIG_INTERVAL:
3800 parse->interval = g_value_get_int (value);
3801 break;
3802 case PROP_UPDATE_TIMECODE:
3803 parse->update_timecode = g_value_get_boolean (value);
3804 break;
3805 default:
3806 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
3807 break;
3808 }
3809 }
3810
3811 static void
gst_h264_parse_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)3812 gst_h264_parse_get_property (GObject * object, guint prop_id,
3813 GValue * value, GParamSpec * pspec)
3814 {
3815 GstH264Parse *parse;
3816
3817 parse = GST_H264_PARSE (object);
3818
3819 switch (prop_id) {
3820 case PROP_CONFIG_INTERVAL:
3821 g_value_set_int (value, parse->interval);
3822 break;
3823 case PROP_UPDATE_TIMECODE:
3824 g_value_set_boolean (value, parse->update_timecode);
3825 break;
3826 default:
3827 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
3828 break;
3829 }
3830 }
3831