1 /* GStreamer 2 * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu> 3 * 4 * This library is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU Library General Public 6 * License as published by the Free Software Foundation; either 7 * version 2 of the License, or (at your option) any later version. 8 * 9 * This library is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 * Library General Public License for more details. 13 * 14 * You should have received a copy of the GNU Library General Public 15 * License along with this library; if not, write to the 16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, 17 * Boston, MA 02110-1301, USA. 18 */ 19 20 21 #ifndef __GST_QTDEMUX_H__ 22 #define __GST_QTDEMUX_H__ 23 24 #include <gst/gst.h> 25 #include <gst/base/gstadapter.h> 26 #include <gst/base/gstflowcombiner.h> 27 #include <gst/base/gstbytereader.h> 28 #include <gst/video/video.h> 29 #include "gstisoff.h" 30 31 G_BEGIN_DECLS 32 33 #define GST_TYPE_QTDEMUX \ 34 (gst_qtdemux_get_type()) 35 #define GST_QTDEMUX(obj) \ 36 (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_QTDEMUX,GstQTDemux)) 37 #define GST_QTDEMUX_CLASS(klass) \ 38 (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_QTDEMUX,GstQTDemuxClass)) 39 #define GST_IS_QTDEMUX(obj) \ 40 (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_QTDEMUX)) 41 #define GST_IS_QTDEMUX_CLASS(klass) \ 42 (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_QTDEMUX)) 43 44 #define GST_QTDEMUX_CAST(obj) ((GstQTDemux *)(obj)) 45 46 /* qtdemux produces these for atoms it cannot parse */ 47 #define GST_QT_DEMUX_PRIVATE_TAG "private-qt-tag" 48 #define GST_QT_DEMUX_CLASSIFICATION_TAG "classification" 49 50 typedef struct _GstQTDemux GstQTDemux; 51 typedef struct _GstQTDemuxClass GstQTDemuxClass; 52 typedef struct _QtDemuxStream QtDemuxStream; 53 typedef struct _QtDemuxSample QtDemuxSample; 54 typedef struct _QtDemuxSegment QtDemuxSegment; 55 typedef struct _QtDemuxRandomAccessEntry QtDemuxRandomAccessEntry; 56 typedef struct _QtDemuxStreamStsdEntry QtDemuxStreamStsdEntry; 57 58 typedef GstBuffer * (*QtDemuxProcessFunc)(GstQTDemux * qtdemux, QtDemuxStream * stream, GstBuffer * buf); 59 60 enum QtDemuxState 61 { 62 QTDEMUX_STATE_INITIAL, /* Initial state (haven't got the header yet) */ 63 QTDEMUX_STATE_HEADER, /* Parsing the header */ 64 QTDEMUX_STATE_MOVIE, /* Parsing/Playing the media data */ 65 QTDEMUX_STATE_BUFFER_MDAT /* Buffering the mdat atom */ 66 }; 67 68 struct _GstQTDemux { 69 GstElement element; 70 71 /* Global state */ 72 enum QtDemuxState state; 73 74 /* static sink pad */ 75 GstPad *sinkpad; 76 77 /* TRUE if pull-based */ 78 gboolean pullbased; 79 80 gchar *redirect_location; 81 82 /* Protect pad exposing from flush event */ 83 GMutex expose_lock; 84 85 /* list of QtDemuxStream */ 86 GPtrArray *active_streams; 87 GPtrArray *old_streams; 88 89 gint n_video_streams; 90 gint n_audio_streams; 91 gint n_sub_streams; 92 93 GstFlowCombiner *flowcombiner; 94 95 /* Incoming stream group-id to set on downstream STREAM_START events. 96 * If upstream doesn't contain one, a global one will be generated */ 97 gboolean have_group_id; 98 guint group_id; 99 100 guint major_brand; 101 GstBuffer *comp_brands; 102 103 /* [moov] header. 104 * FIXME : This is discarded just after it's created. Just move it 105 * to a temporary variable ? */ 106 GNode *moov_node; 107 108 /* FIXME : This is never freed. It is only assigned once. memleak ? */ 109 GNode *moov_node_compressed; 110 111 /* Set to TRUE when the [moov] header has been fully parsed */ 112 gboolean got_moov; 113 114 /* Global timescale for the incoming stream. Use the QTTIME macros 115 * to convert values to/from GstClockTime */ 116 guint32 timescale; 117 118 /* Global duration (in global timescale). Use QTTIME macros to get GstClockTime */ 119 guint64 duration; 120 121 /* Total size of header atoms. Used to calculate fallback overall bitrate */ 122 guint header_size; 123 124 GstTagList *tag_list; 125 126 /* configured playback region */ 127 GstSegment segment; 128 129 /* State for key_units trickmode */ 130 GstClockTime trickmode_interval; 131 132 /* PUSH-BASED only: If the initial segment event, or a segment consequence of 133 * a seek or incoming TIME segment from upstream needs to be pushed. This 134 * variable is used instead of pushing the event directly because at that 135 * point we may not have yet emitted the srcpads. */ 136 gboolean need_segment; 137 138 guint32 segment_seqnum; 139 140 /* flag to indicate that we're working with a smoothstreaming fragment 141 * Mss doesn't have 'moov' or any information about the streams format, 142 * requiring qtdemux to expose and create the streams */ 143 gboolean mss_mode; 144 145 /* Set to TRUE if the incoming stream is either a MSS stream or 146 * a Fragmented MP4 (containing the [mvex] atom in the header) */ 147 gboolean fragmented; 148 149 /* PULL-BASED only : If TRUE there is a pending seek */ 150 gboolean fragmented_seek_pending; 151 152 /* PULL-BASED : offset of first [moof] or of fragment to seek to 153 * PUSH-BASED : offset of latest [moof] */ 154 guint64 moof_offset; 155 156 /* MSS streams have a single media that is unspecified at the atoms, so 157 * upstream provides it at the caps */ 158 GstCaps *media_caps; 159 160 /* Set to TRUE when all streams have been exposed */ 161 gboolean exposed; 162 163 gint64 chapters_track_id; 164 165 /* protection support */ 166 GPtrArray *protection_system_ids; /* Holds identifiers of all content protection systems for all tracks */ 167 GQueue protection_event_queue; /* holds copy of upstream protection events */ 168 guint64 cenc_aux_info_offset; 169 guint8 *cenc_aux_info_sizes; 170 guint32 cenc_aux_sample_count; 171 gchar *preferred_protection_system_id; 172 173 /* Whether the parent bin is streams-aware, meaning we can 174 * add/remove streams at any point in time */ 175 gboolean streams_aware; 176 177 /* 178 * ALL VARIABLES BELOW ARE ONLY USED IN PUSH-BASED MODE 179 */ 180 GstAdapter *adapter; 181 guint neededbytes; 182 guint todrop; 183 /* Used to store data if [mdat] is before the headers */ 184 GstBuffer *mdatbuffer; 185 /* Amount of bytes left to read in the current [mdat] */ 186 guint64 mdatleft, mdatsize; 187 188 /* When restoring the mdat to the adapter, this buffer stores any 189 * trailing data that was after the last atom parsed as it has to be 190 * restored later along with the correct offset. Used in fragmented 191 * scenario where mdat/moof are one after the other in any order. 192 * 193 * Check https://bugzilla.gnome.org/show_bug.cgi?id=710623 */ 194 GstBuffer *restoredata_buffer; 195 guint64 restoredata_offset; 196 197 /* The current offset in bytes from upstream. 198 * Note: While it makes complete sense when we are PULL-BASED (pulling 199 * in BYTES from upstream) and PUSH-BASED with a BYTE SEGMENT (receiving 200 * buffers with actual offsets), it is undefined in PUSH-BASED with a 201 * TIME SEGMENT */ 202 guint64 offset; 203 204 /* offset of the mdat atom */ 205 guint64 mdatoffset; 206 /* Offset of the first mdat */ 207 guint64 first_mdat; 208 /* offset of last [moov] seen */ 209 guint64 last_moov_offset; 210 211 /* If TRUE, qtdemux received upstream newsegment in TIME format 212 * which likely means that upstream is driving the pipeline (such as 213 * adaptive demuxers or dlna sources) */ 214 gboolean upstream_format_is_time; 215 216 /* Seqnum of the seek event sent upstream. Will be used to 217 * detect incoming FLUSH events corresponding to that */ 218 guint32 offset_seek_seqnum; 219 220 /* UPSTREAM BYTE: Requested upstream byte seek offset. 221 * Currently it is only used to check if an incoming BYTE SEGMENT 222 * corresponds to a seek event that was sent upstream */ 223 gint64 seek_offset; 224 225 /* UPSTREAM BYTE: Requested start/stop TIME values from 226 * downstream. 227 * Used to set on the downstream segment once the corresponding upstream 228 * BYTE SEEK has succeeded */ 229 gint64 push_seek_start; 230 gint64 push_seek_stop; 231 232 #if 0 233 /* gst index support */ 234 GstIndex *element_index; 235 gint index_id; 236 #endif 237 238 /* Whether upstream is seekable in BYTES */ 239 gboolean upstream_seekable; 240 /* UPSTREAM BYTE: Size of upstream content. 241 * Note : This is only computed once ! If upstream grows in the meantime 242 * it will not be updated */ 243 gint64 upstream_size; 244 245 /* UPSTREAM TIME : Contains the PTS (if any) of the 246 * buffer that contains a [moof] header. Will be used to establish 247 * the actual PTS of the samples contained within that fragment. */ 248 guint64 fragment_start; 249 /* UPSTREAM TIME : The offset in bytes of the [moof] 250 * header start. 251 * Note : This is not computed from the GST_BUFFER_OFFSET field */ 252 guint64 fragment_start_offset; 253 254 /* These two fields are used to perform an implicit seek when a fragmented 255 * file whose first tfdt is not zero. This way if the first fragment starts 256 * at 1 hour, the user does not have to wait 1 hour or perform a manual seek 257 * for the image to move and the sound to play. 258 * 259 * This implicit seek is only done if the first parsed fragment has a non-zero 260 * decode base time and a seek has not been received previously, hence these 261 * fields. */ 262 gboolean received_seek; 263 gboolean first_moof_already_parsed; 264 }; 265 266 struct _GstQTDemuxClass { 267 GstElementClass parent_class; 268 }; 269 270 GType gst_qtdemux_get_type (void); 271 272 struct _QtDemuxStreamStsdEntry 273 { 274 GstCaps *caps; 275 guint32 fourcc; 276 gboolean sparse; 277 278 /* video info */ 279 gint width; 280 gint height; 281 gint par_w; 282 gint par_h; 283 /* Numerator/denominator framerate */ 284 gint fps_n; 285 gint fps_d; 286 GstVideoColorimetry colorimetry; 287 guint16 bits_per_sample; 288 guint16 color_table_id; 289 GstMemory *rgb8_palette; 290 guint interlace_mode; 291 guint field_order; 292 293 /* audio info */ 294 gdouble rate; 295 gint n_channels; 296 guint samples_per_packet; 297 guint samples_per_frame; 298 guint bytes_per_packet; 299 guint bytes_per_sample; 300 guint bytes_per_frame; 301 guint compression; 302 303 /* if we use chunks or samples */ 304 gboolean sampled; 305 guint padding; 306 307 }; 308 309 struct _QtDemuxSample 310 { 311 guint32 size; 312 gint32 pts_offset; /* Add this value to timestamp to get the pts */ 313 guint64 offset; 314 guint64 timestamp; /* DTS In mov time */ 315 guint32 duration; /* In mov time */ 316 gboolean keyframe; /* TRUE when this packet is a keyframe */ 317 }; 318 319 struct _QtDemuxStream 320 { 321 GstPad *pad; 322 323 GstQTDemux *demux; 324 gchar *stream_id; 325 326 QtDemuxStreamStsdEntry *stsd_entries; 327 guint stsd_entries_length; 328 guint cur_stsd_entry_index; 329 330 /* stream type */ 331 guint32 subtype; 332 333 gboolean new_caps; /* If TRUE, caps need to be generated (by 334 * calling _configure_stream()) This happens 335 * for MSS and fragmented streams */ 336 337 gboolean new_stream; /* signals that a stream_start is required */ 338 gboolean on_keyframe; /* if this stream last pushed buffer was a 339 * keyframe. This is important to identify 340 * where to stop pushing buffers after a 341 * segment stop time */ 342 343 /* if the stream has a redirect URI in its headers, we store it here */ 344 gchar *redirect_uri; 345 346 /* track id */ 347 guint track_id; 348 349 /* duration/scale */ 350 guint64 duration; /* in timescale units */ 351 guint32 timescale; 352 353 /* language */ 354 gchar lang_id[4]; /* ISO 639-2T language code */ 355 356 /* our samples */ 357 guint32 n_samples; 358 QtDemuxSample *samples; 359 gboolean all_keyframe; /* TRUE when all samples are keyframes (no stss) */ 360 guint32 n_samples_moof; /* sample count in a moof */ 361 guint64 duration_moof; /* duration in timescale of a moof, used for figure out 362 * the framerate of fragmented format stream */ 363 guint64 duration_last_moof; 364 365 guint32 offset_in_sample; /* Offset in the current sample, used for 366 * streams which have got exceedingly big 367 * sample size (such as 24s of raw audio). 368 * Only used when max_buffer_size is non-NULL */ 369 guint32 min_buffer_size; /* Minimum allowed size for output buffers. 370 * Currently only set for raw audio streams*/ 371 guint32 max_buffer_size; /* Maximum allowed size for output buffers. 372 * Currently only set for raw audio streams*/ 373 374 /* video info */ 375 /* aspect ratio */ 376 gint display_width; 377 gint display_height; 378 379 /* allocation */ 380 gboolean use_allocator; 381 GstAllocator *allocator; 382 GstAllocationParams params; 383 384 gsize alignment; 385 386 /* when a discontinuity is pending */ 387 gboolean discont; 388 389 /* list of buffers to push first */ 390 GSList *buffers; 391 392 /* if we need to clip this buffer. This is only needed for uncompressed 393 * data */ 394 gboolean need_clip; 395 396 /* If the buffer needs some custom processing, e.g. subtitles, pass them 397 * through this function */ 398 QtDemuxProcessFunc process_func; 399 400 /* buffer needs potentially be split, e.g. CEA608 subtitles */ 401 gboolean need_split; 402 403 /* current position */ 404 guint32 segment_index; 405 guint32 sample_index; 406 GstClockTime time_position; /* in gst time */ 407 guint64 accumulated_base; 408 409 /* the Gst segment we are processing out, used for clipping */ 410 GstSegment segment; 411 412 /* quicktime segments */ 413 guint32 n_segments; 414 QtDemuxSegment *segments; 415 gboolean dummy_segment; 416 guint32 from_sample; 417 guint32 to_sample; 418 419 gboolean sent_eos; 420 GstTagList *stream_tags; 421 gboolean send_global_tags; 422 423 GstEvent *pending_event; 424 425 GstByteReader stco; 426 GstByteReader stsz; 427 GstByteReader stsc; 428 GstByteReader stts; 429 GstByteReader stss; 430 GstByteReader stps; 431 GstByteReader ctts; 432 433 gboolean chunks_are_samples; /* TRUE means treat chunks as samples */ 434 gint64 stbl_index; 435 /* stco */ 436 guint co_size; 437 GstByteReader co_chunk; 438 guint32 first_chunk; 439 guint32 current_chunk; 440 guint32 last_chunk; 441 guint32 samples_per_chunk; 442 guint32 stsd_sample_description_id; 443 guint32 stco_sample_index; 444 /* stsz */ 445 guint32 sample_size; /* 0 means variable sizes are stored in stsz */ 446 /* stsc */ 447 guint32 stsc_index; 448 guint32 n_samples_per_chunk; 449 guint32 stsc_chunk_index; 450 guint32 stsc_sample_index; 451 guint64 chunk_offset; 452 /* stts */ 453 guint32 stts_index; 454 guint32 stts_samples; 455 guint32 n_sample_times; 456 guint32 stts_sample_index; 457 guint64 stts_time; 458 guint32 stts_duration; 459 /* stss */ 460 gboolean stss_present; 461 guint32 n_sample_syncs; 462 guint32 stss_index; 463 /* stps */ 464 gboolean stps_present; 465 guint32 n_sample_partial_syncs; 466 guint32 stps_index; 467 QtDemuxRandomAccessEntry *ra_entries; 468 guint n_ra_entries; 469 470 const QtDemuxRandomAccessEntry *pending_seek; 471 472 /* ctts */ 473 gboolean ctts_present; 474 guint32 n_composition_times; 475 guint32 ctts_index; 476 guint32 ctts_sample_index; 477 guint32 ctts_count; 478 gint32 ctts_soffset; 479 480 /* cslg composition_to_dts_shift or based on the smallest negative 481 * composition time offset. 482 * 483 * This is unsigned because only negative composition time offsets / 484 * positive composition_to_dts_shift matter here. In all other cases, 485 * DTS/PTS can be inferred directly without ending up with PTS>DTS. 486 * 487 * See 14496-12 6.4 488 */ 489 guint64 cslg_shift; 490 491 /* fragmented */ 492 gboolean parsed_trex; 493 guint32 def_sample_description_index; /* index is 1-based */ 494 guint32 def_sample_duration; 495 guint32 def_sample_size; 496 guint32 def_sample_flags; 497 498 gboolean disabled; 499 500 /* stereoscopic video streams */ 501 GstVideoMultiviewMode multiview_mode; 502 GstVideoMultiviewFlags multiview_flags; 503 504 /* protected streams */ 505 gboolean protected; 506 guint32 protection_scheme_type; 507 guint32 protection_scheme_version; 508 gpointer protection_scheme_info; /* specific to the protection scheme */ 509 GQueue protection_scheme_event_queue; 510 511 /* KEY_UNITS trickmode with an interval */ 512 GstClockTime last_keyframe_dts; 513 514 gint ref_count; /* atomic */ 515 #ifdef OHOS_OPT_PERFORMANCE // ohos.opt.performance.0001: demux push first audio/video frame 516 gboolean has_push_first_frame; 517 #endif 518 }; 519 520 G_END_DECLS 521 522 #endif /* __GST_QTDEMUX_H__ */ 523