ext/ttml/gstttmlparse.c

/* GStreamer
 * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
 * Copyright (C) 2004 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
 * Copyright (C) 2006 Tim-Philipp Müller <tim centricular net>
 * Copyright (C) <2015> British Broadcasting Corporation <dash@rd.bbc.co.uk>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:element-ttmlparse
 * @title: ttmlparse
 *
 * Parses timed text subtitle files described using Timed Text Markup Language
 * (TTML). Currently, only the EBU-TT-D profile of TTML, designed for
 * distribution of subtitles over IP, is supported.
 *
 * The parser outputs a #GstBuffer for each scene in the input TTML file, a
 * scene being a period of time during which a static set of subtitles should
 * be visible. The parser places each text element within a scene into its own
 * #GstMemory within the scene's buffer, and attaches metadata to the buffer
 * describing the styling and layout associated with all the contained text
 * elements. A downstream renderer element uses this information to correctly
 * render the text on top of video frames.
 *
 * ## Example launch lines
 * |[
 * gst-launch-1.0 filesrc location=<media file location> ! video/quicktime ! qtdemux name=q ttmlrender name=r q. ! queue ! h264parse ! avdec_h264 ! autovideoconvert ! r.video_sink filesrc location=<subtitle file location> blocksize=16777216 ! queue ! ttmlparse ! r.text_sink r. ! ximagesink q. ! queue ! aacparse ! avdec_aac ! audioconvert ! alsasink
 * ]| Parse and render TTML subtitles contained in a single XML file over an
 * MP4 stream containing H.264 video and AAC audio.
 *
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <glib.h>

#include "gstttmlelements.h"
#include "gstttmlparse.h"
#include "ttmlparse.h"

GST_DEBUG_CATEGORY (ttmlparse_debug);
#define GST_CAT_DEFAULT ttmlparse_debug


#define DEFAULT_ENCODING   NULL

static GstStaticPadTemplate sink_templ = GST_STATIC_PAD_TEMPLATE ("sink",
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS ("application/ttml+xml")
    );

static GstStaticPadTemplate src_templ = GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS ("text/x-raw(meta:GstSubtitleMeta)")
    );

static gboolean gst_ttml_parse_src_event (GstPad * pad, GstObject * parent,
    GstEvent * event);
static gboolean gst_ttml_parse_src_query (GstPad * pad, GstObject * parent,
    GstQuery * query);
static gboolean gst_ttml_parse_sink_event (GstPad * pad, GstObject * parent,
    GstEvent * event);

static GstStateChangeReturn gst_ttml_parse_change_state (GstElement * element,
    GstStateChange transition);

static GstFlowReturn gst_ttml_parse_chain (GstPad * sinkpad, GstObject * parent,
    GstBuffer * buf);
static gboolean gst_element_ttmlparse_init (GstPlugin * plugin);

#define gst_ttml_parse_parent_class parent_class
G_DEFINE_TYPE (GstTtmlParse, gst_ttml_parse, GST_TYPE_ELEMENT);
GST_ELEMENT_REGISTER_DEFINE_CUSTOM (ttmlparse, gst_element_ttmlparse_init);

static void
gst_ttml_parse_dispose (GObject * object)
{
  GstTtmlParse *ttmlparse = GST_TTML_PARSE (object);

  GST_DEBUG_OBJECT (ttmlparse, "cleaning up subtitle parser");

  g_free (ttmlparse->encoding);
  ttmlparse->encoding = NULL;

  g_free (ttmlparse->detected_encoding);
  ttmlparse->detected_encoding = NULL;

  if (ttmlparse->adapter) {
    g_object_unref (ttmlparse->adapter);
    ttmlparse->adapter = NULL;
  }

  if (ttmlparse->textbuf) {
    g_string_free (ttmlparse->textbuf, TRUE);
    ttmlparse->textbuf = NULL;
  }

  GST_CALL_PARENT (G_OBJECT_CLASS, dispose, (object));
}

static void
gst_ttml_parse_class_init (GstTtmlParseClass * klass)
{
  GObjectClass *object_class = G_OBJECT_CLASS (klass);
  GstElementClass *element_class = GST_ELEMENT_CLASS (klass);

  object_class->dispose = gst_ttml_parse_dispose;

  gst_element_class_add_pad_template (element_class,
      gst_static_pad_template_get (&sink_templ));
  gst_element_class_add_pad_template (element_class,
      gst_static_pad_template_get (&src_templ));
  gst_element_class_set_static_metadata (element_class,
      "TTML subtitle parser", "Codec/Parser/Subtitle",
      "Parses TTML subtitle files",
      "GStreamer maintainers <gstreamer-devel@lists.sourceforge.net>, "
      "Chris Bass <dash@rd.bbc.co.uk>");

  element_class->change_state = gst_ttml_parse_change_state;
}

static void
gst_ttml_parse_init (GstTtmlParse * ttmlparse)
{
  ttmlparse->sinkpad = gst_pad_new_from_static_template (&sink_templ, "sink");
  gst_pad_set_chain_function (ttmlparse->sinkpad,
      GST_DEBUG_FUNCPTR (gst_ttml_parse_chain));
  gst_pad_set_event_function (ttmlparse->sinkpad,
      GST_DEBUG_FUNCPTR (gst_ttml_parse_sink_event));
  gst_element_add_pad (GST_ELEMENT (ttmlparse), ttmlparse->sinkpad);

  ttmlparse->srcpad = gst_pad_new_from_static_template (&src_templ, "src");
  gst_pad_set_event_function (ttmlparse->srcpad,
      GST_DEBUG_FUNCPTR (gst_ttml_parse_src_event));
  gst_pad_set_query_function (ttmlparse->srcpad,
      GST_DEBUG_FUNCPTR (gst_ttml_parse_src_query));
  gst_element_add_pad (GST_ELEMENT (ttmlparse), ttmlparse->srcpad);

  ttmlparse->textbuf = g_string_new (NULL);
  gst_segment_init (&ttmlparse->segment, GST_FORMAT_TIME);
  ttmlparse->need_segment = TRUE;
  ttmlparse->encoding = g_strdup (DEFAULT_ENCODING);
  ttmlparse->detected_encoding = NULL;
  ttmlparse->adapter = gst_adapter_new ();
}

/*
 * Source pad functions.
 */
static gboolean
gst_ttml_parse_src_query (GstPad * pad, GstObject * parent, GstQuery * query)
{
  GstTtmlParse *self = GST_TTML_PARSE (parent);
  gboolean ret = FALSE;

  GST_DEBUG ("Handling %s query", GST_QUERY_TYPE_NAME (query));

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_POSITION:{
      GstFormat fmt;

      gst_query_parse_position (query, &fmt, NULL);
      if (fmt != GST_FORMAT_TIME) {
        ret = gst_pad_peer_query (self->sinkpad, query);
      } else {
        ret = TRUE;
        gst_query_set_position (query, GST_FORMAT_TIME, self->segment.position);
      }
      break;
    }
    case GST_QUERY_SEEKING:
    {
      GstFormat fmt;
      gboolean seekable = FALSE;

      ret = TRUE;

      gst_query_parse_seeking (query, &fmt, NULL, NULL, NULL);
      if (fmt == GST_FORMAT_TIME) {
        GstQuery *peerquery = gst_query_new_seeking (GST_FORMAT_BYTES);

        seekable = gst_pad_peer_query (self->sinkpad, peerquery);
        if (seekable)
          gst_query_parse_seeking (peerquery, NULL, &seekable, NULL, NULL);
        gst_query_unref (peerquery);
      }

      gst_query_set_seeking (query, fmt, seekable, seekable ? 0 : -1, -1);
      break;
    }
    default:
      ret = gst_pad_query_default (pad, parent, query);
      break;
  }

  return ret;
}

static gboolean
gst_ttml_parse_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
{
  GstTtmlParse *self = GST_TTML_PARSE (parent);
  gboolean ret = FALSE;

  GST_DEBUG ("Handling %s event", GST_EVENT_TYPE_NAME (event));

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_SEEK:
    {
      GstFormat format;
      GstSeekFlags flags;
      GstSeekType start_type, stop_type;
      gint64 start, stop;
      gdouble rate;
      gboolean update;

      gst_event_parse_seek (event, &rate, &format, &flags,
          &start_type, &start, &stop_type, &stop);

      if (format != GST_FORMAT_TIME) {
        GST_WARNING_OBJECT (self, "we only support seeking in TIME format");
        gst_event_unref (event);
        goto beach;
      }

      /* Convert that seek to a seeking in bytes at position 0,
         FIXME: could use an index */
      ret = gst_pad_push_event (self->sinkpad,
          gst_event_new_seek (rate, GST_FORMAT_BYTES, flags,
              GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_NONE, 0));

      if (ret) {
        /* Apply the seek to our segment */
        gst_segment_do_seek (&self->segment, rate, format, flags,
            start_type, start, stop_type, stop, &update);

        GST_DEBUG_OBJECT (self, "segment after seek: %" GST_SEGMENT_FORMAT,
            &self->segment);

        self->need_segment = TRUE;
      } else {
        GST_WARNING_OBJECT (self, "seek to 0 bytes failed");
      }

      gst_event_unref (event);
      break;
    }
    default:
      ret = gst_pad_event_default (pad, parent, event);
      break;
  }

beach:
  return ret;
}

static gchar *
gst_convert_to_utf8 (const gchar * str, gsize len, const gchar * encoding,
    gsize * consumed, GError ** err)
{
  gchar *ret = NULL;

  *consumed = 0;
  /* The char cast is necessary in glib < 2.24 */
  ret =
      g_convert_with_fallback (str, len, "UTF-8", encoding, (char *) "*",
      consumed, NULL, err);
  if (ret == NULL)
    return ret;

  /* + 3 to skip UTF-8 BOM if it was added */
  len = strlen (ret);
  if (len >= 3 && (guint8) ret[0] == 0xEF && (guint8) ret[1] == 0xBB
      && (guint8) ret[2] == 0xBF)
    memmove (ret, ret + 3, len + 1 - 3);

  return ret;
}

static gchar *
detect_encoding (const gchar * str, gsize len)
{
  if (len >= 3 && (guint8) str[0] == 0xEF && (guint8) str[1] == 0xBB
      && (guint8) str[2] == 0xBF)
    return g_strdup ("UTF-8");

  if (len >= 2 && (guint8) str[0] == 0xFE && (guint8) str[1] == 0xFF)
    return g_strdup ("UTF-16BE");

  if (len >= 2 && (guint8) str[0] == 0xFF && (guint8) str[1] == 0xFE)
    return g_strdup ("UTF-16LE");

  if (len >= 4 && (guint8) str[0] == 0x00 && (guint8) str[1] == 0x00
      && (guint8) str[2] == 0xFE && (guint8) str[3] == 0xFF)
    return g_strdup ("UTF-32BE");

  if (len >= 4 && (guint8) str[0] == 0xFF && (guint8) str[1] == 0xFE
      && (guint8) str[2] == 0x00 && (guint8) str[3] == 0x00)
    return g_strdup ("UTF-32LE");

  return NULL;
}

static gchar *
convert_encoding (GstTtmlParse * self, const gchar * str, gsize len,
    gsize * consumed)
{
  const gchar *encoding;
  GError *err = NULL;
  gchar *ret = NULL;

  *consumed = 0;

  /* First try any detected encoding */
  if (self->detected_encoding) {
    ret =
        gst_convert_to_utf8 (str, len, self->detected_encoding, consumed, &err);

    if (!err)
      return ret;

    GST_WARNING_OBJECT (self, "could not convert string from '%s' to UTF-8: %s",
        self->detected_encoding, err->message);
    g_free (self->detected_encoding);
    self->detected_encoding = NULL;
    g_error_free (err);
  }

  /* Otherwise check if it's UTF8 */
  if (self->valid_utf8) {
    if (g_utf8_validate (str, len, NULL)) {
      GST_LOG_OBJECT (self, "valid UTF-8, no conversion needed");
      *consumed = len;
      return g_strndup (str, len);
    }
    GST_INFO_OBJECT (self, "invalid UTF-8!");
    self->valid_utf8 = FALSE;
  }

  /* Else try fallback */
  encoding = self->encoding;
  if (encoding == NULL || *encoding == '\0') {
    /* if local encoding is UTF-8 and no encoding specified
     * via the environment variable, assume ISO-8859-15 */
    if (g_get_charset (&encoding)) {
      encoding = "ISO-8859-15";
    }
  }

  ret = gst_convert_to_utf8 (str, len, encoding, consumed, &err);

  if (err) {
    GST_WARNING_OBJECT (self, "could not convert string from '%s' to UTF-8: %s",
        encoding, err->message);
    g_error_free (err);

    /* invalid input encoding, fall back to ISO-8859-15 (always succeeds) */
    ret = gst_convert_to_utf8 (str, len, "ISO-8859-15", consumed, NULL);
  }

  GST_LOG_OBJECT (self,
      "successfully converted %" G_GSIZE_FORMAT " characters from %s to UTF-8"
      "%s", len, encoding, (err) ? " , using ISO-8859-15 as fallback" : "");

  return ret;
}

static GstCaps *
gst_ttml_parse_get_src_caps (GstTtmlParse * self)
{
  GstCaps *caps;
  GstCapsFeatures *features = gst_caps_features_new ("meta:GstSubtitleMeta",
      NULL);

  caps = gst_caps_new_empty_simple ("text/x-raw");
  gst_caps_set_features (caps, 0, features);
  return caps;
}

static void
feed_textbuf (GstTtmlParse * self, GstBuffer * buf)
{
  gboolean discont;
  gsize consumed;
  gchar *input = NULL;
  const guint8 *data;
  gsize avail;

  discont = GST_BUFFER_IS_DISCONT (buf);

  if (GST_BUFFER_OFFSET_IS_VALID (buf) &&
      GST_BUFFER_OFFSET (buf) != self->offset) {
    self->offset = GST_BUFFER_OFFSET (buf);
    discont = TRUE;
  }

  if (discont) {
    GST_INFO ("discontinuity");
    /* flush the parser state */
    g_string_truncate (self->textbuf, 0);
    gst_adapter_clear (self->adapter);
    /* we could set a flag to make sure that the next buffer we push out also
     * has the DISCONT flag set, but there's no point really given that it's
     * subtitles which are discontinuous by nature. */
  }

  self->offset += gst_buffer_get_size (buf);

  gst_adapter_push (self->adapter, buf);

  avail = gst_adapter_available (self->adapter);
  data = gst_adapter_map (self->adapter, avail);
  input = convert_encoding (self, (const gchar *) data, avail, &consumed);

  if (input && consumed > 0) {
    if (!self->textbuf)
      self->textbuf = g_string_new (input);
    else
      self->textbuf = g_string_append (self->textbuf, input);

    gst_adapter_unmap (self->adapter);
    gst_adapter_flush (self->adapter, consumed);
  } else {
    gst_adapter_unmap (self->adapter);
  }

  g_free (input);
}

static GstFlowReturn
handle_buffer (GstTtmlParse * self, GstBuffer * buf)
{
  GstFlowReturn ret = GST_FLOW_OK;
  GstCaps *caps = NULL;
  GList *subtitle_list = NULL;
  GList *iter;
  GstClockTime begin = GST_BUFFER_PTS (buf);
  GstClockTime duration = GST_BUFFER_DURATION (buf);
  guint consumed;

  if (self->first_buffer) {
    GstMapInfo map;

    gst_buffer_map (buf, &map, GST_MAP_READ);
    self->detected_encoding = detect_encoding ((gchar *) map.data, map.size);
    gst_buffer_unmap (buf, &map);
    self->first_buffer = FALSE;
  }

  feed_textbuf (self, buf);

  if (!(caps = gst_ttml_parse_get_src_caps (self)))
    return GST_FLOW_EOS;
  gst_caps_unref (caps);

  /* Push newsegment if needed */
  if (self->need_segment) {
    GST_LOG_OBJECT (self, "pushing newsegment event with %" GST_SEGMENT_FORMAT,
        &self->segment);

    gst_pad_push_event (self->srcpad, gst_event_new_segment (&self->segment));
    self->need_segment = FALSE;
  }

  do {
    consumed = ttml_parse (self->textbuf->str, begin, duration, &subtitle_list);

    if (!consumed) {
      GST_DEBUG_OBJECT (self, "need more data");
      return ret;
    }

    self->textbuf = g_string_erase (self->textbuf, 0, consumed);

    for (iter = subtitle_list; iter; iter = g_list_next (iter)) {
      GstBuffer *op_buffer = GST_BUFFER (iter->data);
      self->segment.position = GST_BUFFER_PTS (op_buffer);

      ret = gst_pad_push (self->srcpad, op_buffer);

      if (ret != GST_FLOW_OK) {
        GST_DEBUG_OBJECT (self, "flow: %s", gst_flow_get_name (ret));
        break;
      }
    }

    g_list_free (subtitle_list);
  } while (TRUE);

  return ret;
}

static GstFlowReturn
gst_ttml_parse_chain (GstPad * sinkpad, GstObject * parent, GstBuffer * buf)
{
  GstTtmlParse *self = GST_TTML_PARSE (parent);
  return handle_buffer (self, buf);
}

static gboolean
gst_ttml_parse_sink_event (GstPad * pad, GstObject * parent, GstEvent * event)
{
  GstTtmlParse *self = GST_TTML_PARSE (parent);
  gboolean ret = FALSE;

  GST_DEBUG ("Handling %s event", GST_EVENT_TYPE_NAME (event));

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_SEGMENT:
    {
      const GstSegment *s;
      gst_event_parse_segment (event, &s);
      if (s->format == GST_FORMAT_TIME)
        gst_event_copy_segment (event, &self->segment);
      GST_DEBUG_OBJECT (self, "newsegment (%s)",
          gst_format_get_name (self->segment.format));

      /* if not time format, we'll either start with a 0 timestamp anyway or
       * it's following a seek in which case we'll have saved the requested
       * seek segment and don't want to overwrite it (remember that on a seek
       * we always just seek back to the start in BYTES format and just throw
       * away all text that's before the requested position; if the subtitles
       * come from an upstream demuxer, it won't be able to handle our BYTES
       * seek request and instead send us a newsegment from the seek request
       * it received via its video pads instead, so all is fine then too) */
      ret = TRUE;
      self->need_segment = TRUE;
      gst_event_unref (event);
      break;
    }
    case GST_EVENT_CAPS:
    {
      GstCaps *caps;
      gst_event_unref (event);

      caps = gst_ttml_parse_get_src_caps (self);
      event = gst_event_new_caps (caps);
      gst_caps_unref (caps);

      ret = gst_pad_push_event (self->srcpad, event);
      break;
    }
    default:
      ret = gst_pad_event_default (pad, parent, event);
      break;
  }

  return ret;
}

static GstStateChangeReturn
gst_ttml_parse_change_state (GstElement * element, GstStateChange transition)
{
  GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
  GstTtmlParse *self = GST_TTML_PARSE (element);

  switch (transition) {
    case GST_STATE_CHANGE_READY_TO_PAUSED:
      /* format detection will init the parser state */
      self->offset = 0;
      self->valid_utf8 = TRUE;
      self->first_buffer = TRUE;
      g_free (self->detected_encoding);
      self->detected_encoding = NULL;
      g_string_truncate (self->textbuf, 0);
      gst_adapter_clear (self->adapter);
      break;
    default:
      break;
  }

  ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
  if (ret == GST_STATE_CHANGE_FAILURE)
    return ret;

  switch (transition) {
    case GST_STATE_CHANGE_PAUSED_TO_READY:
      break;
    default:
      break;
  }

  return ret;
}

static gboolean
gst_element_ttmlparse_init (GstPlugin * plugin)
{
  guint rank = GST_RANK_NONE;

  ttml_element_init (plugin);

  GST_DEBUG_CATEGORY_INIT (ttmlparse_debug, "ttmlparse", 0, "TTML parser");

  /* We don't want this autoplugged by default yet for now */
  if (g_getenv ("GST_TTML_AUTOPLUG")) {
    GST_INFO_OBJECT (plugin, "Registering ttml elements with primary rank.");
    rank = GST_RANK_PRIMARY;
  }

  return gst_element_register (plugin, "ttmlparse", rank, GST_TYPE_TTML_PARSE);
}