ext/closedcaption/gstccextractor.c

/*
 * GStreamer
 * Copyright (C) 2018 Edward Hervey <edward@centricular.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:element-ccextractor
 * @title: ccextractor
 * @short_description: Extract GstVideoCaptionMeta from input stream
 *
 * Note: This element must be added after a pipeline's decoder, otherwise closed captions may
 * be extracted out of order.
 *
 */

#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif

#include <gst/gst.h>
#include <gst/video/video.h>
#include <string.h>

#include "gstccextractor.h"

GST_DEBUG_CATEGORY_STATIC (gst_cc_extractor_debug);
#define GST_CAT_DEFAULT gst_cc_extractor_debug

enum
{
  PROP_0,
  PROP_REMOVE_CAPTION_META,
};

static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS_ANY);

static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS_ANY);

static GstStaticPadTemplate captiontemplate =
    GST_STATIC_PAD_TEMPLATE ("caption",
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS
    ("closedcaption/x-cea-608,format={ (string) raw, (string) s334-1a}; "
        "closedcaption/x-cea-708,format={ (string) cc_data, (string) cdp }"));

#define parent_class gst_cc_extractor_parent_class
G_DEFINE_TYPE (GstCCExtractor, gst_cc_extractor, GST_TYPE_ELEMENT);
GST_ELEMENT_REGISTER_DEFINE (ccextractor, "ccextractor",
    GST_RANK_NONE, GST_TYPE_CCEXTRACTOR);

static gboolean gst_cc_extractor_sink_event (GstPad * pad, GstObject * parent,
    GstEvent * event);
static gboolean gst_cc_extractor_sink_query (GstPad * pad, GstObject * parent,
    GstQuery * query);
static GstFlowReturn gst_cc_extractor_chain (GstPad * pad, GstObject * parent,
    GstBuffer * buf);
static GstStateChangeReturn gst_cc_extractor_change_state (GstElement *
    element, GstStateChange transition);

static void gst_cc_extractor_finalize (GObject * self);
static void gst_cc_extractor_set_property (GObject * self, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_cc_extractor_get_property (GObject * self, guint prop_id,
    GValue * value, GParamSpec * pspec);

static void
gst_cc_extractor_class_init (GstCCExtractorClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *gstelement_class;

  gobject_class = (GObjectClass *) klass;
  gstelement_class = (GstElementClass *) klass;

  gobject_class->finalize = gst_cc_extractor_finalize;
  gobject_class->set_property = gst_cc_extractor_set_property;
  gobject_class->get_property = gst_cc_extractor_get_property;

  /**
   * GstCCExtractor:remove-caption-meta
   *
   * Selects whether the #GstVideoCaptionMeta should be removed from the
   * outgoing video buffers or whether it should be kept.
   *
   * Since: 1.18
   */
  g_object_class_install_property (G_OBJECT_CLASS (klass),
      PROP_REMOVE_CAPTION_META, g_param_spec_boolean ("remove-caption-meta",
          "Remove Caption Meta",
          "Remove caption meta from outgoing video buffers", FALSE,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  gstelement_class->change_state =
      GST_DEBUG_FUNCPTR (gst_cc_extractor_change_state);

  gst_element_class_set_static_metadata (gstelement_class,
      "Closed Caption Extractor",
      "Filter",
      "Extract GstVideoCaptionMeta from input stream",
      "Edward Hervey <edward@centricular.com>");

  gst_element_class_add_static_pad_template (gstelement_class, &sinktemplate);
  gst_element_class_add_static_pad_template (gstelement_class, &srctemplate);
  gst_element_class_add_static_pad_template (gstelement_class,
      &captiontemplate);

  GST_DEBUG_CATEGORY_INIT (gst_cc_extractor_debug, "ccextractor",
      0, "Closed Caption extractor");
}

static GstIterator *
gst_cc_extractor_iterate_internal_links (GstPad * pad, GstObject * parent)
{
  GstCCExtractor *filter = (GstCCExtractor *) parent;
  GstIterator *it = NULL;
  GstPad *opad = NULL;

  if (pad == filter->sinkpad)
    opad = filter->srcpad;
  else if (pad == filter->srcpad || pad == filter->captionpad)
    opad = filter->sinkpad;

  if (opad) {
    GValue value = { 0, };

    g_value_init (&value, GST_TYPE_PAD);
    g_value_set_object (&value, opad);
    it = gst_iterator_new_single (GST_TYPE_PAD, &value);
    g_value_unset (&value);
  }

  return it;
}

static void
gst_cc_extractor_reset (GstCCExtractor * filter)
{
  filter->caption_type = GST_VIDEO_CAPTION_TYPE_UNKNOWN;
  gst_flow_combiner_reset (filter->combiner);
  gst_flow_combiner_add_pad (filter->combiner, filter->srcpad);

  if (filter->captionpad) {
    gst_flow_combiner_remove_pad (filter->combiner, filter->captionpad);
    gst_pad_set_active (filter->captionpad, FALSE);
    gst_element_remove_pad ((GstElement *) filter, filter->captionpad);
    filter->captionpad = NULL;
  }

  memset (&filter->video_info, 0, sizeof (filter->video_info));
}

static void
gst_cc_extractor_init (GstCCExtractor * filter)
{
  filter->sinkpad = gst_pad_new_from_static_template (&sinktemplate, "sink");
  gst_pad_set_event_function (filter->sinkpad,
      GST_DEBUG_FUNCPTR (gst_cc_extractor_sink_event));
  gst_pad_set_query_function (filter->sinkpad,
      GST_DEBUG_FUNCPTR (gst_cc_extractor_sink_query));
  gst_pad_set_chain_function (filter->sinkpad,
      GST_DEBUG_FUNCPTR (gst_cc_extractor_chain));
  gst_pad_set_iterate_internal_links_function (filter->sinkpad,
      GST_DEBUG_FUNCPTR (gst_cc_extractor_iterate_internal_links));
  GST_PAD_SET_PROXY_CAPS (filter->sinkpad);
  GST_PAD_SET_PROXY_ALLOCATION (filter->sinkpad);
  GST_PAD_SET_PROXY_SCHEDULING (filter->sinkpad);

  filter->srcpad = gst_pad_new_from_static_template (&srctemplate, "src");
  gst_pad_set_iterate_internal_links_function (filter->srcpad,
      GST_DEBUG_FUNCPTR (gst_cc_extractor_iterate_internal_links));
  GST_PAD_SET_PROXY_CAPS (filter->srcpad);
  GST_PAD_SET_PROXY_ALLOCATION (filter->srcpad);
  GST_PAD_SET_PROXY_SCHEDULING (filter->srcpad);

  gst_element_add_pad (GST_ELEMENT (filter), filter->sinkpad);
  gst_element_add_pad (GST_ELEMENT (filter), filter->srcpad);

  filter->combiner = gst_flow_combiner_new ();

  gst_cc_extractor_reset (filter);
}

static GstEvent *
create_stream_start_event_from_stream_start_event (GstEvent * event)
{
  GstEvent *new_event;
  const gchar *stream_id;
  gchar *new_stream_id;
  guint group_id;

  gst_event_parse_stream_start (event, &stream_id);
  new_stream_id = g_strdup_printf ("%s/caption", stream_id);

  new_event = gst_event_new_stream_start (new_stream_id);
  g_free (new_stream_id);
  if (gst_event_parse_group_id (event, &group_id))
    gst_event_set_group_id (new_event, group_id);

  return new_event;
}

static gboolean
gst_cc_extractor_sink_event (GstPad * pad, GstObject * parent, GstEvent * event)
{
  GstCCExtractor *filter = GST_CCEXTRACTOR (parent);

  GST_LOG_OBJECT (pad, "received %s event: %" GST_PTR_FORMAT,
      GST_EVENT_TYPE_NAME (event), event);
  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_CAPS:{
      GstCaps *caps;

      gst_event_parse_caps (event, &caps);
      if (!gst_video_info_from_caps (&filter->video_info, caps)) {
        /* We require any kind of video caps here */
        gst_event_unref (event);
        return FALSE;
      }
      break;
    }
    case GST_EVENT_STREAM_START:
      if (filter->captionpad) {
        GstEvent *new_event =
            create_stream_start_event_from_stream_start_event (event);
        gst_pad_push_event (filter->captionpad, new_event);
      }
      break;
    default:
      /* Also forward all other events to the caption pad if present */
      if (filter->captionpad)
        gst_pad_push_event (filter->captionpad, gst_event_ref (event));
      break;
  }

  /* This only forwards to the non-caption source pad */
  return gst_pad_event_default (pad, parent, event);
}

static gboolean
gst_cc_extractor_sink_query (GstPad * pad, GstObject * parent, GstQuery * query)
{
  GST_LOG_OBJECT (pad, "received %s query: %" GST_PTR_FORMAT,
      GST_QUERY_TYPE_NAME (query), query);
  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_ACCEPT_CAPS:{
      GstCaps *caps;
      const GstStructure *s;

      gst_query_parse_accept_caps (query, &caps);

      /* FIXME: Ideally we would declare this in our caps but there's no way
       * to declare caps of type "video/" and "image/" that would match all
       * such caps
       */
      s = gst_caps_get_structure (caps, 0);
      if (s && (g_str_has_prefix (gst_structure_get_name (s), "video/")
              || g_str_has_prefix (gst_structure_get_name (s), "image/")))
        gst_query_set_accept_caps_result (query, TRUE);
      else
        gst_query_set_accept_caps_result (query, FALSE);

      return TRUE;
    }
    default:
      break;
  }

  return gst_pad_query_default (pad, parent, query);
}

static GstCaps *
create_caps_from_caption_type (GstVideoCaptionType caption_type,
    const GstVideoInfo * video_info)
{
  GstCaps *caption_caps = gst_video_caption_type_to_caps (caption_type);

  gst_caps_set_simple (caption_caps, "framerate", GST_TYPE_FRACTION,
      video_info->fps_n, video_info->fps_d, NULL);

  return caption_caps;
}

static gboolean
forward_sticky_events (GstPad * pad, GstEvent ** event, gpointer user_data)
{
  GstCCExtractor *filter = user_data;

  switch (GST_EVENT_TYPE (*event)) {
    case GST_EVENT_CAPS:{
      GstCaps *caption_caps =
          create_caps_from_caption_type (filter->caption_type,
          &filter->video_info);

      if (caption_caps) {
        GstEvent *new_event = gst_event_new_caps (caption_caps);
        gst_event_set_seqnum (new_event, gst_event_get_seqnum (*event));
        gst_pad_store_sticky_event (filter->captionpad, new_event);
        gst_event_unref (new_event);
        gst_caps_unref (caption_caps);
      }

      break;
    }
    case GST_EVENT_STREAM_START:{
      GstEvent *new_event =
          create_stream_start_event_from_stream_start_event (*event);
      gst_pad_store_sticky_event (filter->captionpad, new_event);
      gst_event_unref (new_event);

      break;
    }
    default:
      gst_pad_store_sticky_event (filter->captionpad, *event);
      break;
  }

  return TRUE;
}

static GstFlowReturn
gst_cc_extractor_handle_meta (GstCCExtractor * filter, GstBuffer * buf,
    GstVideoCaptionMeta * meta, GstVideoTimeCodeMeta * tc_meta)
{
  GstBuffer *outbuf = NULL;
  GstFlowReturn flow;

  GST_DEBUG_OBJECT (filter, "Handling meta");

  /* Check if the meta type matches the configured one */
  if (filter->captionpad == NULL) {
    GST_DEBUG_OBJECT (filter, "Creating new caption pad");

    /* Create the caption pad and set the caps */
    filter->captionpad =
        gst_pad_new_from_static_template (&captiontemplate, "caption");
    gst_pad_set_iterate_internal_links_function (filter->sinkpad,
        GST_DEBUG_FUNCPTR (gst_cc_extractor_iterate_internal_links));
    gst_pad_set_active (filter->captionpad, TRUE);

    filter->caption_type = meta->caption_type;

    gst_pad_sticky_events_foreach (filter->sinkpad, forward_sticky_events,
        filter);

    if (!gst_pad_has_current_caps (filter->captionpad)) {
      GST_ERROR_OBJECT (filter, "Unknown/invalid caption type");
      return GST_FLOW_NOT_NEGOTIATED;
    }

    gst_element_add_pad (GST_ELEMENT (filter), filter->captionpad);
    gst_flow_combiner_add_pad (filter->combiner, filter->captionpad);
  } else if (meta->caption_type != filter->caption_type) {
    GstCaps *caption_caps =
        create_caps_from_caption_type (meta->caption_type, &filter->video_info);

    GST_DEBUG_OBJECT (filter, "Caption type changed from %d to %d",
        filter->caption_type, meta->caption_type);
    if (caption_caps == NULL) {
      GST_ERROR_OBJECT (filter, "Unknown/invalid caption type");
      return GST_FLOW_NOT_NEGOTIATED;
    }

    gst_pad_push_event (filter->captionpad, gst_event_new_caps (caption_caps));
    gst_caps_unref (caption_caps);

    filter->caption_type = meta->caption_type;
  }

  GST_DEBUG_OBJECT (filter,
      "Creating new buffer of size %" G_GSIZE_FORMAT " bytes", meta->size);
  /* Extract caption data into new buffer with identical buffer timestamps */
  outbuf = gst_buffer_new_allocate (NULL, meta->size, NULL);
  gst_buffer_fill (outbuf, 0, meta->data, meta->size);
  GST_BUFFER_PTS (outbuf) = GST_BUFFER_PTS (buf);
  GST_BUFFER_DTS (outbuf) = GST_BUFFER_DTS (buf);
  GST_BUFFER_DURATION (outbuf) = GST_BUFFER_DURATION (buf);

  if (tc_meta)
    gst_buffer_add_video_time_code_meta (outbuf, &tc_meta->tc);

  gst_buffer_set_flags (outbuf, gst_buffer_get_flags (buf));
  /* We don't really care about the flow return */
  flow = gst_pad_push (filter->captionpad, outbuf);

  /* Set flow return on pad and return combined value */
  return gst_flow_combiner_update_pad_flow (filter->combiner,
      filter->captionpad, flow);
}

static gboolean
remove_caption_meta (GstBuffer * buffer, GstMeta ** meta, gpointer user_data)
{
  if ((*meta)->info->api == GST_VIDEO_CAPTION_META_API_TYPE)
    *meta = NULL;

  return TRUE;
}

static GstFlowReturn
gst_cc_extractor_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
{
  GstCCExtractor *filter = (GstCCExtractor *) parent;
  GstFlowReturn flow = GST_FLOW_OK;
  GstVideoCaptionMeta *cc_meta;
  GstVideoTimeCodeMeta *tc_meta;
  gboolean had_cc_meta = FALSE;
  gpointer iter = NULL;

  tc_meta = gst_buffer_get_video_time_code_meta (buf);

  while ((cc_meta =
          (GstVideoCaptionMeta *) gst_buffer_iterate_meta_filtered (buf, &iter,
              GST_VIDEO_CAPTION_META_API_TYPE)) && flow == GST_FLOW_OK) {
    had_cc_meta = TRUE;
    flow = gst_cc_extractor_handle_meta (filter, buf, cc_meta, tc_meta);
  }

  /* If there's an issue handling the CC, return immediately */
  if (flow != GST_FLOW_OK) {
    gst_buffer_unref (buf);
    return flow;
  }

  if (filter->remove_caption_meta) {
    buf = gst_buffer_make_writable (buf);
    gst_buffer_foreach_meta (buf, remove_caption_meta, NULL);
  }

  if (!had_cc_meta && filter->captionpad && GST_BUFFER_PTS_IS_VALID (buf)) {
    gst_pad_push_event (filter->captionpad,
        gst_event_new_gap (GST_BUFFER_PTS (buf), GST_BUFFER_DURATION (buf)));
  }

  /* Push the buffer downstream and return the combined flow return */
  return gst_flow_combiner_update_pad_flow (filter->combiner, filter->srcpad,
      gst_pad_push (filter->srcpad, buf));
}

static GstStateChangeReturn
gst_cc_extractor_change_state (GstElement * element, GstStateChange transition)
{
  GstStateChangeReturn ret;
  GstCCExtractor *filter = GST_CCEXTRACTOR (element);

  switch (transition) {
    case GST_STATE_CHANGE_NULL_TO_READY:
      break;
    case GST_STATE_CHANGE_READY_TO_PAUSED:
      break;
    case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
      break;
    default:
      break;
  }

  ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
  if (ret != GST_STATE_CHANGE_SUCCESS)
    return ret;

  switch (transition) {
    case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
      break;
    case GST_STATE_CHANGE_PAUSED_TO_READY:
      gst_cc_extractor_reset (filter);
      break;
    case GST_STATE_CHANGE_READY_TO_NULL:
    default:
      break;
  }

  return ret;
}

static void
gst_cc_extractor_finalize (GObject * object)
{
  GstCCExtractor *filter = GST_CCEXTRACTOR (object);

  gst_flow_combiner_free (filter->combiner);

  G_OBJECT_CLASS (parent_class)->finalize (object);
}

static void
gst_cc_extractor_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstCCExtractor *filter = GST_CCEXTRACTOR (object);

  switch (prop_id) {
    case PROP_REMOVE_CAPTION_META:
      filter->remove_caption_meta = g_value_get_boolean (value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_cc_extractor_get_property (GObject * object, guint prop_id, GValue * value,
    GParamSpec * pspec)
{
  GstCCExtractor *filter = GST_CCEXTRACTOR (object);

  switch (prop_id) {
    case PROP_REMOVE_CAPTION_META:
      g_value_set_boolean (value, filter->remove_caption_meta);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}