1 /*
2 * gloverlaycompositor element
3 * Copyrithg (C) 2018 Matthew Waters <matthew@centricular.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 */
20
21 /**
22 * SECTION:element-glcompositoroverlay
23 * @title: glcompositoroverlay
24 *
25 */
26 #ifdef HAVE_CONFIG_H
27 #include "config.h"
28 #endif
29
30 #include <gst/gl/gstglfuncs.h>
31 #include <gst/video/video.h>
32
33 #include "gstglelements.h"
34 #include "gstgloverlaycompositorelement.h"
35
36 enum
37 {
38 PROP_0,
39 PROP_LAST,
40 };
41
42 #define GST_CAT_DEFAULT gst_gl_overlay_compositor_element_debug
43 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
44
45 #define DEBUG_INIT \
46 GST_DEBUG_CATEGORY_INIT (gst_gl_overlay_compositor_element_debug, "gloverlaycompositorelement", 0, "gloverlaycompositor element");
47 #define gst_gl_overlay_compositor_element_parent_class parent_class
48 G_DEFINE_TYPE_WITH_CODE (GstGLOverlayCompositorElement,
49 gst_gl_overlay_compositor_element, GST_TYPE_GL_FILTER, DEBUG_INIT);
50 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (gloverlaycompositor,
51 "gloverlaycompositor", GST_RANK_NONE,
52 GST_TYPE_GL_OVERLAY_COMPOSITOR_ELEMENT, gl_element_init (plugin));
53
54 static GstStaticPadTemplate overlay_sink_pad_template =
55 GST_STATIC_PAD_TEMPLATE ("sink",
56 GST_PAD_SINK,
57 GST_PAD_ALWAYS,
58 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
59 (GST_CAPS_FEATURE_MEMORY_GL_MEMORY ","
60 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
61 "RGBA") ", texture-target=(string) { 2D, rectangle } ; "
62 GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
63 "RGBA") ", texture-target=(string) { 2D, rectangle } ; "
64 GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY",
65 "RGBA") ", texture-target=(string) { 2D, rectangle } "));
66
67 static GstStaticPadTemplate overlay_src_pad_template =
68 GST_STATIC_PAD_TEMPLATE ("src",
69 GST_PAD_SRC,
70 GST_PAD_ALWAYS,
71 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
72 (GST_CAPS_FEATURE_MEMORY_GL_MEMORY ","
73 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
74 "RGBA") ", texture-target=(string) { 2D, rectangle } ; "
75 GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
76 "RGBA") ", texture-target=(string) { 2D, rectangle } ; "
77 GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY",
78 "RGBA") ", texture-target=(string) { 2D, rectangle } "));
79
80 static gboolean
81 gst_gl_overlay_compositor_element_propose_allocation (GstBaseTransform * trans,
82 GstQuery * decide_query, GstQuery * query);
83 static GstFlowReturn _oce_prepare_output_buffer (GstBaseTransform * bt,
84 GstBuffer * buffer, GstBuffer ** outbuf);
85
86 static gboolean gst_gl_overlay_compositor_element_gl_start (GstGLBaseFilter *
87 base);
88 static void gst_gl_overlay_compositor_element_gl_stop (GstGLBaseFilter * base);
89
90 static GstCaps *_oce_transform_internal_caps (GstGLFilter *
91 filter, GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps);
92 static gboolean gst_gl_overlay_compositor_element_filter (GstGLFilter * filter,
93 GstBuffer * inbuf, GstBuffer * outbuf);
94 static gboolean gst_gl_overlay_compositor_element_filter_texture (GstGLFilter *
95 filter, GstGLMemory * in_tex, GstGLMemory * out_tex);
96 static gboolean gst_gl_overlay_compositor_element_callback (GstGLFilter *
97 filter, GstGLMemory * in_tex, gpointer stuff);
98
99 static void
gst_gl_overlay_compositor_element_class_init(GstGLOverlayCompositorElementClass * klass)100 gst_gl_overlay_compositor_element_class_init (GstGLOverlayCompositorElementClass
101 * klass)
102 {
103 GstElementClass *element_class;
104
105 element_class = GST_ELEMENT_CLASS (klass);
106
107 gst_element_class_set_metadata (element_class,
108 "OpenGL overlaying filter", "Filter/Effect",
109 "Flatten a stream containing GstVideoOverlayCompositionMeta",
110 "<matthew@centricular.com>");
111
112 gst_element_class_add_static_pad_template (element_class,
113 &overlay_src_pad_template);
114 gst_element_class_add_static_pad_template (element_class,
115 &overlay_sink_pad_template);
116
117 GST_BASE_TRANSFORM_CLASS (klass)->passthrough_on_same_caps = TRUE;
118 GST_BASE_TRANSFORM_CLASS (klass)->propose_allocation =
119 gst_gl_overlay_compositor_element_propose_allocation;
120 GST_BASE_TRANSFORM_CLASS (klass)->prepare_output_buffer =
121 _oce_prepare_output_buffer;
122
123 GST_GL_FILTER_CLASS (klass)->filter =
124 gst_gl_overlay_compositor_element_filter;
125 GST_GL_FILTER_CLASS (klass)->filter_texture =
126 gst_gl_overlay_compositor_element_filter_texture;
127 GST_GL_FILTER_CLASS (klass)->transform_internal_caps =
128 _oce_transform_internal_caps;
129
130 GST_GL_BASE_FILTER_CLASS (klass)->gl_start =
131 gst_gl_overlay_compositor_element_gl_start;
132 GST_GL_BASE_FILTER_CLASS (klass)->gl_stop =
133 gst_gl_overlay_compositor_element_gl_stop;
134 GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
135 GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3;
136 }
137
138 static void
gst_gl_overlay_compositor_element_init(GstGLOverlayCompositorElement * overlay_compositor_element)139 gst_gl_overlay_compositor_element_init (GstGLOverlayCompositorElement *
140 overlay_compositor_element)
141 {
142 }
143
144 static GstCaps *
_oce_transform_internal_caps(GstGLFilter * filter,GstPadDirection direction,GstCaps * caps,GstCaps * filter_caps)145 _oce_transform_internal_caps (GstGLFilter * filter,
146 GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps)
147 {
148 GstCaps *ret;
149
150 /* add/remove the composition overlay meta as necessary */
151 if (direction == GST_PAD_SRC) {
152 ret = gst_gl_overlay_compositor_add_caps (gst_caps_copy (caps));
153 } else {
154 guint i, n;
155 GstCaps *removed;
156
157 ret = gst_caps_copy (caps);
158 removed = gst_caps_copy (caps);
159 n = gst_caps_get_size (removed);
160 for (i = 0; i < n; i++) {
161 GstCapsFeatures *feat = gst_caps_get_features (removed, i);
162
163 if (feat && gst_caps_features_contains (feat,
164 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) {
165 feat = gst_caps_features_copy (feat);
166 /* prefer the passthrough case */
167 gst_caps_features_remove (feat,
168 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
169 gst_caps_set_features (removed, i, feat);
170 }
171 }
172
173 ret = gst_caps_merge (ret, removed);
174 }
175
176 GST_DEBUG_OBJECT (filter, "meta modifications returned caps %" GST_PTR_FORMAT,
177 ret);
178 return ret;
179 }
180
181 static gboolean
gst_gl_overlay_compositor_element_propose_allocation(GstBaseTransform * trans,GstQuery * decide_query,GstQuery * query)182 gst_gl_overlay_compositor_element_propose_allocation (GstBaseTransform * trans,
183 GstQuery * decide_query, GstQuery * query)
184 {
185 GstStructure *allocation_meta = NULL;
186 guint width = 0, height = 0;
187
188 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->propose_allocation (trans,
189 decide_query, query))
190 return FALSE;
191
192 if (decide_query) {
193 GstCaps *decide_caps;
194 gst_query_parse_allocation (decide_query, &decide_caps, NULL);
195
196 if (decide_caps) {
197 GstVideoInfo vinfo;
198
199 if (gst_video_info_from_caps (&vinfo, decide_caps)) {
200 width = GST_VIDEO_INFO_WIDTH (&vinfo);
201 height = GST_VIDEO_INFO_HEIGHT (&vinfo);
202 }
203 }
204 }
205
206 if ((width == 0 || height == 0) && query) {
207 GstCaps *caps;
208 gst_query_parse_allocation (query, &caps, NULL);
209
210 if (caps) {
211 GstVideoInfo vinfo;
212
213 if (gst_video_info_from_caps (&vinfo, caps)) {
214 width = GST_VIDEO_INFO_WIDTH (&vinfo);
215 height = GST_VIDEO_INFO_HEIGHT (&vinfo);
216 }
217 }
218 }
219
220 if (width != 0 && height != 0) {
221 allocation_meta =
222 gst_structure_new ("GstVideoOverlayCompositionMeta",
223 "width", G_TYPE_UINT, width, "height", G_TYPE_UINT, height, NULL);
224 }
225
226 GST_DEBUG_OBJECT (trans, "Adding overlay composition meta with size %ux%u",
227 width, height);
228 if (allocation_meta) {
229 if (query)
230 gst_query_add_allocation_meta (query,
231 GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, allocation_meta);
232 gst_structure_free (allocation_meta);
233 }
234 return TRUE;
235 }
236
237 static void
gst_gl_overlay_compositor_element_gl_stop(GstGLBaseFilter * base)238 gst_gl_overlay_compositor_element_gl_stop (GstGLBaseFilter * base)
239 {
240 GstGLOverlayCompositorElement *self =
241 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (base);
242
243 if (self->shader)
244 gst_object_unref (self->shader);
245 self->shader = NULL;
246
247 if (self->overlay_compositor) {
248 gst_gl_overlay_compositor_free_overlays (self->overlay_compositor);
249 gst_object_unref (self->overlay_compositor);
250 }
251 self->overlay_compositor = NULL;
252
253 GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base);
254 }
255
256 static gboolean
gst_gl_overlay_compositor_element_gl_start(GstGLBaseFilter * base)257 gst_gl_overlay_compositor_element_gl_start (GstGLBaseFilter * base)
258 {
259 GstGLOverlayCompositorElement *self =
260 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (base);
261 GError *error = NULL;
262
263 self->overlay_compositor = gst_gl_overlay_compositor_new (base->context);
264 g_object_set (self->overlay_compositor, "yinvert", TRUE, NULL);
265
266 if (!(self->shader = gst_gl_shader_new_default (base->context, &error))) {
267 GST_ELEMENT_ERROR (base, RESOURCE, NOT_FOUND, ("%s",
268 "Failed to compile identity shader"), ("%s", error->message));
269 return FALSE;
270 }
271
272 return GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base);
273 }
274
275 static GstFlowReturn
_oce_prepare_output_buffer(GstBaseTransform * bt,GstBuffer * buffer,GstBuffer ** outbuf)276 _oce_prepare_output_buffer (GstBaseTransform * bt,
277 GstBuffer * buffer, GstBuffer ** outbuf)
278 {
279 GstGLOverlayCompositorElement *self = GST_GL_OVERLAY_COMPOSITOR_ELEMENT (bt);
280 GstVideoOverlayCompositionMeta *comp_meta;
281
282 if (gst_base_transform_is_passthrough (bt))
283 goto passthrough;
284
285 if (!self->overlay_compositor)
286 return GST_FLOW_NOT_NEGOTIATED;
287
288 comp_meta = gst_buffer_get_video_overlay_composition_meta (buffer);
289 if (!comp_meta)
290 goto passthrough;
291
292 if (gst_video_overlay_composition_n_rectangles (comp_meta->overlay) == 0)
293 goto passthrough;
294
295 return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (bt,
296 buffer, outbuf);
297
298 passthrough:
299 GST_LOG_OBJECT (bt, "passthrough detected, forwarding input buffer");
300 *outbuf = buffer;
301 return GST_FLOW_OK;
302 }
303
304 static gboolean
gst_gl_overlay_compositor_element_filter(GstGLFilter * filter,GstBuffer * inbuf,GstBuffer * outbuf)305 gst_gl_overlay_compositor_element_filter (GstGLFilter * filter,
306 GstBuffer * inbuf, GstBuffer * outbuf)
307 {
308 GstGLOverlayCompositorElement *self =
309 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter);
310
311 if (inbuf == outbuf)
312 return TRUE;
313
314 gst_gl_overlay_compositor_upload_overlays (self->overlay_compositor, inbuf);
315
316 return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
317 }
318
319 static gboolean
gst_gl_overlay_compositor_element_filter_texture(GstGLFilter * filter,GstGLMemory * in_tex,GstGLMemory * out_tex)320 gst_gl_overlay_compositor_element_filter_texture (GstGLFilter * filter,
321 GstGLMemory * in_tex, GstGLMemory * out_tex)
322 {
323 GstGLOverlayCompositorElement *self =
324 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter);
325
326 gst_gl_filter_render_to_target_with_shader (filter, in_tex, out_tex,
327 self->shader);
328
329 gst_gl_filter_render_to_target (filter, NULL, out_tex,
330 gst_gl_overlay_compositor_element_callback, NULL);
331
332 return TRUE;
333 }
334
335 static gboolean
gst_gl_overlay_compositor_element_callback(GstGLFilter * filter,GstGLMemory * in_tex,gpointer stuff)336 gst_gl_overlay_compositor_element_callback (GstGLFilter * filter,
337 GstGLMemory * in_tex, gpointer stuff)
338 {
339 GstGLOverlayCompositorElement *self =
340 GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter);
341
342 GST_LOG_OBJECT (self, "drawing overlays");
343
344 gst_gl_overlay_compositor_draw_overlays (self->overlay_compositor);
345
346 return TRUE;
347 }
348