1 /*
2 * GStreamer
3 * Copyright (C) 2014 Lubosz Sarnecki <lubosz@gmail.com>
4 * Copyright (C) 2016 Matthew Waters <matthew@centricular.com>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
15 *
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
20 */
21
22 /**
23 * SECTION:element-gltransformation
24 * @title: gltransformation
25 *
26 * Transforms video on the GPU.
27 *
28 * ## Examples
29 * |[
30 * gst-launch-1.0 gltestsrc ! gltransformation rotation-z=45 ! glimagesink
31 * ]| A pipeline to rotate by 45 degrees
32 * |[
33 * gst-launch-1.0 gltestsrc ! gltransformation translation-x=0.5 ! glimagesink
34 * ]| Translate the video by 0.5
35 * |[
36 * gst-launch-1.0 gltestsrc ! gltransformation scale-y=0.5 scale-x=0.5 ! glimagesink
37 * ]| Resize the video by 0.5
38 * |[
39 * gst-launch-1.0 gltestsrc ! gltransformation rotation-x=-45 ortho=True ! glimagesink
40 * ]| Rotate the video around the X-Axis by -45° with an orthographic projection
41 *
42 */
43
44 #ifdef HAVE_CONFIG_H
45 #include "config.h"
46 #endif
47
48 #include "gstglelements.h"
49 #include "gstgltransformation.h"
50
51 #include <gst/gl/gstglapi.h>
52 #include <graphene-gobject.h>
53 #include "gstglutils.h"
54
55 #define GST_CAT_DEFAULT gst_gl_transformation_debug
56 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
57
58 #define gst_gl_transformation_parent_class parent_class
59
60 #define VEC4_FORMAT "%f,%f,%f,%f"
61 #define VEC4_ARGS(v) graphene_vec4_get_x (v), graphene_vec4_get_y (v), graphene_vec4_get_z (v), graphene_vec4_get_w (v)
62 #define VEC3_FORMAT "%f,%f,%f"
63 #define VEC3_ARGS(v) graphene_vec3_get_x (v), graphene_vec3_get_y (v), graphene_vec3_get_z (v)
64 #define VEC2_FORMAT "%f,%f"
65 #define VEC2_ARGS(v) graphene_vec2_get_x (v), graphene_vec2_get_y (v)
66 #define POINT3D_FORMAT "%f,%f,%f"
67 #define POINT3D_ARGS(p) (p)->x, (p)->y, (p)->z
68
69 enum
70 {
71 PROP_0,
72 PROP_FOV,
73 PROP_ORTHO,
74 PROP_TRANSLATION_X,
75 PROP_TRANSLATION_Y,
76 PROP_TRANSLATION_Z,
77 PROP_ROTATION_X,
78 PROP_ROTATION_Y,
79 PROP_ROTATION_Z,
80 PROP_SCALE_X,
81 PROP_SCALE_Y,
82 PROP_MVP,
83 PROP_PIVOT_X,
84 PROP_PIVOT_Y,
85 PROP_PIVOT_Z,
86 };
87
88 #define DEBUG_INIT \
89 GST_DEBUG_CATEGORY_INIT (gst_gl_transformation_debug, "gltransformation", 0, "gltransformation element");
90
91 G_DEFINE_TYPE_WITH_CODE (GstGLTransformation, gst_gl_transformation,
92 GST_TYPE_GL_FILTER, DEBUG_INIT);
93 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (gltransformation, "gltransformation",
94 GST_RANK_NONE, GST_TYPE_GL_TRANSFORMATION, gl_element_init (plugin));
95
96 static void gst_gl_transformation_finalize (GObject * object);
97 static void gst_gl_transformation_set_property (GObject * object, guint prop_id,
98 const GValue * value, GParamSpec * pspec);
99 static void gst_gl_transformation_get_property (GObject * object, guint prop_id,
100 GValue * value, GParamSpec * pspec);
101
102 static gboolean gst_gl_transformation_set_caps (GstGLFilter * filter,
103 GstCaps * incaps, GstCaps * outcaps);
104 static gboolean gst_gl_transformation_src_event (GstBaseTransform * trans,
105 GstEvent * event);
106 static gboolean gst_gl_transformation_filter_meta (GstBaseTransform * trans,
107 GstQuery * query, GType api, const GstStructure * params);
108 static gboolean gst_gl_transformation_decide_allocation (GstBaseTransform *
109 trans, GstQuery * query);
110
111 static void gst_gl_transformation_gl_stop (GstGLBaseFilter * filter);
112 static gboolean gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter);
113 static gboolean gst_gl_transformation_callback (gpointer stuff);
114 static void gst_gl_transformation_build_mvp (GstGLTransformation *
115 transformation);
116
117 static GstFlowReturn
118 gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
119 GstBuffer * inbuf, GstBuffer ** outbuf);
120 static gboolean gst_gl_transformation_filter (GstGLFilter * filter,
121 GstBuffer * inbuf, GstBuffer * outbuf);
122 static gboolean gst_gl_transformation_filter_texture (GstGLFilter * filter,
123 GstGLMemory * in_tex, GstGLMemory * out_tex);
124
125 static void
gst_gl_transformation_class_init(GstGLTransformationClass * klass)126 gst_gl_transformation_class_init (GstGLTransformationClass * klass)
127 {
128 GObjectClass *gobject_class;
129 GstElementClass *element_class;
130 GstBaseTransformClass *base_transform_class;
131
132 gobject_class = (GObjectClass *) klass;
133 element_class = GST_ELEMENT_CLASS (klass);
134 base_transform_class = GST_BASE_TRANSFORM_CLASS (klass);
135
136 gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));
137
138 gobject_class->set_property = gst_gl_transformation_set_property;
139 gobject_class->get_property = gst_gl_transformation_get_property;
140
141 base_transform_class->src_event = gst_gl_transformation_src_event;
142 base_transform_class->decide_allocation =
143 gst_gl_transformation_decide_allocation;
144 base_transform_class->filter_meta = gst_gl_transformation_filter_meta;
145
146 GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_transformation_gl_start;
147 GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_transformation_gl_stop;
148
149 GST_GL_FILTER_CLASS (klass)->set_caps = gst_gl_transformation_set_caps;
150 GST_GL_FILTER_CLASS (klass)->filter = gst_gl_transformation_filter;
151 GST_GL_FILTER_CLASS (klass)->filter_texture =
152 gst_gl_transformation_filter_texture;
153 GST_BASE_TRANSFORM_CLASS (klass)->prepare_output_buffer =
154 gst_gl_transformation_prepare_output_buffer;
155
156 g_object_class_install_property (gobject_class, PROP_FOV,
157 g_param_spec_float ("fov", "Fov", "Field of view angle in degrees",
158 0.0, G_MAXFLOAT, 90.0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
159
160 g_object_class_install_property (gobject_class, PROP_ORTHO,
161 g_param_spec_boolean ("ortho", "Orthographic",
162 "Use orthographic projection", FALSE,
163 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
164
165 /* Rotation */
166 g_object_class_install_property (gobject_class, PROP_ROTATION_X,
167 g_param_spec_float ("rotation-x", "X Rotation",
168 "Rotates the video around the X-Axis in degrees.",
169 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
170 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
171
172 g_object_class_install_property (gobject_class, PROP_ROTATION_Y,
173 g_param_spec_float ("rotation-y", "Y Rotation",
174 "Rotates the video around the Y-Axis in degrees.",
175 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
176 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
177
178 g_object_class_install_property (gobject_class, PROP_ROTATION_Z,
179 g_param_spec_float ("rotation-z", "Z Rotation",
180 "Rotates the video around the Z-Axis in degrees.",
181 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
182 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
183
184 /* Translation */
185 g_object_class_install_property (gobject_class, PROP_TRANSLATION_X,
186 g_param_spec_float ("translation-x", "X Translation",
187 "Translates the video at the X-Axis, in universal [0-1] coordinate.",
188 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
189 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
190
191 g_object_class_install_property (gobject_class, PROP_TRANSLATION_Y,
192 g_param_spec_float ("translation-y", "Y Translation",
193 "Translates the video at the Y-Axis, in universal [0-1] coordinate.",
194 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
195 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
196
197 g_object_class_install_property (gobject_class, PROP_TRANSLATION_Z,
198 g_param_spec_float ("translation-z", "Z Translation",
199 "Translates the video at the Z-Axis, in universal [0-1] coordinate.",
200 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
201 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
202
203 /* Scale */
204 g_object_class_install_property (gobject_class, PROP_SCALE_X,
205 g_param_spec_float ("scale-x", "X Scale",
206 "Scale multiplier for the X-Axis.",
207 -G_MAXFLOAT, G_MAXFLOAT, 1.0,
208 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
209
210 g_object_class_install_property (gobject_class, PROP_SCALE_Y,
211 g_param_spec_float ("scale-y", "Y Scale",
212 "Scale multiplier for the Y-Axis.",
213 -G_MAXFLOAT, G_MAXFLOAT, 1.0,
214 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
215
216 /* Pivot */
217 g_object_class_install_property (gobject_class, PROP_PIVOT_X,
218 g_param_spec_float ("pivot-x", "X Pivot",
219 "Rotation pivot point X coordinate, where 0 is the center,"
220 " -1 the left border, +1 the right border and <-1, >1 outside.",
221 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
222 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
223
224 g_object_class_install_property (gobject_class, PROP_PIVOT_Y,
225 g_param_spec_float ("pivot-y", "Y Pivot",
226 "Rotation pivot point X coordinate, where 0 is the center,"
227 " -1 the left border, +1 the right border and <-1, >1 outside.",
228 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
229 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
230
231 g_object_class_install_property (gobject_class, PROP_PIVOT_Z,
232 g_param_spec_float ("pivot-z", "Z Pivot",
233 "Relevant for rotation in 3D space. You look into the negative Z axis direction",
234 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
235 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
236
237 /* MVP */
238 g_object_class_install_property (gobject_class, PROP_MVP,
239 g_param_spec_boxed ("mvp-matrix",
240 "Modelview Projection Matrix",
241 "The final Graphene 4x4 Matrix for transformation",
242 GRAPHENE_TYPE_MATRIX, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
243
244 gst_element_class_set_metadata (element_class, "OpenGL transformation filter",
245 "Filter/Effect/Video", "Transform video on the GPU",
246 "Lubosz Sarnecki <lubosz@gmail.com>\n"
247 "Matthew Waters <matthew@centricular.com>");
248
249 GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
250 GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2;
251
252 gobject_class->finalize = gst_gl_transformation_finalize;
253 }
254
255 static void
gst_gl_transformation_init(GstGLTransformation * filter)256 gst_gl_transformation_init (GstGLTransformation * filter)
257 {
258 filter->shader = NULL;
259 filter->fov = 90;
260 filter->aspect = 1.0;
261 filter->znear = 0.1f;
262 filter->zfar = 100.0;
263
264 filter->xscale = 1.0;
265 filter->yscale = 1.0;
266
267 filter->in_tex = 0;
268
269 /* let graphene alloc its structures with correct memory alignment, otherwise
270 * unaligned memory access faults can occur */
271 filter->model_matrix = graphene_matrix_alloc ();
272 filter->view_matrix = graphene_matrix_alloc ();
273 filter->projection_matrix = graphene_matrix_alloc ();
274 filter->inv_model_matrix = graphene_matrix_alloc ();
275 filter->inv_view_matrix = graphene_matrix_alloc ();
276 filter->inv_projection_matrix = graphene_matrix_alloc ();
277 filter->mvp_matrix = graphene_matrix_alloc ();
278
279 filter->camera_position = graphene_vec3_alloc ();
280
281 gst_gl_transformation_build_mvp (filter);
282 }
283
284 static void
gst_gl_transformation_finalize(GObject * object)285 gst_gl_transformation_finalize (GObject * object)
286 {
287 GstGLTransformation *transformation;
288
289 g_return_if_fail (GST_IS_GL_TRANSFORMATION (object));
290
291 transformation = GST_GL_TRANSFORMATION (object);
292
293 graphene_matrix_free (transformation->model_matrix);
294 graphene_matrix_free (transformation->view_matrix);
295 graphene_matrix_free (transformation->projection_matrix);
296 graphene_matrix_free (transformation->inv_model_matrix);
297 graphene_matrix_free (transformation->inv_view_matrix);
298 graphene_matrix_free (transformation->inv_projection_matrix);
299 graphene_matrix_free (transformation->mvp_matrix);
300
301 graphene_vec3_free (transformation->camera_position);
302
303 G_OBJECT_CLASS (parent_class)->finalize (object);
304 }
305
306 static void
gst_gl_transformation_build_mvp(GstGLTransformation * transformation)307 gst_gl_transformation_build_mvp (GstGLTransformation * transformation)
308 {
309 GstGLFilter *filter = GST_GL_FILTER (transformation);
310 graphene_matrix_t modelview_matrix;
311
312 if (!filter->out_info.finfo) {
313 graphene_matrix_init_identity (transformation->model_matrix);
314 graphene_matrix_init_identity (transformation->view_matrix);
315 graphene_matrix_init_identity (transformation->projection_matrix);
316 } else {
317 graphene_point3d_t translation_vector =
318 GRAPHENE_POINT3D_INIT (transformation->xtranslation * 2.0 *
319 transformation->aspect,
320 transformation->ytranslation * 2.0,
321 transformation->ztranslation * 2.0);
322
323 graphene_point3d_t pivot_vector =
324 GRAPHENE_POINT3D_INIT (-transformation->xpivot * transformation->aspect,
325 transformation->ypivot,
326 -transformation->zpivot);
327
328 graphene_point3d_t negative_pivot_vector;
329
330 graphene_vec3_t center;
331 graphene_vec3_t up;
332
333 gboolean current_passthrough;
334 gboolean passthrough;
335
336 graphene_vec3_init (transformation->camera_position, 0.f, 0.f, 1.f);
337 graphene_vec3_init (¢er, 0.f, 0.f, 0.f);
338 graphene_vec3_init (&up, 0.f, 1.f, 0.f);
339
340 /* Translate into pivot origin */
341 graphene_matrix_init_translate (transformation->model_matrix,
342 &pivot_vector);
343
344 /* Scale */
345 graphene_matrix_scale (transformation->model_matrix,
346 transformation->xscale, transformation->yscale, 1.0f);
347
348 /* Rotation */
349 graphene_matrix_rotate (transformation->model_matrix,
350 transformation->xrotation, graphene_vec3_x_axis ());
351 graphene_matrix_rotate (transformation->model_matrix,
352 transformation->yrotation, graphene_vec3_y_axis ());
353 graphene_matrix_rotate (transformation->model_matrix,
354 transformation->zrotation, graphene_vec3_z_axis ());
355
356 /* Translate back from pivot origin */
357 graphene_point3d_scale (&pivot_vector, -1.0, &negative_pivot_vector);
358 graphene_matrix_translate (transformation->model_matrix,
359 &negative_pivot_vector);
360
361 /* Translation */
362 graphene_matrix_translate (transformation->model_matrix,
363 &translation_vector);
364
365 if (transformation->ortho) {
366 graphene_matrix_init_ortho (transformation->projection_matrix,
367 -transformation->aspect, transformation->aspect,
368 -1, 1, transformation->znear, transformation->zfar);
369 } else {
370 graphene_matrix_init_perspective (transformation->projection_matrix,
371 transformation->fov,
372 transformation->aspect, transformation->znear, transformation->zfar);
373 }
374
375 graphene_matrix_init_look_at (transformation->view_matrix,
376 transformation->camera_position, ¢er, &up);
377
378 current_passthrough =
379 gst_base_transform_is_passthrough (GST_BASE_TRANSFORM (transformation));
380 passthrough = transformation->xtranslation == 0.
381 && transformation->ytranslation == 0.
382 && transformation->ztranslation == 0. && transformation->xrotation == 0.
383 && transformation->yrotation == 0. && transformation->zrotation == 0.
384 && transformation->xscale == 1. && transformation->yscale == 1.
385 && gst_video_info_is_equal (&filter->in_info, &filter->out_info);
386 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (transformation),
387 passthrough);
388 if (current_passthrough != passthrough) {
389 gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (transformation));
390 }
391 }
392
393 graphene_matrix_multiply (transformation->model_matrix,
394 transformation->view_matrix, &modelview_matrix);
395 graphene_matrix_multiply (&modelview_matrix,
396 transformation->projection_matrix, transformation->mvp_matrix);
397
398 graphene_matrix_inverse (transformation->model_matrix,
399 transformation->inv_model_matrix);
400 graphene_matrix_inverse (transformation->view_matrix,
401 transformation->inv_view_matrix);
402 graphene_matrix_inverse (transformation->projection_matrix,
403 transformation->inv_projection_matrix);
404 }
405
406 static void
gst_gl_transformation_set_property(GObject * object,guint prop_id,const GValue * value,GParamSpec * pspec)407 gst_gl_transformation_set_property (GObject * object, guint prop_id,
408 const GValue * value, GParamSpec * pspec)
409 {
410 GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
411
412 switch (prop_id) {
413 case PROP_FOV:
414 filter->fov = g_value_get_float (value);
415 break;
416 case PROP_ORTHO:
417 filter->ortho = g_value_get_boolean (value);
418 break;
419 case PROP_TRANSLATION_X:
420 filter->xtranslation = g_value_get_float (value);
421 break;
422 case PROP_TRANSLATION_Y:
423 filter->ytranslation = g_value_get_float (value);
424 break;
425 case PROP_TRANSLATION_Z:
426 filter->ztranslation = g_value_get_float (value);
427 break;
428 case PROP_ROTATION_X:
429 filter->xrotation = g_value_get_float (value);
430 break;
431 case PROP_ROTATION_Y:
432 filter->yrotation = g_value_get_float (value);
433 break;
434 case PROP_ROTATION_Z:
435 filter->zrotation = g_value_get_float (value);
436 break;
437 case PROP_SCALE_X:
438 filter->xscale = g_value_get_float (value);
439 break;
440 case PROP_SCALE_Y:
441 filter->yscale = g_value_get_float (value);
442 break;
443 case PROP_PIVOT_X:
444 filter->xpivot = g_value_get_float (value);
445 break;
446 case PROP_PIVOT_Y:
447 filter->ypivot = g_value_get_float (value);
448 break;
449 case PROP_PIVOT_Z:
450 filter->zpivot = g_value_get_float (value);
451 break;
452 default:
453 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
454 break;
455 }
456 gst_gl_transformation_build_mvp (filter);
457 }
458
459 static void
gst_gl_transformation_get_property(GObject * object,guint prop_id,GValue * value,GParamSpec * pspec)460 gst_gl_transformation_get_property (GObject * object, guint prop_id,
461 GValue * value, GParamSpec * pspec)
462 {
463 GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
464
465 switch (prop_id) {
466 case PROP_FOV:
467 g_value_set_float (value, filter->fov);
468 break;
469 case PROP_ORTHO:
470 g_value_set_boolean (value, filter->ortho);
471 break;
472 case PROP_TRANSLATION_X:
473 g_value_set_float (value, filter->xtranslation);
474 break;
475 case PROP_TRANSLATION_Y:
476 g_value_set_float (value, filter->ytranslation);
477 break;
478 case PROP_TRANSLATION_Z:
479 g_value_set_float (value, filter->ztranslation);
480 break;
481 case PROP_ROTATION_X:
482 g_value_set_float (value, filter->xrotation);
483 break;
484 case PROP_ROTATION_Y:
485 g_value_set_float (value, filter->yrotation);
486 break;
487 case PROP_ROTATION_Z:
488 g_value_set_float (value, filter->zrotation);
489 break;
490 case PROP_SCALE_X:
491 g_value_set_float (value, filter->xscale);
492 break;
493 case PROP_SCALE_Y:
494 g_value_set_float (value, filter->yscale);
495 break;
496 case PROP_PIVOT_X:
497 g_value_set_float (value, filter->xpivot);
498 break;
499 case PROP_PIVOT_Y:
500 g_value_set_float (value, filter->ypivot);
501 break;
502 case PROP_PIVOT_Z:
503 g_value_set_float (value, filter->zpivot);
504 break;
505 case PROP_MVP:
506 /* FIXME: need to decompose this to support navigation events */
507 g_value_set_boxed (value, (gconstpointer) filter->mvp_matrix);
508 break;
509 default:
510 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
511 break;
512 }
513 }
514
515 static gboolean
gst_gl_transformation_set_caps(GstGLFilter * filter,GstCaps * incaps,GstCaps * outcaps)516 gst_gl_transformation_set_caps (GstGLFilter * filter, GstCaps * incaps,
517 GstCaps * outcaps)
518 {
519 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
520
521 transformation->aspect =
522 (gdouble) GST_VIDEO_INFO_WIDTH (&filter->out_info) /
523 (gdouble) GST_VIDEO_INFO_HEIGHT (&filter->out_info);
524
525 transformation->caps_change = TRUE;
526
527 gst_gl_transformation_build_mvp (transformation);
528
529 return TRUE;
530 }
531
532 static void
_intersect_plane_and_ray(graphene_plane_t * video_plane,graphene_ray_t * ray,graphene_point3d_t * result)533 _intersect_plane_and_ray (graphene_plane_t * video_plane, graphene_ray_t * ray,
534 graphene_point3d_t * result)
535 {
536 float t = graphene_ray_get_distance_to_plane (ray, video_plane);
537 GST_TRACE ("Calculated a distance of %f to the plane", t);
538 graphene_ray_get_position_at (ray, t, result);
539 }
540
541 static void
_screen_coord_to_world_ray(GstGLTransformation * transformation,float x,float y,graphene_ray_t * ray)542 _screen_coord_to_world_ray (GstGLTransformation * transformation, float x,
543 float y, graphene_ray_t * ray)
544 {
545 GstGLFilter *filter = GST_GL_FILTER (transformation);
546 gfloat w = (gfloat) GST_VIDEO_INFO_WIDTH (&filter->in_info);
547 gfloat h = (gfloat) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
548 graphene_vec3_t ray_eye_vec3, ray_world_dir, *ray_origin, *ray_direction;
549 graphene_vec3_t ray_ortho_dir;
550 graphene_point3d_t ray_clip, ray_eye;
551 graphene_vec2_t screen_coord;
552
553 /* GL is y-flipped. i.e. 0, 0 is the bottom left corner in screen space */
554 graphene_vec2_init (&screen_coord, (2. * x / w - 1.) / transformation->aspect,
555 1. - 2. * y / h);
556
557 graphene_point3d_init (&ray_clip, graphene_vec2_get_x (&screen_coord),
558 graphene_vec2_get_y (&screen_coord), -1.);
559 graphene_matrix_transform_point3d (transformation->inv_projection_matrix,
560 &ray_clip, &ray_eye);
561
562 graphene_vec3_init (&ray_eye_vec3, ray_eye.x, ray_eye.y, -1.);
563
564 if (transformation->ortho) {
565 graphene_vec3_init (&ray_ortho_dir, 0., 0., 1.);
566
567 ray_origin = &ray_eye_vec3;
568 ray_direction = &ray_ortho_dir;
569 } else {
570 graphene_matrix_transform_vec3 (transformation->inv_view_matrix,
571 &ray_eye_vec3, &ray_world_dir);
572 graphene_vec3_normalize (&ray_world_dir, &ray_world_dir);
573
574 ray_origin = transformation->camera_position;
575 ray_direction = &ray_world_dir;
576 }
577
578 graphene_ray_init_from_vec3 (ray, ray_origin, ray_direction);
579
580 GST_TRACE_OBJECT (transformation, "Calculated ray origin: " VEC3_FORMAT
581 " direction: " VEC3_FORMAT " from screen coordinates: " VEC2_FORMAT
582 " with %s projection",
583 VEC3_ARGS (ray_origin), VEC3_ARGS (ray_direction),
584 VEC2_ARGS (&screen_coord),
585 transformation->ortho ? "ortho" : "perspection");
586 }
587
588 static void
_init_world_video_plane(GstGLTransformation * transformation,graphene_plane_t * video_plane)589 _init_world_video_plane (GstGLTransformation * transformation,
590 graphene_plane_t * video_plane)
591 {
592 graphene_point3d_t bottom_left, bottom_right, top_left, top_right;
593 graphene_point3d_t world_bottom_left, world_bottom_right;
594 graphene_point3d_t world_top_left, world_top_right;
595
596 graphene_point3d_init (&top_left, -transformation->aspect, 1., 0.);
597 graphene_point3d_init (&top_right, transformation->aspect, 1., 0.);
598 graphene_point3d_init (&bottom_left, -transformation->aspect, -1., 0.);
599 graphene_point3d_init (&bottom_right, transformation->aspect, -1., 0.);
600
601 graphene_matrix_transform_point3d (transformation->model_matrix,
602 &bottom_left, &world_bottom_left);
603 graphene_matrix_transform_point3d (transformation->model_matrix,
604 &bottom_right, &world_bottom_right);
605 graphene_matrix_transform_point3d (transformation->model_matrix,
606 &top_left, &world_top_left);
607 graphene_matrix_transform_point3d (transformation->model_matrix,
608 &top_right, &world_top_right);
609
610 graphene_plane_init_from_points (video_plane, &world_bottom_left,
611 &world_top_right, &world_top_left);
612 }
613
614 static gboolean
_screen_coord_to_model_coord(GstGLTransformation * transformation,double x,double y,double * res_x,double * res_y)615 _screen_coord_to_model_coord (GstGLTransformation * transformation,
616 double x, double y, double *res_x, double *res_y)
617 {
618 GstGLFilter *filter = GST_GL_FILTER (transformation);
619 double w = (double) GST_VIDEO_INFO_WIDTH (&filter->in_info);
620 double h = (double) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
621 graphene_point3d_t world_point, model_coord;
622 graphene_plane_t video_plane;
623 graphene_ray_t ray;
624 double new_x, new_y;
625
626 _init_world_video_plane (transformation, &video_plane);
627 _screen_coord_to_world_ray (transformation, x, y, &ray);
628 _intersect_plane_and_ray (&video_plane, &ray, &world_point);
629 graphene_matrix_transform_point3d (transformation->inv_model_matrix,
630 &world_point, &model_coord);
631
632 /* ndc to pixels. We render the frame Y-flipped so need to unflip the
633 * y coordinate */
634 new_x = (model_coord.x + 1.) * w / 2;
635 new_y = (1. - model_coord.y) * h / 2;
636
637 if (new_x < 0. || new_x > w || new_y < 0. || new_y > h)
638 /* coords off video surface */
639 return FALSE;
640
641 GST_DEBUG_OBJECT (transformation, "converted %f,%f to %f,%f", x, y, new_x,
642 new_y);
643
644 if (res_x)
645 *res_x = new_x;
646 if (res_y)
647 *res_y = new_y;
648
649 return TRUE;
650 }
651
652 #if 0
653 /* debugging facilities for transforming vertices from model space to screen
654 * space */
655 static void
656 _ndc_to_viewport (GstGLTransformation * transformation, graphene_vec3_t * ndc,
657 int x, int y, int w, int h, float near, float far, graphene_vec3_t * result)
658 {
659 GstGLFilter *filter = GST_GL_FILTER (transformation);
660 /* center of the viewport */
661 int o_x = x + w / 2;
662 int o_y = y + h / 2;
663
664 graphene_vec3_init (result, graphene_vec3_get_x (ndc) * w / 2 + o_x,
665 graphene_vec3_get_y (ndc) * h / 2 + o_y,
666 (far - near) * graphene_vec3_get_z (ndc) / 2 + (far + near) / 2);
667 }
668
669 static void
670 _perspective_division (graphene_vec4_t * clip, graphene_vec3_t * result)
671 {
672 float w = graphene_vec4_get_w (clip);
673
674 graphene_vec3_init (result, graphene_vec4_get_x (clip) / w,
675 graphene_vec4_get_y (clip) / w, graphene_vec4_get_z (clip) / w);
676 }
677
678 static void
679 _vertex_to_screen_coord (GstGLTransformation * transformation,
680 graphene_vec4_t * vertex, graphene_vec3_t * view)
681 {
682 GstGLFilter *filter = GST_GL_FILTER (transformation);
683 gint w = GST_VIDEO_INFO_WIDTH (&filter->in_info);
684 gint h = GST_VIDEO_INFO_HEIGHT (&filter->in_info);
685 graphene_vec4_t clip;
686 graphene_vec3_t ndc;
687
688 graphene_matrix_transform_vec4 (transformation->mvp_matrix, vertex, &clip);
689 _perspective_division (&clip, &ndc);
690 _ndc_to_viewport (transformation, &ndc, 0, 0, w, h, 0., 1., view);
691 }
692 #endif
693
694 static gboolean
gst_gl_transformation_src_event(GstBaseTransform * trans,GstEvent * event)695 gst_gl_transformation_src_event (GstBaseTransform * trans, GstEvent * event)
696 {
697 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
698 GstStructure *structure;
699 gboolean ret;
700
701 GST_DEBUG_OBJECT (trans, "handling %s event", GST_EVENT_TYPE_NAME (event));
702
703 switch (GST_EVENT_TYPE (event)) {
704 case GST_EVENT_NAVIGATION:{
705 gdouble x, y;
706 event =
707 GST_EVENT (gst_mini_object_make_writable (GST_MINI_OBJECT (event)));
708
709 structure = (GstStructure *) gst_event_get_structure (event);
710 if (gst_structure_get_double (structure, "pointer_x", &x) &&
711 gst_structure_get_double (structure, "pointer_y", &y)) {
712 gdouble new_x, new_y;
713
714 if (!_screen_coord_to_model_coord (transformation, x, y, &new_x,
715 &new_y)) {
716 gst_event_unref (event);
717 return TRUE;
718 }
719
720 gst_structure_set (structure, "pointer_x", G_TYPE_DOUBLE, new_x,
721 "pointer_y", G_TYPE_DOUBLE, new_y, NULL);
722 }
723 break;
724 }
725 default:
726 break;
727 }
728
729 ret = GST_BASE_TRANSFORM_CLASS (parent_class)->src_event (trans, event);
730
731 return ret;
732 }
733
734 static gboolean
gst_gl_transformation_filter_meta(GstBaseTransform * trans,GstQuery * query,GType api,const GstStructure * params)735 gst_gl_transformation_filter_meta (GstBaseTransform * trans, GstQuery * query,
736 GType api, const GstStructure * params)
737 {
738 if (api == GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE)
739 return TRUE;
740
741 if (api == GST_GL_SYNC_META_API_TYPE)
742 return TRUE;
743
744 return FALSE;
745 }
746
747 static gboolean
gst_gl_transformation_decide_allocation(GstBaseTransform * trans,GstQuery * query)748 gst_gl_transformation_decide_allocation (GstBaseTransform * trans,
749 GstQuery * query)
750 {
751 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
752
753 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->decide_allocation (trans,
754 query))
755 return FALSE;
756
757 if (gst_query_find_allocation_meta (query,
758 GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, NULL)) {
759 transformation->downstream_supports_affine_meta = TRUE;
760 } else {
761 transformation->downstream_supports_affine_meta = FALSE;
762 }
763
764 return TRUE;
765 }
766
767 static void
gst_gl_transformation_gl_stop(GstGLBaseFilter * base_filter)768 gst_gl_transformation_gl_stop (GstGLBaseFilter * base_filter)
769 {
770 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
771 const GstGLFuncs *gl = base_filter->context->gl_vtable;
772
773 if (transformation->vao) {
774 gl->DeleteVertexArrays (1, &transformation->vao);
775 transformation->vao = 0;
776 }
777
778 if (transformation->vertex_buffer) {
779 gl->DeleteBuffers (1, &transformation->vertex_buffer);
780 transformation->vertex_buffer = 0;
781 }
782
783 if (transformation->vbo_indices) {
784 gl->DeleteBuffers (1, &transformation->vbo_indices);
785 transformation->vbo_indices = 0;
786 }
787
788 if (transformation->shader) {
789 gst_object_unref (transformation->shader);
790 transformation->shader = NULL;
791 }
792
793 GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base_filter);
794 }
795
796 static gboolean
gst_gl_transformation_gl_start(GstGLBaseFilter * base_filter)797 gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter)
798 {
799 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
800
801 if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base_filter))
802 return FALSE;
803
804 if (gst_gl_context_get_gl_api (base_filter->context)) {
805 gchar *frag_str;
806 gboolean ret;
807
808 frag_str =
809 gst_gl_shader_string_fragment_get_default (base_filter->context,
810 GST_GLSL_VERSION_NONE,
811 GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY);
812
813 /* blocking call, wait until the opengl thread has compiled the shader */
814 ret = gst_gl_context_gen_shader (base_filter->context,
815 gst_gl_shader_string_vertex_mat4_vertex_transform,
816 frag_str, &transformation->shader);
817
818 g_free (frag_str);
819
820 return ret;
821 }
822 return TRUE;
823 }
824
825 static GstFlowReturn
gst_gl_transformation_prepare_output_buffer(GstBaseTransform * trans,GstBuffer * inbuf,GstBuffer ** outbuf)826 gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
827 GstBuffer * inbuf, GstBuffer ** outbuf)
828 {
829 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
830 GstGLFilter *filter = GST_GL_FILTER (trans);
831
832 if (transformation->downstream_supports_affine_meta &&
833 gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
834 GstVideoAffineTransformationMeta *af_meta;
835 graphene_matrix_t upstream_matrix, tmp, tmp2, inv_aspect, yflip;
836 float upstream[16], downstream[16];
837
838 *outbuf = gst_buffer_make_writable (inbuf);
839
840 af_meta = gst_buffer_get_video_affine_transformation_meta (inbuf);
841 if (!af_meta)
842 af_meta = gst_buffer_add_video_affine_transformation_meta (*outbuf);
843
844 GST_LOG_OBJECT (trans, "applying transformation to existing affine "
845 "transformation meta");
846
847 gst_gl_get_affine_transformation_meta_as_ndc (af_meta, upstream);
848
849 /* apply the transformation to the existing affine meta */
850 graphene_matrix_init_from_float (&upstream_matrix, upstream);
851 graphene_matrix_init_scale (&inv_aspect, transformation->aspect, -1., 1.);
852 graphene_matrix_init_scale (&yflip, 1., -1., 1.);
853
854 /* invert the aspect effects */
855 graphene_matrix_multiply (&upstream_matrix, &inv_aspect, &tmp2);
856 /* apply the transformation */
857 graphene_matrix_multiply (&tmp2, transformation->mvp_matrix, &tmp);
858 /* and undo yflip */
859 graphene_matrix_multiply (&tmp, &yflip, &tmp2);
860
861 graphene_matrix_to_float (&tmp2, downstream);
862 gst_gl_set_affine_transformation_meta_from_ndc (af_meta, downstream);
863
864 return GST_FLOW_OK;
865 }
866
867 return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (trans,
868 inbuf, outbuf);
869 }
870
871 static gboolean
gst_gl_transformation_filter(GstGLFilter * filter,GstBuffer * inbuf,GstBuffer * outbuf)872 gst_gl_transformation_filter (GstGLFilter * filter,
873 GstBuffer * inbuf, GstBuffer * outbuf)
874 {
875 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
876
877 if (transformation->downstream_supports_affine_meta &&
878 gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
879 return TRUE;
880 } else {
881 return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
882 }
883 }
884
885 static gboolean
gst_gl_transformation_filter_texture(GstGLFilter * filter,GstGLMemory * in_tex,GstGLMemory * out_tex)886 gst_gl_transformation_filter_texture (GstGLFilter * filter,
887 GstGLMemory * in_tex, GstGLMemory * out_tex)
888 {
889 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
890
891 transformation->in_tex = in_tex;
892
893 gst_gl_framebuffer_draw_to_texture (filter->fbo, out_tex,
894 (GstGLFramebufferFunc) gst_gl_transformation_callback, transformation);
895
896 return TRUE;
897 }
898
899 static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
900
901 static void
_upload_vertices(GstGLTransformation * transformation)902 _upload_vertices (GstGLTransformation * transformation)
903 {
904 const GstGLFuncs *gl =
905 GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
906
907 /* *INDENT-OFF* */
908 GLfloat vertices[] = {
909 -transformation->aspect, -1.0, 0.0, 1.0, 0.0, 0.0,
910 transformation->aspect, -1.0, 0.0, 1.0, 1.0, 0.0,
911 transformation->aspect, 1.0, 0.0, 1.0, 1.0, 1.0,
912 -transformation->aspect, 1.0, 0.0, 1.0, 0.0, 1.0,
913 };
914 /* *INDENT-ON* */
915
916 gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
917
918 gl->BufferData (GL_ARRAY_BUFFER, 4 * 6 * sizeof (GLfloat), vertices,
919 GL_STATIC_DRAW);
920 }
921
922 static void
_bind_buffer(GstGLTransformation * transformation)923 _bind_buffer (GstGLTransformation * transformation)
924 {
925 const GstGLFuncs *gl =
926 GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
927
928 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
929 gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
930
931 /* Load the vertex position */
932 gl->VertexAttribPointer (transformation->attr_position, 4, GL_FLOAT,
933 GL_FALSE, 6 * sizeof (GLfloat), (void *) 0);
934
935 /* Load the texture coordinate */
936 gl->VertexAttribPointer (transformation->attr_texture, 2, GL_FLOAT, GL_FALSE,
937 6 * sizeof (GLfloat), (void *) (4 * sizeof (GLfloat)));
938
939 gl->EnableVertexAttribArray (transformation->attr_position);
940 gl->EnableVertexAttribArray (transformation->attr_texture);
941 }
942
943 static void
_unbind_buffer(GstGLTransformation * transformation)944 _unbind_buffer (GstGLTransformation * transformation)
945 {
946 const GstGLFuncs *gl =
947 GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
948
949 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
950 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
951
952 gl->DisableVertexAttribArray (transformation->attr_position);
953 gl->DisableVertexAttribArray (transformation->attr_texture);
954 }
955
956 static gboolean
gst_gl_transformation_callback(gpointer stuff)957 gst_gl_transformation_callback (gpointer stuff)
958 {
959 GstGLFilter *filter = GST_GL_FILTER (stuff);
960 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
961 GstGLFuncs *gl = GST_GL_BASE_FILTER (filter)->context->gl_vtable;
962
963 GLfloat temp_matrix[16];
964
965 gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
966 gl->BindTexture (GL_TEXTURE_2D, 0);
967
968 gl->ClearColor (0.f, 0.f, 0.f, 0.f);
969 gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
970
971 gst_gl_shader_use (transformation->shader);
972
973 gl->ActiveTexture (GL_TEXTURE0);
974 gl->BindTexture (GL_TEXTURE_2D, transformation->in_tex->tex_id);
975 gst_gl_shader_set_uniform_1i (transformation->shader, "texture", 0);
976
977 graphene_matrix_to_float (transformation->mvp_matrix, temp_matrix);
978 gst_gl_shader_set_uniform_matrix_4fv (transformation->shader,
979 "u_transformation", 1, GL_FALSE, temp_matrix);
980
981 if (!transformation->vertex_buffer) {
982 transformation->attr_position =
983 gst_gl_shader_get_attribute_location (transformation->shader,
984 "a_position");
985
986 transformation->attr_texture =
987 gst_gl_shader_get_attribute_location (transformation->shader,
988 "a_texcoord");
989
990 if (gl->GenVertexArrays) {
991 gl->GenVertexArrays (1, &transformation->vao);
992 gl->BindVertexArray (transformation->vao);
993 }
994
995 gl->GenBuffers (1, &transformation->vertex_buffer);
996
997 gl->GenBuffers (1, &transformation->vbo_indices);
998 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
999 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
1000 GL_STATIC_DRAW);
1001
1002 transformation->caps_change = TRUE;
1003 }
1004
1005 if (gl->GenVertexArrays)
1006 gl->BindVertexArray (transformation->vao);
1007
1008 if (transformation->caps_change)
1009 _upload_vertices (transformation);
1010 _bind_buffer (transformation);
1011
1012 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
1013
1014 if (gl->GenVertexArrays)
1015 gl->BindVertexArray (0);
1016 else
1017 _unbind_buffer (transformation);
1018
1019 gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
1020 transformation->caps_change = FALSE;
1021
1022 return TRUE;
1023 }
1024