1/* 2 * GStreamer 3 * Copyright (C) 2015 Matthew Waters <matthew@centricular.com> 4 * 5 * This library is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU Library General Public 7 * License as published by the Free Software Foundation; either 8 * version 2 of the License, or (at your option) any later version. 9 * 10 * This library is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * Library General Public License for more details. 14 * 15 * You should have received a copy of the GNU Library General Public 16 * License along with this library; if not, write to the 17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, 18 * Boston, MA 02110-1301, USA. 19 */ 20 21/** 22 * SECTION:element-caopengllayersink 23 * 24 * caopengllayersink renders incoming video frames to CAOpenGLLayer that 25 * can be retrieved through the layer property and placed in the Core 26 * Animation render tree. 27 */ 28 29#ifdef HAVE_CONFIG_H 30#include "config.h" 31#endif 32 33#include "gstglelements.h" 34#include "caopengllayersink.h" 35#include "gstglsinkbin.h" 36#include <QuartzCore/QuartzCore.h> 37 38GST_DEBUG_CATEGORY (gst_debug_ca_sink); 39#define GST_CAT_DEFAULT gst_debug_ca_sink 40 41typedef GstGLSinkBin GstCAOpenGLLayerSinkBin; 42typedef GstGLSinkBinClass GstCAOpenGLLayerSinkBinClass; 43 44G_DEFINE_TYPE (GstCAOpenGLLayerSinkBin, gst_ca_opengl_layer_sink_bin, 45 GST_TYPE_GL_SINK_BIN); 46 47#define _do_init \ 48 gl_element_init (plugin); 49GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (caopengllayersink, "caopengllayersink", 50 GST_RANK_NONE, GST_TYPE_CA_OPENGL_LAYER_SINK, _do_init); 51 52enum 53{ 54 PROP_BIN_0, 55 PROP_BIN_QOS, 56 PROP_BIN_FORCE_ASPECT_RATIO, 57 PROP_BIN_LAST_SAMPLE, 58 PROP_BIN_LAYER, 59}; 60 61static void 62_on_notify_layer (GObject * object, GParamSpec *pspec, gpointer user_data) 63{ 64 GstCAOpenGLLayerSinkBin *self = user_data; 65 66 g_object_notify (G_OBJECT (self), "layer"); 67} 68 69static void 70gst_ca_opengl_layer_sink_bin_set_property (GObject * object, guint prop_id, 71 const GValue * value, GParamSpec * param_spec) 72{ 73 g_object_set_property (G_OBJECT (GST_GL_SINK_BIN (object)->sink), 74 param_spec->name, value); 75} 76 77static void 78gst_ca_opengl_layer_sink_bin_get_property (GObject * object, guint prop_id, 79 GValue * value, GParamSpec * param_spec) 80{ 81 g_object_get_property (G_OBJECT (GST_GL_SINK_BIN (object)->sink), 82 param_spec->name, value); 83} 84 85static void 86gst_ca_opengl_layer_sink_bin_init (GstCAOpenGLLayerSinkBin * self) 87{ 88 gpointer *sink = g_object_new (GST_TYPE_CA_OPENGL_LAYER_SINK, NULL); 89 90 g_signal_connect (sink, "notify::layer", G_CALLBACK (_on_notify_layer), self); 91 92 gst_gl_sink_bin_finish_init_with_element (GST_GL_SINK_BIN (self), 93 GST_ELEMENT (sink)); 94} 95 96static void 97gst_ca_opengl_layer_sink_bin_class_init (GstCAOpenGLLayerSinkBinClass * klass) 98{ 99 GObjectClass *gobject_class = G_OBJECT_CLASS (klass); 100 101 gobject_class->get_property = gst_ca_opengl_layer_sink_bin_get_property; 102 gobject_class->set_property = gst_ca_opengl_layer_sink_bin_set_property; 103 104 g_object_class_install_property (gobject_class, PROP_BIN_FORCE_ASPECT_RATIO, 105 g_param_spec_boolean ("force-aspect-ratio", 106 "Force aspect ratio", 107 "When enabled, scaling will respect original aspect ratio", TRUE, 108 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); 109 110 g_object_class_install_property (gobject_class, PROP_BIN_LAST_SAMPLE, 111 g_param_spec_boxed ("last-sample", "Last Sample", 112 "The last sample received in the sink", GST_TYPE_SAMPLE, 113 G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); 114 115 g_object_class_install_property (gobject_class, PROP_BIN_LAYER, 116 g_param_spec_pointer ("layer", "CAOpenGLLayer", 117 "OpenGL Core Animation layer", 118 G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); 119 120 g_object_class_install_property (gobject_class, PROP_BIN_QOS, 121 g_param_spec_boolean ("qos", "Quality of Service", 122 "Generate Quality-of-Service events upstream", TRUE, 123 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); 124} 125 126#define GST_CA_OPENGL_LAYER_SINK_GET_LOCK(glsink) \ 127 (GST_CA_OPENGL_LAYER_SINK(glsink)->drawing_lock) 128#define GST_CA_OPENGL_LAYER_SINK_LOCK(glsink) \ 129 (g_mutex_lock(&GST_CA_OPENGL_LAYER_SINK_GET_LOCK (glsink))) 130#define GST_CA_OPENGL_LAYER_SINK_UNLOCK(glsink) \ 131 (g_mutex_unlock(&GST_CA_OPENGL_LAYER_SINK_GET_LOCK (glsink))) 132 133#define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0)) 134#define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1)) 135#define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0)) 136#define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0)) 137#define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0)) 138 139#define SUPPORTED_GL_APIS GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3 140 141static void gst_ca_opengl_layer_sink_thread_init_redisplay (GstCAOpenGLLayerSink * ca_sink); 142static void gst_ca_opengl_layer_sink_cleanup_glthread (GstCAOpenGLLayerSink * ca_sink); 143static void gst_ca_opengl_layer_sink_on_resize (GstCAOpenGLLayerSink * ca_sink, 144 gint width, gint height); 145static void gst_ca_opengl_layer_sink_on_draw (GstCAOpenGLLayerSink * ca_sink); 146 147static void gst_ca_opengl_layer_sink_finalize (GObject * object); 148static void gst_ca_opengl_layer_sink_set_property (GObject * object, guint prop_id, 149 const GValue * value, GParamSpec * param_spec); 150static void gst_ca_opengl_layer_sink_get_property (GObject * object, guint prop_id, 151 GValue * value, GParamSpec * param_spec); 152 153static gboolean gst_ca_opengl_layer_sink_stop (GstBaseSink * bsink); 154 155static gboolean gst_ca_opengl_layer_sink_query (GstBaseSink * bsink, GstQuery * query); 156static void gst_ca_opengl_layer_sink_set_context (GstElement * element, 157 GstContext * context); 158 159static GstStateChangeReturn gst_ca_opengl_layer_sink_change_state (GstElement * 160 element, GstStateChange transition); 161 162static void gst_ca_opengl_layer_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, 163 GstClockTime * start, GstClockTime * end); 164static gboolean gst_ca_opengl_layer_sink_set_caps (GstBaseSink * bsink, GstCaps * caps); 165static GstFlowReturn gst_ca_opengl_layer_sink_prepare (GstBaseSink * bsink, 166 GstBuffer * buf); 167static GstFlowReturn gst_ca_opengl_layer_sink_show_frame (GstVideoSink * bsink, 168 GstBuffer * buf); 169static gboolean gst_ca_opengl_layer_sink_propose_allocation (GstBaseSink * bsink, 170 GstQuery * query); 171 172static GstStaticPadTemplate gst_ca_opengl_layer_sink_template = 173 GST_STATIC_PAD_TEMPLATE ("sink", 174 GST_PAD_SINK, 175 GST_PAD_ALWAYS, 176 GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " 177 "format = (string) RGBA, " 178 "width = " GST_VIDEO_SIZE_RANGE ", " 179 "height = " GST_VIDEO_SIZE_RANGE ", " 180 "framerate = " GST_VIDEO_FPS_RANGE "," 181 "texture-target = (string) 2D") 182 ); 183 184enum 185{ 186 PROP_0, 187 PROP_FORCE_ASPECT_RATIO, 188 PROP_CONTEXT, 189 PROP_LAYER, 190}; 191 192#define gst_ca_opengl_layer_sink_parent_class parent_class 193G_DEFINE_TYPE_WITH_CODE (GstCAOpenGLLayerSink, gst_ca_opengl_layer_sink, 194 GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT (gst_debug_ca_sink, 195 "caopengllayersink", 0, "CAOpenGLLayer Video Sink")); 196 197static void 198gst_ca_opengl_layer_sink_class_init (GstCAOpenGLLayerSinkClass * klass) 199{ 200 GObjectClass *gobject_class; 201 GstElementClass *gstelement_class; 202 GstBaseSinkClass *gstbasesink_class; 203 GstVideoSinkClass *gstvideosink_class; 204 GstElementClass *element_class; 205 206 gobject_class = (GObjectClass *) klass; 207 gstelement_class = (GstElementClass *) klass; 208 gstbasesink_class = (GstBaseSinkClass *) klass; 209 gstvideosink_class = (GstVideoSinkClass *) klass; 210 element_class = GST_ELEMENT_CLASS (klass); 211 212 gobject_class->set_property = gst_ca_opengl_layer_sink_set_property; 213 gobject_class->get_property = gst_ca_opengl_layer_sink_get_property; 214 215 g_object_class_install_property (gobject_class, PROP_FORCE_ASPECT_RATIO, 216 g_param_spec_boolean ("force-aspect-ratio", 217 "Force aspect ratio", 218 "When enabled, scaling will respect original aspect ratio", TRUE, 219 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); 220 221 g_object_class_install_property (gobject_class, PROP_CONTEXT, 222 g_param_spec_object ("context", 223 "OpenGL context", 224 "Get OpenGL context", 225 GST_TYPE_GL_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); 226 227 g_object_class_install_property (gobject_class, PROP_LAYER, 228 g_param_spec_pointer ("layer", "CAOpenGLLayer", 229 "OpenGL Core Animation layer", 230 G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); 231 232 gst_element_class_set_metadata (element_class, "CAOpenGLLayer video sink", 233 "Sink/Video", "A video sink based on CAOpenGLLayer", 234 "Matthew Waters <matthew@centricular.com>"); 235 236 gst_element_class_add_static_pad_template (element_class, &gst_ca_opengl_layer_sink_template); 237 238 gobject_class->finalize = gst_ca_opengl_layer_sink_finalize; 239 240 gstelement_class->change_state = gst_ca_opengl_layer_sink_change_state; 241 gstelement_class->set_context = gst_ca_opengl_layer_sink_set_context; 242 gstbasesink_class->query = GST_DEBUG_FUNCPTR (gst_ca_opengl_layer_sink_query); 243 gstbasesink_class->set_caps = gst_ca_opengl_layer_sink_set_caps; 244 gstbasesink_class->get_times = gst_ca_opengl_layer_sink_get_times; 245 gstbasesink_class->prepare = gst_ca_opengl_layer_sink_prepare; 246 gstbasesink_class->propose_allocation = gst_ca_opengl_layer_sink_propose_allocation; 247 gstbasesink_class->stop = gst_ca_opengl_layer_sink_stop; 248 249 gstvideosink_class->show_frame = 250 GST_DEBUG_FUNCPTR (gst_ca_opengl_layer_sink_show_frame); 251} 252 253static void 254gst_ca_opengl_layer_sink_init (GstCAOpenGLLayerSink * ca_sink) 255{ 256 ca_sink->display = NULL; 257 ca_sink->keep_aspect_ratio = TRUE; 258 ca_sink->stored_buffer = NULL; 259 ca_sink->redisplay_texture = 0; 260 261 g_mutex_init (&ca_sink->drawing_lock); 262} 263 264static void 265gst_ca_opengl_layer_sink_finalize (GObject * object) 266{ 267 GstCAOpenGLLayerSink *ca_sink; 268 269 g_return_if_fail (GST_IS_CA_OPENGL_LAYER_SINK (object)); 270 271 ca_sink = GST_CA_OPENGL_LAYER_SINK (object); 272 273 g_mutex_clear (&ca_sink->drawing_lock); 274 275 if (ca_sink->layer) { 276 CFRelease(ca_sink->layer); 277 ca_sink->layer = NULL; 278 } 279 280 GST_DEBUG ("finalized"); 281 G_OBJECT_CLASS (parent_class)->finalize (object); 282} 283 284static void 285gst_ca_opengl_layer_sink_set_property (GObject * object, guint prop_id, 286 const GValue * value, GParamSpec * pspec) 287{ 288 GstCAOpenGLLayerSink *ca_sink; 289 290 g_return_if_fail (GST_IS_CA_OPENGL_LAYER_SINK (object)); 291 292 ca_sink = GST_CA_OPENGL_LAYER_SINK (object); 293 294 switch (prop_id) { 295 case PROP_FORCE_ASPECT_RATIO: 296 { 297 ca_sink->keep_aspect_ratio = g_value_get_boolean (value); 298 break; 299 } 300 default: 301 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 302 break; 303 } 304} 305 306static void 307gst_ca_opengl_layer_sink_get_property (GObject * object, guint prop_id, 308 GValue * value, GParamSpec * pspec) 309{ 310 GstCAOpenGLLayerSink *ca_sink; 311 312 g_return_if_fail (GST_IS_CA_OPENGL_LAYER_SINK (object)); 313 314 ca_sink = GST_CA_OPENGL_LAYER_SINK (object); 315 316 switch (prop_id) { 317 case PROP_FORCE_ASPECT_RATIO: 318 g_value_set_boolean (value, ca_sink->keep_aspect_ratio); 319 break; 320 case PROP_CONTEXT: 321 g_value_set_object (value, ca_sink->context); 322 break; 323 case PROP_LAYER: 324 g_value_set_pointer (value, ca_sink->layer); 325 break; 326 default: 327 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 328 break; 329 } 330} 331 332static void 333_create_layer (gpointer data) 334{ 335 GstCAOpenGLLayerSink *ca_sink = data; 336 id layer; 337 338 if (!ca_sink->layer) { 339 layer = [[NSClassFromString(@"GstGLCAOpenGLLayer") alloc] 340 initWithGstGLContext:ca_sink->context]; 341 342 ca_sink->layer = (__bridge_retained gpointer)layer; 343 [layer setDrawCallback:(GstGLWindowCB)gst_ca_opengl_layer_sink_on_draw 344 data:ca_sink notify:NULL]; 345 [layer setResizeCallback:(GstGLWindowResizeCB)gst_ca_opengl_layer_sink_on_resize 346 data:ca_sink notify:NULL]; 347 g_object_notify (G_OBJECT (ca_sink), "layer"); 348 } 349} 350 351static void 352_invoke_on_main (GstGLWindowCB func, gpointer data) 353{ 354 if ([NSThread isMainThread]) { 355 func (data); 356 } else { 357 dispatch_sync (dispatch_get_main_queue (), ^{ 358 func (data); 359 }); 360 } 361} 362 363static gboolean 364_ensure_gl_setup (GstCAOpenGLLayerSink * ca_sink) 365{ 366 GError *error = NULL; 367 368 if (!gst_gl_ensure_element_data (ca_sink, &ca_sink->display, 369 &ca_sink->other_context)) 370 return FALSE; 371 372 gst_gl_display_filter_gl_api (ca_sink->display, SUPPORTED_GL_APIS); 373 374 if (!ca_sink->context) { 375 if (!gst_gl_display_create_context (ca_sink->display, 376 ca_sink->other_context, &ca_sink->context, &error)) { 377 goto context_error; 378 } 379 } 380 381 if (!ca_sink->layer) 382 _invoke_on_main ((GstGLWindowCB) _create_layer, ca_sink); 383 384 return TRUE; 385 386context_error: 387 { 388 GST_ELEMENT_ERROR (ca_sink, RESOURCE, NOT_FOUND, ("%s", error->message), 389 (NULL)); 390 gst_object_unref (ca_sink->context); 391 ca_sink->context = NULL; 392 return FALSE; 393 } 394} 395 396static gboolean 397gst_ca_opengl_layer_sink_query (GstBaseSink * bsink, GstQuery * query) 398{ 399 GstCAOpenGLLayerSink *ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 400 401 switch (GST_QUERY_TYPE (query)) { 402 case GST_QUERY_CONTEXT: 403 { 404 if (gst_gl_handle_context_query ((GstElement *) ca_sink, query, 405 ca_sink->display, ca_sink->context, ca_sink->other_context)) 406 return TRUE; 407 break; 408 } 409 case GST_QUERY_DRAIN: 410 { 411 GstBuffer *buf = NULL; 412 413 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 414 ca_sink->redisplay_texture = 0; 415 buf = ca_sink->stored_buffer; 416 ca_sink->stored_buffer = NULL; 417 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 418 419 if (buf) 420 gst_buffer_unref (buf); 421 422 gst_buffer_replace (&ca_sink->next_buffer, NULL); 423 gst_buffer_replace (&ca_sink->next_sync, NULL); 424 425 break; 426 } 427 default: 428 break; 429 } 430 431 return GST_BASE_SINK_CLASS (parent_class)->query (bsink, query); 432} 433 434static gboolean 435gst_ca_opengl_layer_sink_stop (GstBaseSink * bsink) 436{ 437 GstCAOpenGLLayerSink *ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 438 439 if (ca_sink->gl_caps) { 440 gst_caps_unref (ca_sink->gl_caps); 441 ca_sink->gl_caps = NULL; 442 } 443 444 return TRUE; 445} 446 447static void 448gst_ca_opengl_layer_sink_set_context (GstElement * element, GstContext * context) 449{ 450 GstCAOpenGLLayerSink *ca_sink = GST_CA_OPENGL_LAYER_SINK (element); 451 452 gst_gl_handle_set_context (element, context, &ca_sink->display, 453 &ca_sink->other_context); 454 455 if (ca_sink->display) 456 gst_gl_display_filter_gl_api (ca_sink->display, SUPPORTED_GL_APIS); 457 458 GST_ELEMENT_CLASS (parent_class)->set_context (element, context); 459} 460 461static GstStateChangeReturn 462gst_ca_opengl_layer_sink_change_state (GstElement * element, GstStateChange transition) 463{ 464 GstCAOpenGLLayerSink *ca_sink; 465 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; 466 467 GST_DEBUG ("changing state: %s => %s", 468 gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), 469 gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); 470 471 ca_sink = GST_CA_OPENGL_LAYER_SINK (element); 472 473 switch (transition) { 474 case GST_STATE_CHANGE_NULL_TO_READY: 475 _ensure_gl_setup (ca_sink); 476 break; 477 case GST_STATE_CHANGE_READY_TO_PAUSED: 478 break; 479 case GST_STATE_CHANGE_PAUSED_TO_PLAYING: 480 break; 481 default: 482 break; 483 } 484 485 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); 486 if (ret == GST_STATE_CHANGE_FAILURE) 487 return ret; 488 489 switch (transition) { 490 case GST_STATE_CHANGE_PLAYING_TO_PAUSED: 491 break; 492 case GST_STATE_CHANGE_PAUSED_TO_READY: 493 { 494 /* mark the redisplay_texture as unavailable (=0) 495 * to avoid drawing 496 */ 497 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 498 ca_sink->redisplay_texture = 0; 499 500 gst_buffer_replace (&ca_sink->stored_sync, NULL); 501 502 if (ca_sink->stored_buffer) { 503 gst_buffer_unref (ca_sink->stored_buffer); 504 ca_sink->stored_buffer = NULL; 505 } 506 gst_buffer_replace (&ca_sink->next_buffer, NULL); 507 gst_buffer_replace (&ca_sink->next_sync, NULL); 508 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 509 510 GST_VIDEO_SINK_WIDTH (ca_sink) = 1; 511 GST_VIDEO_SINK_HEIGHT (ca_sink) = 1; 512 if (ca_sink->context) { 513 gst_object_unref (ca_sink->context); 514 ca_sink->context = NULL; 515 } 516 517 if (ca_sink->display) { 518 gst_object_unref (ca_sink->display); 519 ca_sink->display = NULL; 520 } 521 break; 522 } 523 case GST_STATE_CHANGE_READY_TO_NULL: 524 if (ca_sink->layer) { 525 CFRelease(ca_sink->layer); 526 ca_sink->layer = NULL; 527 } 528 break; 529 default: 530 break; 531 } 532 533 return ret; 534} 535 536static void 537gst_ca_opengl_layer_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, 538 GstClockTime * start, GstClockTime * end) 539{ 540 GstCAOpenGLLayerSink *ca_sink; 541 542 ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 543 544 if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { 545 *start = GST_BUFFER_TIMESTAMP (buf); 546 if (GST_BUFFER_DURATION_IS_VALID (buf)) 547 *end = *start + GST_BUFFER_DURATION (buf); 548 else { 549 if (GST_VIDEO_INFO_FPS_N (&ca_sink->info) > 0) { 550 *end = *start + 551 gst_util_uint64_scale_int (GST_SECOND, 552 GST_VIDEO_INFO_FPS_D (&ca_sink->info), 553 GST_VIDEO_INFO_FPS_N (&ca_sink->info)); 554 } 555 } 556 } 557} 558 559static gboolean 560gst_ca_opengl_layer_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) 561{ 562 GstCAOpenGLLayerSink *ca_sink; 563 gint width; 564 gint height; 565 gboolean ok; 566 gint par_n, par_d; 567 gint display_par_n, display_par_d; 568 guint display_ratio_num, display_ratio_den; 569 GstVideoInfo vinfo; 570 571 GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps); 572 573 ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 574 575 ok = gst_video_info_from_caps (&vinfo, caps); 576 if (!ok) 577 return FALSE; 578 579 width = GST_VIDEO_INFO_WIDTH (&vinfo); 580 height = GST_VIDEO_INFO_HEIGHT (&vinfo); 581 582 par_n = GST_VIDEO_INFO_PAR_N (&vinfo); 583 par_d = GST_VIDEO_INFO_PAR_D (&vinfo); 584 585 if (!par_n) 586 par_n = 1; 587 588 display_par_n = 1; 589 display_par_d = 1; 590 591 ok = gst_video_calculate_display_ratio (&display_ratio_num, 592 &display_ratio_den, width, height, par_n, par_d, display_par_n, 593 display_par_d); 594 595 if (!ok) 596 return FALSE; 597 598 GST_TRACE ("PAR: %u/%u DAR:%u/%u", par_n, par_d, display_par_n, 599 display_par_d); 600 601 if (height % display_ratio_den == 0) { 602 GST_DEBUG ("keeping video height"); 603 GST_VIDEO_SINK_WIDTH (ca_sink) = (guint) 604 gst_util_uint64_scale_int (height, display_ratio_num, 605 display_ratio_den); 606 GST_VIDEO_SINK_HEIGHT (ca_sink) = height; 607 } else if (width % display_ratio_num == 0) { 608 GST_DEBUG ("keeping video width"); 609 GST_VIDEO_SINK_WIDTH (ca_sink) = width; 610 GST_VIDEO_SINK_HEIGHT (ca_sink) = (guint) 611 gst_util_uint64_scale_int (width, display_ratio_den, display_ratio_num); 612 } else { 613 GST_DEBUG ("approximating while keeping video height"); 614 GST_VIDEO_SINK_WIDTH (ca_sink) = (guint) 615 gst_util_uint64_scale_int (height, display_ratio_num, 616 display_ratio_den); 617 GST_VIDEO_SINK_HEIGHT (ca_sink) = height; 618 } 619 GST_DEBUG ("scaling to %dx%d", GST_VIDEO_SINK_WIDTH (ca_sink), 620 GST_VIDEO_SINK_HEIGHT (ca_sink)); 621 622 ca_sink->info = vinfo; 623 if (!_ensure_gl_setup (ca_sink)) 624 return FALSE; 625 626 ca_sink->caps_change = TRUE; 627 628 return TRUE; 629} 630 631static GstFlowReturn 632gst_ca_opengl_layer_sink_prepare (GstBaseSink * bsink, GstBuffer * buf) 633{ 634 GstCAOpenGLLayerSink *ca_sink; 635 GstBuffer *next_sync, *old_sync, *old_buffer; 636 GstVideoFrame gl_frame; 637 GstGLSyncMeta *sync_meta; 638 639 ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 640 641 GST_TRACE ("preparing buffer:%p", buf); 642 643 if (GST_VIDEO_SINK_WIDTH (ca_sink) < 1 || 644 GST_VIDEO_SINK_HEIGHT (ca_sink) < 1) { 645 return GST_FLOW_NOT_NEGOTIATED; 646 } 647 648 if (!_ensure_gl_setup (ca_sink)) 649 return GST_FLOW_NOT_NEGOTIATED; 650 651 if (!gst_video_frame_map (&gl_frame, &ca_sink->info, buf, 652 GST_MAP_READ | GST_MAP_GL)) { 653 goto upload_failed; 654 } 655 656 ca_sink->next_tex = *(guint *) gl_frame.data[0]; 657 658 next_sync = gst_buffer_new (); 659 sync_meta = gst_buffer_add_gl_sync_meta (ca_sink->context, next_sync); 660 gst_gl_sync_meta_set_sync_point (sync_meta, ca_sink->context); 661 662 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 663 ca_sink->next_tex = *(guint *) gl_frame.data[0]; 664 665 old_buffer = ca_sink->next_buffer; 666 ca_sink->next_buffer = gst_buffer_ref (buf); 667 668 old_sync = ca_sink->next_sync; 669 ca_sink->next_sync = next_sync; 670 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 671 672 if (old_buffer) 673 gst_buffer_unref (old_buffer); 674 if (old_sync) 675 gst_buffer_unref (old_sync); 676 677 gst_video_frame_unmap (&gl_frame); 678 679 return GST_FLOW_OK; 680 681upload_failed: 682 { 683 GST_ELEMENT_ERROR (ca_sink, RESOURCE, NOT_FOUND, 684 ("%s", "Failed to upload buffer"), (NULL)); 685 return GST_FLOW_ERROR; 686 } 687} 688 689static GstFlowReturn 690gst_ca_opengl_layer_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) 691{ 692 GstCAOpenGLLayerSink *ca_sink; 693 GstBuffer *stored_buffer, *old_sync; 694 695 GST_TRACE ("rendering buffer:%p", buf); 696 697 ca_sink = GST_CA_OPENGL_LAYER_SINK (vsink); 698 699 GST_TRACE ("redisplay texture:%u of size:%ux%u, window size:%ux%u", 700 ca_sink->next_tex, GST_VIDEO_INFO_WIDTH (&ca_sink->info), 701 GST_VIDEO_INFO_HEIGHT (&ca_sink->info), 702 GST_VIDEO_SINK_WIDTH (ca_sink), 703 GST_VIDEO_SINK_HEIGHT (ca_sink)); 704 705 /* Avoid to release the texture while drawing */ 706 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 707 ca_sink->redisplay_texture = ca_sink->next_tex; 708 709 stored_buffer = ca_sink->stored_buffer; 710 ca_sink->stored_buffer = gst_buffer_ref (ca_sink->next_buffer); 711 712 old_sync = ca_sink->stored_sync; 713 ca_sink->stored_sync = gst_buffer_ref (ca_sink->next_sync); 714 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 715 716 /* The layer will automatically call the draw callback to draw the new 717 * content */ 718 [CATransaction begin]; 719 [(__bridge GstGLCAOpenGLLayer *)(ca_sink->layer) setNeedsDisplay]; 720 [CATransaction commit]; 721 722 GST_TRACE ("post redisplay"); 723 724 if (stored_buffer) 725 gst_buffer_unref (stored_buffer); 726 if (old_sync) 727 gst_buffer_unref (old_sync); 728 729 return GST_FLOW_OK; 730} 731 732static gboolean 733gst_ca_opengl_layer_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) 734{ 735 GstCAOpenGLLayerSink *ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 736 GstBufferPool *pool = NULL; 737 GstStructure *config; 738 GstCaps *caps; 739 GstVideoInfo info; 740 guint size; 741 gboolean need_pool; 742 743 if (!_ensure_gl_setup (ca_sink)) 744 return FALSE; 745 746 gst_query_parse_allocation (query, &caps, &need_pool); 747 748 if (caps == NULL) 749 goto no_caps; 750 751 if (!gst_video_info_from_caps (&info, caps)) 752 goto invalid_caps; 753 754 /* the normal size of a frame */ 755 size = info.size; 756 757 if (need_pool) { 758 GST_DEBUG_OBJECT (ca_sink, "create new pool"); 759 760 pool = gst_gl_buffer_pool_new (ca_sink->context); 761 config = gst_buffer_pool_get_config (pool); 762 gst_buffer_pool_config_set_params (config, caps, size, 0, 0); 763 764 if (!gst_buffer_pool_set_config (pool, config)) 765 goto config_failed; 766 } 767 768 /* we need at least 2 buffer because we hold on to the last one */ 769 gst_query_add_allocation_pool (query, pool, size, 2, 0); 770 if (pool) 771 gst_object_unref (pool); 772 773 if (ca_sink->context->gl_vtable->FenceSync) 774 gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); 775 776 return TRUE; 777 778 /* ERRORS */ 779no_caps: 780 { 781 GST_DEBUG_OBJECT (bsink, "no caps specified"); 782 return FALSE; 783 } 784invalid_caps: 785 { 786 GST_DEBUG_OBJECT (bsink, "invalid caps specified"); 787 return FALSE; 788 } 789config_failed: 790 { 791 GST_DEBUG_OBJECT (bsink, "failed setting config"); 792 return FALSE; 793 } 794} 795 796/* *INDENT-OFF* */ 797static const GLfloat vertices[] = { 798 1.0f, 1.0f, 0.0f, 1.0f, 0.0f, 799 -1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 800 -1.0f, -1.0f, 0.0f, 0.0f, 1.0f, 801 1.0f, -1.0f, 0.0f, 1.0f, 1.0f 802}; 803 804static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 }; 805/* *INDENT-ON* */ 806 807static void 808_bind_buffer (GstCAOpenGLLayerSink * ca_sink) 809{ 810 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 811 812 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, ca_sink->vbo_indices); 813 gl->BindBuffer (GL_ARRAY_BUFFER, ca_sink->vertex_buffer); 814 815 /* Load the vertex position */ 816 gl->VertexAttribPointer (ca_sink->attr_position, 3, GL_FLOAT, GL_FALSE, 817 5 * sizeof (GLfloat), (void *) 0); 818 819 /* Load the texture coordinate */ 820 gl->VertexAttribPointer (ca_sink->attr_texture, 2, GL_FLOAT, GL_FALSE, 821 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat))); 822 823 gl->EnableVertexAttribArray (ca_sink->attr_position); 824 gl->EnableVertexAttribArray (ca_sink->attr_texture); 825} 826 827static void 828_unbind_buffer (GstCAOpenGLLayerSink * ca_sink) 829{ 830 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 831 832 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0); 833 gl->BindBuffer (GL_ARRAY_BUFFER, 0); 834 835 gl->DisableVertexAttribArray (ca_sink->attr_position); 836 gl->DisableVertexAttribArray (ca_sink->attr_texture); 837} 838 839/* Called in the gl thread */ 840static void 841gst_ca_opengl_layer_sink_thread_init_redisplay (GstCAOpenGLLayerSink * ca_sink) 842{ 843 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 844 GError *error = NULL; 845 846 if (!(ca_sink->redisplay_shader = gst_gl_shader_new_default (ca_sink->context, &error))) { 847 GST_ERROR_OBJECT (ca_sink, "Failed to link shader: %s", error->message); 848 gst_ca_opengl_layer_sink_cleanup_glthread (ca_sink); 849 return; 850 } 851 852 ca_sink->attr_position = 853 gst_gl_shader_get_attribute_location (ca_sink->redisplay_shader, 854 "a_position"); 855 ca_sink->attr_texture = 856 gst_gl_shader_get_attribute_location (ca_sink->redisplay_shader, 857 "a_texcoord"); 858 859 if (gl->GenVertexArrays) { 860 gl->GenVertexArrays (1, &ca_sink->vao); 861 gl->BindVertexArray (ca_sink->vao); 862 } 863 864 if (!ca_sink->vertex_buffer) { 865 gl->GenBuffers (1, &ca_sink->vertex_buffer); 866 gl->BindBuffer (GL_ARRAY_BUFFER, ca_sink->vertex_buffer); 867 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices, 868 GL_STATIC_DRAW); 869 } 870 871 if (!ca_sink->vbo_indices) { 872 gl->GenBuffers (1, &ca_sink->vbo_indices); 873 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, ca_sink->vbo_indices); 874 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices, 875 GL_STATIC_DRAW); 876 } 877 878 if (gl->GenVertexArrays) { 879 _bind_buffer (ca_sink); 880 gl->BindVertexArray (0); 881 } 882 883 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0); 884 gl->BindBuffer (GL_ARRAY_BUFFER, 0); 885} 886 887static void 888gst_ca_opengl_layer_sink_cleanup_glthread (GstCAOpenGLLayerSink * ca_sink) 889{ 890 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 891 892 if (ca_sink->redisplay_shader) { 893 gst_object_unref (ca_sink->redisplay_shader); 894 ca_sink->redisplay_shader = NULL; 895 } 896 897 if (ca_sink->vao) { 898 gl->DeleteVertexArrays (1, &ca_sink->vao); 899 ca_sink->vao = 0; 900 } 901 902 if (ca_sink->vbo_indices) { 903 gl->DeleteBuffers (1, &ca_sink->vbo_indices); 904 ca_sink->vbo_indices = 0; 905 } 906 907 if (ca_sink->vertex_buffer) { 908 gl->DeleteBuffers (1, &ca_sink->vertex_buffer); 909 ca_sink->vertex_buffer = 0; 910 } 911} 912 913static void 914gst_ca_opengl_layer_sink_on_resize (GstCAOpenGLLayerSink * ca_sink, gint width, gint height) 915{ 916 /* Here ca_sink members (ex:ca_sink->info) have a life time of set_caps. 917 * It means that they cannot not change between two set_caps 918 */ 919 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 920 921 GST_TRACE ("GL Window resized to %ux%u", width, height); 922 923 width = MAX (1, width); 924 height = MAX (1, height); 925 926 ca_sink->window_width = width; 927 ca_sink->window_height = height; 928 929 /* default reshape */ 930 if (ca_sink->keep_aspect_ratio) { 931 GstVideoRectangle src, dst, result; 932 933 src.x = 0; 934 src.y = 0; 935 src.w = GST_VIDEO_SINK_WIDTH (ca_sink); 936 src.h = GST_VIDEO_SINK_HEIGHT (ca_sink); 937 938 dst.x = 0; 939 dst.y = 0; 940 dst.w = width; 941 dst.h = height; 942 943 gst_video_sink_center_rect (src, dst, &result, TRUE); 944 gl->Viewport (result.x, result.y, result.w, result.h); 945 } else { 946 gl->Viewport (0, 0, width, height); 947 } 948} 949 950static void 951gst_ca_opengl_layer_sink_on_draw (GstCAOpenGLLayerSink * ca_sink) 952{ 953 /* Here ca_sink members (ex:ca_sink->info) have a life time of set_caps. 954 * It means that they cannot not change between two set_caps as well as 955 * for the redisplay_texture size. 956 * Whereas redisplay_texture id changes every sink_render 957 */ 958 959 const GstGLFuncs *gl = NULL; 960 GstGLSyncMeta *sync_meta; 961 962 g_return_if_fail (GST_IS_CA_OPENGL_LAYER_SINK (ca_sink)); 963 964 gl = ca_sink->context->gl_vtable; 965 966 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 967 968 if (G_UNLIKELY (!ca_sink->redisplay_shader)) { 969 gst_ca_opengl_layer_sink_thread_init_redisplay (ca_sink); 970 } 971 972 /* check if texture is ready for being drawn */ 973 if (!ca_sink->redisplay_texture) { 974 gl->ClearColor (0.0f, 0.0f, 0.0f, 1.0f); 975 gl->Clear (GL_COLOR_BUFFER_BIT); 976 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 977 return; 978 } 979 980 /* opengl scene */ 981 GST_TRACE ("redrawing texture:%u", ca_sink->redisplay_texture); 982 983 if (ca_sink->caps_change) { 984 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 985 gst_ca_opengl_layer_sink_on_resize (ca_sink, ca_sink->window_width, 986 ca_sink->window_height); 987 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 988 ca_sink->caps_change = FALSE; 989 } 990 991 sync_meta = gst_buffer_get_gl_sync_meta (ca_sink->stored_sync); 992 if (sync_meta) 993 gst_gl_sync_meta_wait (sync_meta, gst_gl_context_get_current ()); 994 995 gl->BindTexture (GL_TEXTURE_2D, 0); 996 997 gl->ClearColor (0.0, 0.0, 0.0, 0.0); 998 gl->Clear (GL_COLOR_BUFFER_BIT); 999 1000 gst_gl_shader_use (ca_sink->redisplay_shader); 1001 1002 if (gl->GenVertexArrays) 1003 gl->BindVertexArray (ca_sink->vao); 1004 _bind_buffer (ca_sink); 1005 1006 gl->ActiveTexture (GL_TEXTURE0); 1007 gl->BindTexture (GL_TEXTURE_2D, ca_sink->redisplay_texture); 1008 gst_gl_shader_set_uniform_1i (ca_sink->redisplay_shader, "tex", 0); 1009 1010 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0); 1011 1012 if (gl->GenVertexArrays) 1013 gl->BindVertexArray (0); 1014 else 1015 _unbind_buffer (ca_sink); 1016 1017 /* end default opengl scene */ 1018 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 1019} 1020