1 /*
2 * Copyright (C) 2010 Ole André Vadla Ravnås <oleavr@soundrop.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 #ifdef HAVE_CONFIG_H
21 # include "config.h"
22 #endif
23 #include "corevideobuffer.h"
24 #include "corevideomemory.h"
25 #if !HAVE_IOS
26 #include "iosurfaceglmemory.h"
27 #endif
28 #include "videotexturecache-gl.h"
29 #if defined(APPLEMEDIA_MOLTENVK)
30 #include "videotexturecache-vulkan.h"
31 #if !HAVE_IOS
32 #include "iosurfacevulkanmemory.h"
33 #endif
34 #endif
35
36 static const GstMetaInfo *gst_core_video_meta_get_info (void);
37
38 static void
gst_core_video_meta_add(GstBuffer * buffer,CVBufferRef cvbuf)39 gst_core_video_meta_add (GstBuffer * buffer, CVBufferRef cvbuf)
40 {
41 GstCoreVideoMeta *meta;
42
43 meta = (GstCoreVideoMeta *) gst_buffer_add_meta (buffer,
44 gst_core_video_meta_get_info (), NULL);
45 meta->cvbuf = CVBufferRetain (cvbuf);
46 meta->pixbuf = (CVPixelBufferRef) cvbuf;
47 }
48
49 static gboolean
gst_core_video_meta_init(GstCoreVideoMeta * meta,gpointer params,GstBuffer * buf)50 gst_core_video_meta_init (GstCoreVideoMeta * meta, gpointer params,
51 GstBuffer * buf)
52 {
53 meta->cvbuf = NULL;
54 meta->pixbuf = NULL;
55
56 return TRUE;
57 }
58
59 static void
gst_core_video_meta_free(GstCoreVideoMeta * meta,GstBuffer * buf)60 gst_core_video_meta_free (GstCoreVideoMeta * meta, GstBuffer * buf)
61 {
62 CVBufferRelease (meta->cvbuf);
63 }
64
65 static gboolean
gst_core_video_meta_transform(GstBuffer * transbuf,GstCoreVideoMeta * meta,GstBuffer * buffer,GQuark type,GstMetaTransformCopy * data)66 gst_core_video_meta_transform (GstBuffer * transbuf, GstCoreVideoMeta * meta,
67 GstBuffer * buffer, GQuark type, GstMetaTransformCopy * data)
68 {
69 if (!data->region) {
70 /* only copy if the complete data is copied as well */
71 gst_core_video_meta_add (transbuf, meta->cvbuf);
72 } else {
73 GST_WARNING_OBJECT (transbuf,
74 "dropping Core Video metadata due to partial buffer");
75 }
76
77 return TRUE; /* retval unused */
78 }
79
80 GType
gst_core_video_meta_api_get_type(void)81 gst_core_video_meta_api_get_type (void)
82 {
83 static GType type;
84 static const gchar *tags[] = { "memory", NULL };
85
86 if (g_once_init_enter (&type)) {
87 GType _type = gst_meta_api_type_register ("GstCoreVideoMetaAPI", tags);
88 g_once_init_leave (&type, _type);
89 }
90 return type;
91 }
92
93 static const GstMetaInfo *
gst_core_video_meta_get_info(void)94 gst_core_video_meta_get_info (void)
95 {
96 static const GstMetaInfo *core_video_meta_info = NULL;
97
98 if (g_once_init_enter (&core_video_meta_info)) {
99 const GstMetaInfo *meta = gst_meta_register (GST_CORE_VIDEO_META_API_TYPE,
100 "GstCoreVideoMeta", sizeof (GstCoreVideoMeta),
101 (GstMetaInitFunction) gst_core_video_meta_init,
102 (GstMetaFreeFunction) gst_core_video_meta_free,
103 (GstMetaTransformFunction) gst_core_video_meta_transform);
104 g_once_init_leave (&core_video_meta_info, meta);
105 }
106 return core_video_meta_info;
107 }
108
109 static GstMemory *
_create_glmem(GstAppleCoreVideoPixelBuffer * gpixbuf,GstVideoInfo * info,guint plane,gsize size,GstVideoTextureCache * cache)110 _create_glmem (GstAppleCoreVideoPixelBuffer * gpixbuf,
111 GstVideoInfo * info, guint plane, gsize size, GstVideoTextureCache * cache)
112 {
113 #if HAVE_IOS
114 return gst_video_texture_cache_create_memory (cache, gpixbuf, plane, size);
115 #else
116 GstIOSurfaceGLMemory *mem;
117 CVPixelBufferRef pixel_buf = gpixbuf->buf;
118 IOSurfaceRef surface = CVPixelBufferGetIOSurface (pixel_buf);
119 GstGLFormat tex_format;
120 GstVideoTextureCacheGL *cache_gl = GST_VIDEO_TEXTURE_CACHE_GL (cache);
121
122 tex_format = gst_gl_format_from_video_info (cache_gl->ctx, info, plane);
123
124 CFRetain (pixel_buf);
125 mem = gst_io_surface_gl_memory_wrapped (cache_gl->ctx,
126 surface, GST_GL_TEXTURE_TARGET_RECTANGLE, tex_format,
127 info, plane, NULL, pixel_buf, (GDestroyNotify) CFRelease);
128 return GST_MEMORY_CAST (mem);
129 #endif
130 }
131
132 #if defined(APPLEMEDIA_MOLTENVK)
133 /* in videotexturecache-vulkan.m to avoid objc-ism from Metal being included
134 * in a non-objc file */
135 extern GstMemory *_create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
136 GstVideoInfo * info, guint plane, gsize size, GstVideoTextureCache * cache);
137 #endif
138
139 void
gst_core_video_wrap_pixel_buffer(GstBuffer * buf,GstVideoInfo * info,CVPixelBufferRef pixel_buf,GstVideoTextureCache * cache,gboolean * has_padding)140 gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
141 GstVideoInfo * info,
142 CVPixelBufferRef pixel_buf,
143 GstVideoTextureCache * cache, gboolean * has_padding)
144 {
145 guint n_planes;
146 gsize offset[GST_VIDEO_MAX_PLANES] = { 0 };
147 gint stride[GST_VIDEO_MAX_PLANES] = { 0 };
148 UInt32 size;
149 GstAppleCoreVideoPixelBuffer *gpixbuf;
150 GstMemory *mem = NULL;
151 gboolean do_gl = GST_IS_VIDEO_TEXTURE_CACHE_GL (cache);
152 #if defined(APPLEMEDIA_MOLTENVK)
153 gboolean do_vulkan = GST_IS_VIDEO_TEXTURE_CACHE_VULKAN (cache);
154 #endif
155
156 gpixbuf = gst_apple_core_video_pixel_buffer_new (pixel_buf);
157
158 if (has_padding)
159 *has_padding = FALSE;
160
161 if (CVPixelBufferIsPlanar (pixel_buf)) {
162 gint i, size = 0, plane_offset = 0;
163
164 n_planes = CVPixelBufferGetPlaneCount (pixel_buf);
165 for (i = 0; i < n_planes; i++) {
166 stride[i] = CVPixelBufferGetBytesPerRowOfPlane (pixel_buf, i);
167
168 if (stride[i] != GST_VIDEO_INFO_PLANE_STRIDE (info, i) && has_padding)
169 *has_padding = TRUE;
170
171 size = stride[i] * CVPixelBufferGetHeightOfPlane (pixel_buf, i);
172 offset[i] = plane_offset;
173 plane_offset += size;
174
175 if (do_gl)
176 mem = _create_glmem (gpixbuf, info, i, size, cache);
177 #if defined(APPLEMEDIA_MOLTENVK)
178 else if (do_vulkan)
179 mem = _create_vulkan_memory (gpixbuf, info, i, size, cache);
180 #endif
181 else
182 mem =
183 GST_MEMORY_CAST (gst_apple_core_video_memory_new_wrapped (gpixbuf,
184 i, size));
185 gst_buffer_append_memory (buf, mem);
186 }
187 } else {
188 n_planes = 1;
189 stride[0] = CVPixelBufferGetBytesPerRow (pixel_buf);
190 offset[0] = 0;
191 size = stride[0] * CVPixelBufferGetHeight (pixel_buf);
192
193 if (do_gl)
194 mem = _create_glmem (gpixbuf, info, 0, size, cache);
195 #if defined(APPLEMEDIA_MOLTENVK)
196 else if (do_vulkan)
197 mem = _create_vulkan_memory (gpixbuf, info, 0, size, cache);
198 #endif
199 else
200 mem =
201 GST_MEMORY_CAST (gst_apple_core_video_memory_new_wrapped (gpixbuf, 0,
202 size));
203 gst_buffer_append_memory (buf, mem);
204 }
205
206 gst_apple_core_video_pixel_buffer_unref (gpixbuf);
207
208 if (info) {
209 gst_buffer_add_video_meta_full (buf, GST_VIDEO_FRAME_FLAG_NONE,
210 GST_VIDEO_INFO_FORMAT (info), info->width, info->height, n_planes,
211 offset, stride);
212 }
213 }
214
215 static GstVideoFormat
gst_core_video_get_video_format(OSType format)216 gst_core_video_get_video_format (OSType format)
217 {
218 switch (format) {
219 case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
220 return GST_VIDEO_FORMAT_NV12;
221 case kCVPixelFormatType_422YpCbCr8_yuvs:
222 return GST_VIDEO_FORMAT_YUY2;
223 case kCVPixelFormatType_422YpCbCr8:
224 return GST_VIDEO_FORMAT_UYVY;
225 case kCVPixelFormatType_32BGRA:
226 return GST_VIDEO_FORMAT_BGRA;
227 case kCVPixelFormatType_32RGBA:
228 return GST_VIDEO_FORMAT_RGBA;
229 default:
230 GST_WARNING ("Unknown OSType format: %d", (gint) format);
231 return GST_VIDEO_FORMAT_UNKNOWN;
232 }
233 }
234
235
236 gboolean
gst_core_video_info_init_from_pixel_buffer(GstVideoInfo * info,CVPixelBufferRef pixel_buf)237 gst_core_video_info_init_from_pixel_buffer (GstVideoInfo * info,
238 CVPixelBufferRef pixel_buf)
239 {
240 size_t width, height;
241 OSType format_type;
242 GstVideoFormat video_format;
243
244 width = CVPixelBufferGetWidth (pixel_buf);
245 height = CVPixelBufferGetHeight (pixel_buf);
246 format_type = CVPixelBufferGetPixelFormatType (pixel_buf);
247 video_format = gst_core_video_get_video_format (format_type);
248
249 if (video_format == GST_VIDEO_FORMAT_UNKNOWN) {
250 return FALSE;
251 }
252
253 gst_video_info_init (info);
254 gst_video_info_set_format (info, video_format, width, height);
255
256 return TRUE;
257 }
258
259
260 GstBuffer *
gst_core_video_buffer_new(CVBufferRef cvbuf,GstVideoInfo * vinfo,GstVideoTextureCache * cache)261 gst_core_video_buffer_new (CVBufferRef cvbuf, GstVideoInfo * vinfo,
262 GstVideoTextureCache * cache)
263 {
264 CVPixelBufferRef pixbuf = NULL;
265 GstBuffer *buf;
266 GstCoreVideoMeta *meta;
267
268 if (CFGetTypeID (cvbuf) != CVPixelBufferGetTypeID ())
269 /* TODO: Do we need to handle other buffer types? */
270 return NULL;
271
272 pixbuf = (CVPixelBufferRef) cvbuf;
273
274 buf = gst_buffer_new ();
275
276 /* add the corevideo meta to pass the underlying corevideo buffer */
277 meta = (GstCoreVideoMeta *) gst_buffer_add_meta (buf,
278 gst_core_video_meta_get_info (), NULL);
279 meta->cvbuf = CVBufferRetain (cvbuf);
280 meta->pixbuf = pixbuf;
281
282 gst_core_video_wrap_pixel_buffer (buf, vinfo, pixbuf, cache, NULL);
283
284 return buf;
285 }
286