• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/*
2 * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20#ifdef HAVE_CONFIG_H
21# include "config.h"
22#endif
23
24#include <Metal/Metal.h>
25
26#if !HAVE_IOS
27#import <AppKit/AppKit.h>
28#endif
29#include "iosurfacevulkanmemory.h"
30
31/* silence macor redefinition warnings */
32#undef VK_USE_PLATFORM_MACOS_MVK
33#undef VK_USE_PLATFORM_IOS_MVK
34#include <MoltenVK/vk_mvk_moltenvk.h>
35/* MoltenVK uses some enums/typedefs that are only available in newer macOS/iOS
36 * versions. At time of writing:
37 *  - MTLTextureSwizzle
38 *  - MTLTextureSwizzleChannels
39 *  - MTLMultisampleDepthResolveFilter
40 */
41#pragma clang diagnostic push
42#pragma clang diagnostic warning "-Wunguarded-availability-new"
43#include <MoltenVK/mvk_datatypes.h>
44#pragma clang diagnostic pop
45/* silence macro redefinition warnings */
46#undef VK_USE_PLATFORM_MACOS_MVK
47#undef VK_USE_PLATFORM_IOS_MVK
48
49#include "coremediabuffer.h"
50#include "corevideobuffer.h"
51#include "vtutil.h"
52
53#include "videotexturecache-vulkan.h"
54#include "metal-helpers.h"
55
56typedef struct _GstVideoTextureCacheVulkanPrivate
57{
58  GstBufferPool *pool;
59} GstVideoTextureCacheVulkanPrivate;
60
61G_DEFINE_TYPE_WITH_PRIVATE (GstVideoTextureCacheVulkan, gst_video_texture_cache_vulkan, GST_TYPE_VIDEO_TEXTURE_CACHE);
62
63#define GET_PRIV(instance) \
64    G_TYPE_INSTANCE_GET_PRIVATE (instance, GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanPrivate)
65
66typedef struct _IOSTextureWrapper
67{
68  CVMetalTextureCacheRef cache;
69  CVMetalTextureRef texture;
70} IOSTextureWrapper;
71
72enum
73{
74  PROP_0,
75  PROP_DEVICE,
76};
77
78static GstMemory * gst_video_texture_cache_vulkan_create_memory (GstVideoTextureCache * cache,
79      GstAppleCoreVideoPixelBuffer *gpixbuf, guint plane, gsize size);
80
81GstVideoTextureCache *
82gst_video_texture_cache_vulkan_new (GstVulkanDevice * device)
83{
84  g_return_val_if_fail (GST_IS_VULKAN_DEVICE (device), NULL);
85
86  return (GstVideoTextureCache *) g_object_new (GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN,
87      "device", device, NULL);
88}
89
90static void
91gst_video_texture_cache_vulkan_finalize (GObject * object)
92{
93  GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
94
95#if 0
96  gst_buffer_pool_set_active (cache->pool, FALSE);
97  gst_object_unref (cache->pool);
98#endif
99  gst_object_unref (cache_vulkan->device);
100
101  G_OBJECT_CLASS (gst_video_texture_cache_vulkan_parent_class)->finalize (object);
102}
103
104static void
105gst_video_texture_cache_vulkan_set_property (GObject * object, guint prop_id,
106    const GValue * value, GParamSpec * pspec)
107{
108  GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
109
110  switch (prop_id) {
111    case PROP_DEVICE:
112      cache_vulkan->device = (GstVulkanDevice *) g_value_dup_object (value);
113      break;
114    default:
115      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
116      break;
117  }
118}
119
120static void
121gst_video_texture_cache_vulkan_get_property (GObject * object, guint prop_id,
122    GValue * value, GParamSpec * pspec)
123{
124  GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
125
126  switch (prop_id) {
127    case PROP_DEVICE:
128      g_value_set_object (value, cache_vulkan->device);
129      break;
130    default:
131      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
132      break;
133  }
134}
135
136static void
137gst_video_texture_cache_vulkan_constructed (GObject * object)
138{
139  GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
140
141  g_return_if_fail (GST_IS_VULKAN_DEVICE (cache_vulkan->device));
142
143  gst_io_surface_vulkan_memory_init ();
144#if 0
145  cache->pool = GST_BUFFER_POOL (gst_vulkan_buffer_pool_new (ctx));
146#endif
147}
148
149static void
150gst_video_texture_cache_vulkan_init (GstVideoTextureCacheVulkan * cache_vulkan)
151{
152}
153
154static void
155gst_video_texture_cache_vulkan_class_init (GstVideoTextureCacheVulkanClass *klass)
156{
157  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
158  GstVideoTextureCacheClass *cache_class = (GstVideoTextureCacheClass *) klass;
159
160  gobject_class->set_property = gst_video_texture_cache_vulkan_set_property;
161  gobject_class->get_property = gst_video_texture_cache_vulkan_get_property;
162  gobject_class->constructed = gst_video_texture_cache_vulkan_constructed;
163  gobject_class->finalize = gst_video_texture_cache_vulkan_finalize;
164
165  g_object_class_install_property (gobject_class, PROP_DEVICE,
166      g_param_spec_object ("device", "device",
167          "Associated Vulkan device", GST_TYPE_VULKAN_DEVICE,
168          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS)));
169
170  cache_class->create_memory = gst_video_texture_cache_vulkan_create_memory;
171}
172
173static GstMemory *
174gst_video_texture_cache_vulkan_create_memory (GstVideoTextureCache * cache,
175      GstAppleCoreVideoPixelBuffer *gpixbuf, guint plane, gsize size)
176{
177  return (GstMemory *) gpixbuf;
178}
179
180VkFormat
181metal_format_to_vulkan (unsigned int fmt)
182{
183  MTLPixelFormat mtl_fmt = (MTLPixelFormat) fmt;
184
185  switch (mtl_fmt) {
186    case MTLPixelFormatRGBA8Unorm:
187      return VK_FORMAT_R8G8B8A8_UNORM;
188    case MTLPixelFormatRG8Unorm:
189      return VK_FORMAT_R8G8_UNORM;
190    case MTLPixelFormatR8Unorm:
191      return VK_FORMAT_R8_UNORM;
192    default:
193      g_assert_not_reached ();
194  }
195}
196
197unsigned int
198video_info_to_metal_format (GstVideoInfo * info, guint plane)
199{
200  switch (GST_VIDEO_INFO_FORMAT (info)) {
201    case GST_VIDEO_FORMAT_BGRA:
202      return (unsigned int) MTLPixelFormatRGBA8Unorm;
203    case GST_VIDEO_FORMAT_NV12:
204      if (plane == 0)
205        return (unsigned int) MTLPixelFormatR8Unorm;
206      else
207        return (unsigned int) MTLPixelFormatRG8Unorm;
208    default:
209      g_assert_not_reached ();
210  }
211}
212
213GstMemory *
214_create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
215    GstVideoInfo * info, guint plane, gsize size, GstVideoTextureCache * cache)
216{
217  GstIOSurfaceVulkanMemory *mem;
218  CVPixelBufferRef pixel_buf = gpixbuf->buf;
219  IOSurfaceRef surface = CVPixelBufferGetIOSurface (pixel_buf);
220  GstVideoTextureCacheVulkan *cache_vulkan =
221      GST_VIDEO_TEXTURE_CACHE_VULKAN (cache);
222  MTLPixelFormat fmt = (MTLPixelFormat) video_info_to_metal_format (info, plane);
223
224  CFRetain (pixel_buf);
225  mem = gst_io_surface_vulkan_memory_wrapped (cache_vulkan->device,
226      surface, fmt, info, plane, pixel_buf, (GDestroyNotify) CFRelease);
227
228  if (!mem)
229    return NULL;
230
231  return GST_MEMORY_CAST (mem);
232}
233
234typedef struct _IOSurfaceTextureWrapper
235{
236  CVPixelBufferRef pixbuf;
237  gpointer texture; /* id<MTLTexture> */
238} IOSurfaceTextureWrapper;
239
240static void
241free_texture_wrapper (IOSurfaceTextureWrapper * wrapper)
242{
243  CFRelease (wrapper->pixbuf);
244  id<MTLTexture> tex = (__bridge_transfer id<MTLTexture>) wrapper->texture;
245  (void) tex;
246  g_free (wrapper);
247}
248
249static MTLTextureDescriptor *
250gst_new_mtl_tex_descripter_from_memory (GstIOSurfaceVulkanMemory * memory)
251{
252  GstVulkanImageMemory *vk_mem = (GstVulkanImageMemory *) memory;
253  MTLTextureDescriptor* tex_desc = [MTLTextureDescriptor new];
254  tex_desc.pixelFormat = mvkMTLPixelFormatFromVkFormat (vk_mem->create_info.format);
255  tex_desc.textureType = mvkMTLTextureTypeFromVkImageType (vk_mem->create_info.imageType, 0, false);
256  tex_desc.width = vk_mem->create_info.extent.width;
257  tex_desc.height = vk_mem->create_info.extent.height;
258  tex_desc.depth = vk_mem->create_info.extent.depth;
259  tex_desc.mipmapLevelCount = vk_mem->create_info.mipLevels;
260  tex_desc.sampleCount = mvkSampleCountFromVkSampleCountFlagBits(vk_mem->create_info.samples);
261  tex_desc.arrayLength = vk_mem->create_info.arrayLayers;
262  tex_desc.usage = MTLTextureUsageShaderRead | MTLTextureUsagePixelFormatView;//mvkMTLTextureUsageFromVkImageUsageFlags(vk_mem->create_info.usage);
263  tex_desc.storageMode = MTLStorageModeShared;
264  tex_desc.cpuCacheMode = MTLCPUCacheModeDefaultCache;//mvkMTLCPUCacheModeFromVkMemoryPropertyFlags(vk_mem->vk_mem->properties);
265
266  return tex_desc;
267}
268
269void
270gst_io_surface_vulkan_memory_set_surface (GstIOSurfaceVulkanMemory * memory,
271    IOSurfaceRef surface)
272{
273  GstVulkanImageMemory *vk_mem = (GstVulkanImageMemory *) memory;
274
275  if (memory->surface) {
276     IOSurfaceDecrementUseCount (memory->surface);
277  } else {
278    g_assert (vk_mem->notify == (GDestroyNotify) CFRelease);
279    g_assert (vk_mem->user_data != NULL);
280  }
281
282  memory->surface = surface;
283  if (surface) {
284    id<MTLDevice> mtl_dev = nil;
285    id<MTLTexture> texture = nil;
286    VkPhysicalDevice gpu;
287
288    IOSurfaceIncrementUseCount (surface);
289
290    gpu = gst_vulkan_device_get_physical_device (vk_mem->device);
291    vkGetMTLDeviceMVK (gpu, &mtl_dev);
292
293    /* We cannot use vkUseIOSurfaceMVK() for multi-planer as MoltenVK does not
294     * support them. */
295
296    MTLTextureDescriptor *tex_desc = gst_new_mtl_tex_descripter_from_memory (memory);
297    texture = [mtl_dev newTextureWithDescriptor:tex_desc iosurface:surface plane:memory->plane];
298
299    IOSurfaceTextureWrapper *texture_data = g_new0 (IOSurfaceTextureWrapper, 1);
300    texture_data->pixbuf = (CVPixelBufferRef) vk_mem->user_data;
301    texture_data->texture = (__bridge_retained gpointer) texture;
302
303    VkResult err = vkSetMTLTextureMVK (memory->vulkan_mem.image, texture);
304    GST_DEBUG ("bound texture %p to image %" GST_VULKAN_NON_DISPATCHABLE_HANDLE_FORMAT ": 0x%x",
305               texture, memory->vulkan_mem.image, err);
306
307    vk_mem->user_data = texture_data;
308    vk_mem->notify = (GDestroyNotify) free_texture_wrapper;
309  }
310}
311