# cdata.set('HAVE_VIDEOTOOLBOX_10_9_6', 1)
# endif
endif
+have_objcpp = add_languages('objcpp', required : false)
have_orcc = false
orcc_args = []
#include "iosurfaceglmemory.h"
#endif
#include "videotexturecache-gl.h"
+#if defined(APPLEMEDIA_MOLTENVK)
+#include "videotexturecache-vulkan.h"
+#if !HAVE_IOS
+#include "iosurfacevulkanmemory.h"
+#endif
+#endif
static const GstMetaInfo *gst_core_video_meta_get_info (void);
#endif
}
+#if defined(APPLEMEDIA_MOLTENVK)
+/* in videotexturecache-vulkan.m to avoid objc-ism from Metal being included
+ * in a non-objc file */
+extern GstMemory *_create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
+ GstVideoInfo * info, guint plane, gsize size, GstVideoTextureCache * cache);
+#endif
+
void
gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
GstVideoInfo * info,
GstAppleCoreVideoPixelBuffer *gpixbuf;
GstMemory *mem = NULL;
gboolean do_gl = GST_IS_VIDEO_TEXTURE_CACHE_GL (cache);
+#if defined(APPLEMEDIA_MOLTENVK)
+ gboolean do_vulkan = GST_IS_VIDEO_TEXTURE_CACHE_VULKAN (cache);
+#endif
gpixbuf = gst_apple_core_video_pixel_buffer_new (pixel_buf);
if (do_gl)
mem = _create_glmem (gpixbuf, info, i, size, cache);
+#if defined(APPLEMEDIA_MOLTENVK)
+ else if (do_vulkan)
+ mem = _create_vulkan_memory (gpixbuf, info, i, size, cache);
+#endif
else
mem =
GST_MEMORY_CAST (gst_apple_core_video_memory_new_wrapped (gpixbuf,
if (do_gl)
mem = _create_glmem (gpixbuf, info, 0, size, cache);
+#if defined(APPLEMEDIA_MOLTENVK)
+ else if (do_vulkan)
+ mem = _create_vulkan_memory (gpixbuf, info, 0, size, cache);
+#endif
else
mem =
GST_MEMORY_CAST (gst_apple_core_video_memory_new_wrapped (gpixbuf, 0,
--- /dev/null
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "iosurfacevulkanmemory.h"
+#include "metal-helpers.h"
+
+GST_DEBUG_CATEGORY_STATIC (GST_CAT_IO_SURFACE_VULKAN_MEMORY);
+#define GST_CAT_DEFAULT GST_CAT_IO_SURFACE_VULKAN_MEMORY
+
+G_DEFINE_TYPE (GstIOSurfaceVulkanMemoryAllocator,
+ gst_io_surface_vulkan_memory_allocator,
+ GST_TYPE_VULKAN_IMAGE_MEMORY_ALLOCATOR);
+
+typedef struct
+{
+ GstIOSurfaceVulkanMemory *memory;
+ IOSurfaceRef surface;
+} ContextThreadData;
+
+static GstAllocator *_io_surface_vulkan_memory_allocator;
+
+static void
+_mem_free (GstAllocator * allocator, GstMemory * mem)
+{
+ gst_io_surface_vulkan_memory_set_surface ((GstIOSurfaceVulkanMemory *) mem,
+ NULL);
+
+ GST_ALLOCATOR_CLASS
+ (gst_io_surface_vulkan_memory_allocator_parent_class)->free (allocator,
+ mem);
+}
+
+static gpointer
+_io_surface_vulkan_memory_allocator_map (GstMemory * bmem,
+ GstMapInfo * info, gsize size)
+{
+ GstIOSurfaceVulkanMemory *mem = (GstIOSurfaceVulkanMemory *) bmem;
+
+ GST_LOG ("mapping surface %p flags %d", mem->surface, info->flags);
+
+ if (!(info->flags & GST_MAP_WRITE)) {
+ IOSurfaceLock (mem->surface, kIOSurfaceLockReadOnly, NULL);
+ return IOSurfaceGetBaseAddressOfPlane (mem->surface, mem->plane);
+ } else {
+ GST_ERROR ("couldn't map IOSurface %p flags %d", mem->surface, info->flags);
+ return NULL;
+ }
+}
+
+static void
+_io_surface_vulkan_memory_allocator_unmap (GstMemory * bmem, GstMapInfo * info)
+{
+ GstIOSurfaceVulkanMemory *mem = (GstIOSurfaceVulkanMemory *) bmem;
+
+ GST_LOG ("unmapping surface %p flags %d", mem->surface, info->flags);
+
+ IOSurfaceUnlock (mem->surface, kIOSurfaceLockReadOnly, NULL);
+}
+
+static GstMemory *
+_mem_alloc (GstAllocator * allocator, gsize size, GstAllocationParams * params)
+{
+ g_warning
+ ("use gst_io_surface_vulkan_memory_wrapped () to allocate from this "
+ "IOSurface allocator");
+
+ return NULL;
+}
+
+static void
+ gst_io_surface_vulkan_memory_allocator_class_init
+ (GstIOSurfaceVulkanMemoryAllocatorClass * klass)
+{
+ GstAllocatorClass *allocator_class = (GstAllocatorClass *) klass;
+
+ allocator_class->alloc = _mem_alloc;
+ allocator_class->free = _mem_free;
+}
+
+static void
+gst_io_surface_vulkan_memory_allocator_init (GstIOSurfaceVulkanMemoryAllocator *
+ allocator)
+{
+ GstAllocator *alloc = GST_ALLOCATOR_CAST (allocator);
+
+ alloc->mem_type = GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_NAME;
+ GST_OBJECT_FLAG_SET (allocator, GST_ALLOCATOR_FLAG_CUSTOM_ALLOC);
+
+ alloc->mem_map_full = _io_surface_vulkan_memory_allocator_map;
+ alloc->mem_unmap_full = _io_surface_vulkan_memory_allocator_unmap;
+}
+
+void
+gst_io_surface_vulkan_memory_init (void)
+{
+ static volatile gsize _init = 0;
+
+ if (g_once_init_enter (&_init)) {
+ GST_DEBUG_CATEGORY_INIT (GST_CAT_IO_SURFACE_VULKAN_MEMORY,
+ "iosurfacevulkan", 0, "IOSurface Vulkan Buffer");
+
+ _io_surface_vulkan_memory_allocator =
+ g_object_new (GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, NULL);
+ gst_object_ref_sink (_io_surface_vulkan_memory_allocator);
+
+ gst_allocator_register (GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_NAME,
+ gst_object_ref (_io_surface_vulkan_memory_allocator));
+ g_once_init_leave (&_init, 1);
+ }
+}
+
+gboolean
+gst_is_io_surface_vulkan_memory (GstMemory * mem)
+{
+ return mem != NULL && mem->allocator != NULL &&
+ g_type_is_a (G_OBJECT_TYPE (mem->allocator),
+ GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR);
+}
+
+static GstIOSurfaceVulkanMemory *
+_io_surface_vulkan_memory_new (GstVulkanDevice * device, IOSurfaceRef surface,
+ unsigned int /* MTLPixelFormat */ fmt, GstVideoInfo * info, guint plane,
+ gpointer user_data, GDestroyNotify notify)
+{
+ GstIOSurfaceVulkanMemory *mem;
+ GstAllocationParams params = { 0, };
+ VkImageCreateInfo image_info;
+ VkPhysicalDevice gpu;
+ VkImageUsageFlags usage;
+ VkImageTiling tiling = VK_IMAGE_TILING_OPTIMAL;
+ VkFormat vk_format;
+ VkImage image;
+ GError *error = NULL;
+ VkResult err;
+
+ mem = g_new0 (GstIOSurfaceVulkanMemory, 1);
+
+ vk_format = metal_format_to_vulkan (fmt);
+ /* FIXME: choose from outside */
+ usage =
+ VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
+ VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT;
+
+ /* *INDENT-OFF* */
+ image_info = (VkImageCreateInfo) {
+ .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
+ .pNext = NULL,
+ .flags = 0,
+ .imageType = VK_IMAGE_TYPE_2D,
+ .format = vk_format,
+ /* MoltenVK double checks these against the IOSurface in vkUseIOSurface()
+ * and will fail if they do not match */
+ .extent = (VkExtent3D) { GST_VIDEO_INFO_COMP_WIDTH (info, plane), GST_VIDEO_INFO_COMP_HEIGHT (info, plane), 1 },
+ .mipLevels = 1,
+ .arrayLayers = 1,
+ .samples = VK_SAMPLE_COUNT_1_BIT,
+ .tiling = tiling,
+ .usage = usage,
+ .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+ .queueFamilyIndexCount = 0,
+ .pQueueFamilyIndices = NULL,
+ .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+ };
+ /* *INDENT-ON* */
+
+ gpu = gst_vulkan_device_get_physical_device (device);
+
+ err = vkCreateImage (device->device, &image_info, NULL, &image);
+ if (gst_vulkan_error_to_g_error (err, &error, "vkCreateImage") < 0)
+ goto vk_error;
+
+ vkGetImageMemoryRequirements (device->device, image,
+ &mem->vulkan_mem.requirements);
+
+ gst_vulkan_image_memory_init (&mem->vulkan_mem,
+ _io_surface_vulkan_memory_allocator, NULL, device, usage, ¶ms,
+ mem->vulkan_mem.requirements.size, user_data, notify);
+ mem->vulkan_mem.create_info = image_info;
+ mem->vulkan_mem.image = image;
+ mem->vulkan_mem.barrier.image_layout = VK_IMAGE_LAYOUT_GENERAL;
+
+ err =
+ vkGetPhysicalDeviceImageFormatProperties (gpu, vk_format,
+ VK_IMAGE_TYPE_2D, tiling, usage, 0, &mem->vulkan_mem.format_properties);
+ if (gst_vulkan_error_to_g_error (err, &error,
+ "vkGetPhysicalDeviceImageFormatProperties") < 0)
+ goto vk_error;
+
+ GST_MINI_OBJECT_FLAG_SET (mem, GST_MEMORY_FLAG_READONLY);
+
+ mem->surface = NULL;
+ mem->plane = plane;
+ gst_io_surface_vulkan_memory_set_surface (mem, surface);
+
+ return mem;
+
+vk_error:
+ {
+ GST_CAT_ERROR (GST_CAT_IO_SURFACE_VULKAN_MEMORY,
+ "Failed to allocate image memory %s", error->message);
+ g_clear_error (&error);
+ goto error;
+ }
+
+error:
+ {
+ if (mem)
+ gst_memory_unref ((GstMemory *) mem);
+ return NULL;
+ }
+}
+
+GstIOSurfaceVulkanMemory *
+gst_io_surface_vulkan_memory_wrapped (GstVulkanDevice * device,
+ IOSurfaceRef surface, unsigned int /* MTLPixelFormat */ fmt,
+ GstVideoInfo * info, guint plane, gpointer user_data, GDestroyNotify notify)
+{
+ return _io_surface_vulkan_memory_new (device, surface, fmt, info, plane,
+ user_data, notify);
+}
--- /dev/null
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _GST_IO_SURFACE_VULKAN_MEMORY_H_
+#define _GST_IO_SURFACE_VULKAN_MEMORY_H_
+
+#include <IOSurface/IOSurfaceRef.h>
+#include <IOSurface/IOSurfaceObjc.h>
+#include <gst/gst.h>
+#include <gst/gstallocator.h>
+#include <gst/video/video.h>
+#include <gst/vulkan/vulkan.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR (gst_io_surface_vulkan_memory_allocator_get_type())
+GType gst_io_surface_vulkan_memory_allocator_get_type(void);
+
+#define GST_IS_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR))
+#define GST_IS_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR))
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, GstIOSurfaceVulkanMemoryAllocatorClass))
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, GstIOSurfaceVulkanMemoryAllocator))
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, GstIOSurfaceVulkanMemoryAllocatorClass))
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_CAST(obj) ((GstIOSurfaceVulkanMemoryAllocator *)(obj))
+
+typedef struct _GstIOSurfaceVulkanMemory
+{
+ GstVulkanImageMemory vulkan_mem;
+ IOSurfaceRef surface;
+ guint plane;
+} GstIOSurfaceVulkanMemory;
+
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_NAME "IOSurfaceVulkanMemory"
+
+void gst_io_surface_vulkan_memory_init (void);
+
+GstIOSurfaceVulkanMemory *
+gst_io_surface_vulkan_memory_wrapped (GstVulkanDevice * device,
+ IOSurfaceRef surface,
+ unsigned int fmt, /* MTLPixelFormat */
+ GstVideoInfo * info,
+ guint plane,
+ gpointer user_data,
+ GDestroyNotify notify);
+
+gboolean gst_is_io_surface_vulkan_memory (GstMemory * mem);
+
+typedef struct _GstIOSurfaceVulkanMemoryAllocator
+{
+ GstVulkanMemoryAllocator allocator;
+} GstIOSurfaceVulkanMemoryAllocator;
+
+typedef struct _GstIOSurfaceVulkanMemoryAllocatorClass
+{
+ GstVulkanMemoryAllocatorClass parent_class;
+} GstIOSurfaceVulkanMemoryAllocatorClass;
+
+G_END_DECLS
+
+#endif /* _GST_IO_SURFACE_MEMORY_H_ */
applemedia_objc_args += ['-fobjc-abi-version=2', '-fobjc-legacy-dispatch']
ios_media_dep = dependency('appleframeworks', modules : ['Foundation', 'AssetsLibrary'], required : applemedia_option)
- applemedia_frameworks += [ios_media_dep]
+ iosurface_dep = dependency('IOSurface', required : applemedia_option)
+ applemedia_frameworks += [ios_media_dep, iosurface_dep]
else
applemedia_sources += [
'iosurfaceglmemory.c'
]
- applemedia_objc_args += ['-mmacosx-version-min=10.8']
cocoa_dep = dependency('Cocoa', required : applemedia_option)
iosurface_dep = dependency('IOSurface', required : applemedia_option)
applemedia_opengl_dep = dependency('appleframeworks', modules : ['OpenGL'], required : applemedia_option)
endif
endforeach
+if gstvulkan_dep.found() and have_objcpp
+ moltenvk_dep = cc.find_library('MoltenVK', required : false)
+ metal_dep = dependency('appleframeworks', modules : ['Metal'], required : false)
+ if metal_dep.found() and moltenvk_dep.found() and cc.has_header ('MoltenVK/vk_mvk_moltenvk.h')
+ applemedia_frameworks += [moltenvk_dep, gstvulkan_dep, metal_dep]
+ applemedia_sources += [
+ 'videotexturecache-vulkan.mm',
+ 'iosurfacevulkanmemory.c',
+ ]
+ applemedia_args += ['-DAPPLEMEDIA_MOLTENVK']
+ endif
+endif
+
if applemedia_found_deps
gstapplemedia = library('gstapplemedia',
applemedia_sources,
c_args : gst_plugins_bad_args + applemedia_args,
objc_args : gst_plugins_bad_args + applemedia_args + applemedia_objc_args,
+ objcpp_args : gst_plugins_bad_args + applemedia_args + applemedia_objc_args,
link_args : noseh_link_args,
include_directories : [configinc, libsinc],
dependencies : [gstvideo_dep, gstaudio_dep, gstpbutils_dep, gst_dep, gstbase_dep, gstgl_dep] + applemedia_frameworks,
--- /dev/null
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _APPLEMEDIA_METAL_HELPERS_H_
+#define _APPLEMEDIA_METAL_HELPERS_H_
+
+#include <gst/gst.h>
+#include "corevideomemory.h"
+#include "videotexturecache.h"
+
+G_BEGIN_DECLS
+
+VkFormat metal_format_to_vulkan (unsigned int fmt);
+unsigned int video_info_to_metal_format (GstVideoInfo * info,
+ guint plane);
+
+GstMemory * _create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
+ GstVideoInfo * info,
+ guint plane,
+ gsize size,
+ GstVideoTextureCache * cache);
+
+void gst_io_surface_vulkan_memory_set_surface (GstIOSurfaceVulkanMemory * memory,
+ IOSurfaceRef surface);
+
+G_END_DECLS
+#endif /* _APPLEMEDIA_METAL_HELPERS_H_ */
--- /dev/null
+/*
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_CORE_VIDEO_TEXTURE_CACHE_VULKAN_H__
+#define __GST_CORE_VIDEO_TEXTURE_CACHE_VULKAN_H__
+
+#include <gst/vulkan/vulkan.h>
+#include "videotexturecache.h"
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN (gst_video_texture_cache_vulkan_get_type())
+#define GST_VIDEO_TEXTURE_CACHE_VULKAN(o) (G_TYPE_CHECK_INSTANCE_CAST((o), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkan))
+#define GST_VIDEO_TEXTURE_CACHE_VULKAN_CLASS(k) (G_TYPE_CHECK_CLASS_CAST((k), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanClass))
+#define GST_IS_VIDEO_TEXTURE_CACHE_VULKAN(o) (G_TYPE_CHECK_INSTANCE_TYPE((o), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN))
+#define GST_IS_VIDEO_TEXTURE_CACHE_VULKAN_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE((k), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN))
+#define GST_VIDEO_TEXTURE_CACHE_VULKAN_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS((o), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanClass))
+GType gst_video_texture_cache_vulkan_get_type (void);
+
+typedef struct _GstVideoTextureCacheVulkan
+{
+ GstVideoTextureCache parent;
+
+ GstVulkanDevice *device;
+} GstVideoTextureCacheVulkan;
+
+typedef struct _GstVideoTextureCacheVulkanClass
+{
+ GstVideoTextureCacheClass parent_class;
+} GstVideoTextureCacheVulkanClass;
+
+GstVideoTextureCache * gst_video_texture_cache_vulkan_new (GstVulkanDevice * device);
+
+G_END_DECLS
+
+#endif /* __GST_CORE_VIDEO_TEXTURE_CACHE_H__ */
--- /dev/null
+/*
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <Metal/Metal.h>
+
+#if !HAVE_IOS
+#import <AppKit/AppKit.h>
+#endif
+#include "iosurfacevulkanmemory.h"
+
+/* silence macor redefinition warnings */
+#undef VK_USE_PLATFORM_MACOS_MVK
+#undef VK_USE_PLATFORM_IOS_MVK
+#include <MoltenVK/vk_mvk_moltenvk.h>
+#include <MoltenVK/mvk_datatypes.h>
+/* silence macro redefinition warnings */
+#undef VK_USE_PLATFORM_MACOS_MVK
+#undef VK_USE_PLATFORM_IOS_MVK
+
+#include "coremediabuffer.h"
+#include "corevideobuffer.h"
+#include "vtutil.h"
+
+#include "videotexturecache-vulkan.h"
+#include "metal-helpers.h"
+
+typedef struct _GstVideoTextureCacheVulkanPrivate
+{
+ GstBufferPool *pool;
+} GstVideoTextureCacheVulkanPrivate;
+
+G_DEFINE_TYPE_WITH_PRIVATE (GstVideoTextureCacheVulkan, gst_video_texture_cache_vulkan, GST_TYPE_VIDEO_TEXTURE_CACHE);
+
+#define GET_PRIV(instance) \
+ G_TYPE_INSTANCE_GET_PRIVATE (instance, GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanPrivate)
+
+typedef struct _IOSTextureWrapper
+{
+ CVMetalTextureCacheRef cache;
+ CVMetalTextureRef texture;
+} IOSTextureWrapper;
+
+enum
+{
+ PROP_0,
+ PROP_DEVICE,
+};
+
+static GstMemory * gst_video_texture_cache_vulkan_create_memory (GstVideoTextureCache * cache,
+ GstAppleCoreVideoPixelBuffer *gpixbuf, guint plane, gsize size);
+
+GstVideoTextureCache *
+gst_video_texture_cache_vulkan_new (GstVulkanDevice * device)
+{
+ g_return_val_if_fail (GST_IS_VULKAN_DEVICE (device), NULL);
+
+ return (GstVideoTextureCache *) g_object_new (GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN,
+ "device", device, NULL);
+}
+
+static void
+gst_video_texture_cache_vulkan_finalize (GObject * object)
+{
+ GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
+
+#if 0
+ gst_buffer_pool_set_active (cache->pool, FALSE);
+ gst_object_unref (cache->pool);
+#endif
+ gst_object_unref (cache_vulkan->device);
+
+ G_OBJECT_CLASS (gst_video_texture_cache_vulkan_parent_class)->finalize (object);
+}
+
+static void
+gst_video_texture_cache_vulkan_set_property (GObject * object, guint prop_id,
+ const GValue * value, GParamSpec * pspec)
+{
+ GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
+
+ switch (prop_id) {
+ case PROP_DEVICE:
+ cache_vulkan->device = (GstVulkanDevice *) g_value_dup_object (value);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+ break;
+ }
+}
+
+static void
+gst_video_texture_cache_vulkan_get_property (GObject * object, guint prop_id,
+ GValue * value, GParamSpec * pspec)
+{
+ GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
+
+ switch (prop_id) {
+ case PROP_DEVICE:
+ g_value_set_object (value, cache_vulkan->device);
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+ break;
+ }
+}
+
+static void
+gst_video_texture_cache_vulkan_constructed (GObject * object)
+{
+ GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
+
+ g_return_if_fail (GST_IS_VULKAN_DEVICE (cache_vulkan->device));
+
+ gst_io_surface_vulkan_memory_init ();
+#if 0
+ cache->pool = GST_BUFFER_POOL (gst_vulkan_buffer_pool_new (ctx));
+#endif
+}
+
+static void
+gst_video_texture_cache_vulkan_init (GstVideoTextureCacheVulkan * cache_vulkan)
+{
+}
+
+static void
+gst_video_texture_cache_vulkan_class_init (GstVideoTextureCacheVulkanClass *klass)
+{
+ GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+ GstVideoTextureCacheClass *cache_class = (GstVideoTextureCacheClass *) klass;
+
+ gobject_class->set_property = gst_video_texture_cache_vulkan_set_property;
+ gobject_class->get_property = gst_video_texture_cache_vulkan_get_property;
+ gobject_class->constructed = gst_video_texture_cache_vulkan_constructed;
+ gobject_class->finalize = gst_video_texture_cache_vulkan_finalize;
+
+ g_object_class_install_property (gobject_class, PROP_DEVICE,
+ g_param_spec_object ("device", "device",
+ "Associated Vulkan device", GST_TYPE_VULKAN_DEVICE,
+ (GParamFlags) (G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS)));
+
+ cache_class->create_memory = gst_video_texture_cache_vulkan_create_memory;
+}
+
+static GstMemory *
+gst_video_texture_cache_vulkan_create_memory (GstVideoTextureCache * cache,
+ GstAppleCoreVideoPixelBuffer *gpixbuf, guint plane, gsize size)
+{
+ return (GstMemory *) gpixbuf;
+}
+
+VkFormat
+metal_format_to_vulkan (unsigned int fmt)
+{
+ MTLPixelFormat mtl_fmt = (MTLPixelFormat) fmt;
+
+ switch (mtl_fmt) {
+ case MTLPixelFormatRGBA8Unorm:
+ return VK_FORMAT_R8G8B8A8_UNORM;
+ case MTLPixelFormatRG8Unorm:
+ return VK_FORMAT_R8G8_UNORM;
+ case MTLPixelFormatR8Unorm:
+ return VK_FORMAT_R8_UNORM;
+ default:
+ g_assert_not_reached ();
+ }
+}
+
+unsigned int
+video_info_to_metal_format (GstVideoInfo * info, guint plane)
+{
+ switch (GST_VIDEO_INFO_FORMAT (info)) {
+ case GST_VIDEO_FORMAT_BGRA:
+ return (unsigned int) MTLPixelFormatRGBA8Unorm;
+ case GST_VIDEO_FORMAT_NV12:
+ if (plane == 0)
+ return (unsigned int) MTLPixelFormatR8Unorm;
+ else
+ return (unsigned int) MTLPixelFormatRG8Unorm;
+ default:
+ g_assert_not_reached ();
+ }
+}
+
+GstMemory *
+_create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
+ GstVideoInfo * info, guint plane, gsize size, GstVideoTextureCache * cache)
+{
+ GstIOSurfaceVulkanMemory *mem;
+ CVPixelBufferRef pixel_buf = gpixbuf->buf;
+ IOSurfaceRef surface = CVPixelBufferGetIOSurface (pixel_buf);
+ GstVideoTextureCacheVulkan *cache_vulkan =
+ GST_VIDEO_TEXTURE_CACHE_VULKAN (cache);
+ MTLPixelFormat fmt = video_info_to_metal_format (info, plane);
+
+ CFRetain (pixel_buf);
+ mem = gst_io_surface_vulkan_memory_wrapped (cache_vulkan->device,
+ surface, fmt, info, plane, pixel_buf, (GDestroyNotify) CFRelease);
+
+ if (!mem)
+ return NULL;
+
+ return GST_MEMORY_CAST (mem);
+}
+
+typedef struct _IOSurfaceTextureWrapper
+{
+ CVPixelBufferRef pixbuf;
+ gpointer texture; /* id<MTLTexture> */
+} IOSurfaceTextureWrapper;
+
+static void
+free_texture_wrapper (IOSurfaceTextureWrapper * wrapper)
+{
+ CFRelease (wrapper->pixbuf);
+ id<MTLTexture> tex = (__bridge_transfer id<MTLTexture>) wrapper->texture;
+ (void) tex;
+ g_free (wrapper);
+}
+
+static MTLTextureDescriptor *
+gst_new_mtl_tex_descripter_from_memory (GstIOSurfaceVulkanMemory * memory)
+{
+ GstVulkanImageMemory *vk_mem = (GstVulkanImageMemory *) memory;
+ MTLTextureDescriptor* tex_desc = [MTLTextureDescriptor new];
+ tex_desc.pixelFormat = mvkMTLPixelFormatFromVkFormat (vk_mem->create_info.format);
+ tex_desc.textureType = mvkMTLTextureTypeFromVkImageType (vk_mem->create_info.imageType, 0, false);
+ tex_desc.width = vk_mem->create_info.extent.width;
+ tex_desc.height = vk_mem->create_info.extent.height;
+ tex_desc.depth = vk_mem->create_info.extent.depth;
+ tex_desc.mipmapLevelCount = vk_mem->create_info.mipLevels;
+ tex_desc.sampleCount = mvkSampleCountFromVkSampleCountFlagBits(vk_mem->create_info.samples);
+ tex_desc.arrayLength = vk_mem->create_info.arrayLayers;
+ tex_desc.usage = MTLTextureUsageShaderRead | MTLTextureUsagePixelFormatView;//mvkMTLTextureUsageFromVkImageUsageFlags(vk_mem->create_info.usage);
+ tex_desc.storageMode = MTLStorageModePrivate;//MTLStorageModeManaged; /* IOSurface == shared/managed */
+ tex_desc.cpuCacheMode = MTLCPUCacheModeDefaultCache;//mvkMTLCPUCacheModeFromVkMemoryPropertyFlags(vk_mem->vk_mem->properties);
+
+ return tex_desc;
+}
+
+void
+gst_io_surface_vulkan_memory_set_surface (GstIOSurfaceVulkanMemory * memory,
+ IOSurfaceRef surface)
+{
+ GstVulkanImageMemory *vk_mem = (GstVulkanImageMemory *) memory;
+
+ if (memory->surface) {
+ IOSurfaceDecrementUseCount (memory->surface);
+ } else {
+ g_assert (vk_mem->notify == (GDestroyNotify) CFRelease);
+ g_assert (vk_mem->user_data != NULL);
+ }
+
+ memory->surface = surface;
+ if (surface) {
+ id<MTLDevice> mtl_dev = nil;
+ id<MTLTexture> texture = nil;
+ VkPhysicalDevice gpu;
+
+ IOSurfaceIncrementUseCount (surface);
+
+ gpu = gst_vulkan_device_get_physical_device (vk_mem->device);
+ vkGetMTLDeviceMVK (gpu, &mtl_dev);
+
+ /* We cannot use vkUseIOSurfaceMVK() for multi-planer as MoltenVK does not
+ * support them. */
+
+ MTLTextureDescriptor *tex_desc = gst_new_mtl_tex_descripter_from_memory (memory);
+ texture = [mtl_dev newTextureWithDescriptor:tex_desc iosurface:surface plane:memory->plane];
+
+ IOSurfaceTextureWrapper *texture_data = g_new0 (IOSurfaceTextureWrapper, 1);
+ texture_data->pixbuf = (CVPixelBufferRef) vk_mem->user_data;
+ texture_data->texture = (__bridge_retained gpointer) texture;
+
+ VkResult err = vkSetMTLTextureMVK (memory->vulkan_mem.image, texture);
+ GST_DEBUG ("bound texture %p to image %p: 0x%x", texture, memory->vulkan_mem.image,
+ err);
+
+ vk_mem->user_data = texture_data;
+ vk_mem->notify = (GDestroyNotify) free_texture_wrapper;
+ }
+}
#include "corevideobuffer.h"
#include "coremediabuffer.h"
#include "videotexturecache-gl.h"
+#if defined(APPLEMEDIA_MOLTENVK)
+#include "videotexturecache-vulkan.h"
+#endif
GST_DEBUG_CATEGORY_STATIC (gst_vtdec_debug_category);
#define GST_CAT_DEFAULT gst_vtdec_debug_category
CFSTR ("RequireHardwareAcceleratedVideoDecoder");
#endif
+#if defined(APPLEMEDIA_MOLTENVK)
#define VIDEO_SRC_CAPS \
GST_VIDEO_CAPS_MAKE("NV12") ";" \
GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
"NV12") ", " \
- "texture-target = (string) rectangle;"
+ "texture-target = (string) rectangle ; " \
+ GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,\
+ "NV12")
+#else
+#define VIDEO_SRC_CAPS \
+ GST_VIDEO_CAPS_MAKE("NV12") ";" \
+ GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
+ "NV12") ", " \
+ "texture-target = (string) rectangle "
+#endif
G_DEFINE_TYPE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER);
CFRelease (vtdec->format_description);
vtdec->format_description = NULL;
+#if defined(APPLEMEDIA_MOLTENVK)
+ gst_clear_object (&vtdec->device);
+ gst_clear_object (&vtdec->instance);
+#endif
+
GST_DEBUG_OBJECT (vtdec, "stop");
return TRUE;
}
static void
-setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
+setup_texture_cache (GstVtdec * vtdec)
{
GstVideoCodecState *output_state;
- g_return_if_fail (vtdec->texture_cache == NULL);
-
output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
- vtdec->texture_cache = gst_video_texture_cache_gl_new (context);
gst_video_texture_cache_set_format (vtdec->texture_cache,
GST_VIDEO_FORMAT_NV12, output_state->caps);
gst_video_codec_state_unref (output_state);
GstVtdec *vtdec;
OSStatus err = noErr;
GstCapsFeatures *features = NULL;
- gboolean output_textures;
+ gboolean output_textures = FALSE;
+#if defined(APPLEMEDIA_MOLTENVK)
+ gboolean output_vulkan = FALSE;
+#endif
vtdec = GST_VTDEC (decoder);
if (vtdec->session)
GST_GL_TEXTURE_TARGET_2D_STR,
#endif
NULL);
+
+#if defined(APPLEMEDIA_MOLTENVK)
+ output_vulkan =
+ gst_caps_features_contains (features,
+ GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE);
+#endif
}
gst_caps_unref (caps);
}
}
- if (vtdec->texture_cache != NULL && !output_textures) {
+ if (vtdec->texture_cache != NULL
+ && ((GST_IS_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache)
+ && !output_textures)
+#if defined(APPLEMEDIA_MOLTENVK)
+ || (GST_IS_VIDEO_TEXTURE_CACHE_VULKAN (vtdec->texture_cache)
+ && !output_vulkan)
+#endif
+ )) {
g_object_unref (vtdec->texture_cache);
vtdec->texture_cache = NULL;
}
- if (err == noErr && output_textures) {
- GstVideoTextureCacheGL *cache_gl = NULL;
-
- if (vtdec->texture_cache)
- cache_gl = GST_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache);
-
- /* call this regardless of whether caps have changed or not since a new
- * local context could have become available
- */
- if (!vtdec->ctxh)
- vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
- gst_gl_context_helper_ensure_context (vtdec->ctxh);
-
- GST_INFO_OBJECT (vtdec, "pushing textures, context %p old context %p",
- vtdec->ctxh->context, cache_gl ? cache_gl->ctx : NULL);
-
- if (cache_gl && cache_gl->ctx != vtdec->ctxh->context) {
- g_object_unref (vtdec->texture_cache);
- vtdec->texture_cache = NULL;
+ if (err == noErr) {
+ if (output_textures) {
+ GstVideoTextureCacheGL *cache_gl = NULL;
+
+ if (vtdec->texture_cache)
+ cache_gl = GST_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache);
+
+ /* call this regardless of whether caps have changed or not since a new
+ * local context could have become available
+ */
+ if (!vtdec->ctxh)
+ vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
+ gst_gl_context_helper_ensure_context (vtdec->ctxh);
+
+ GST_INFO_OBJECT (vtdec, "pushing GL textures, context %p old context %p",
+ vtdec->ctxh->context, cache_gl ? cache_gl->ctx : NULL);
+
+ if (cache_gl && cache_gl->ctx != vtdec->ctxh->context) {
+ g_object_unref (vtdec->texture_cache);
+ vtdec->texture_cache = NULL;
+ }
+ if (!vtdec->texture_cache) {
+ vtdec->texture_cache =
+ gst_video_texture_cache_gl_new (vtdec->ctxh->context);
+ setup_texture_cache (vtdec);
+ }
+ }
+#if defined(APPLEMEDIA_MOLTENVK)
+ if (output_vulkan) {
+ GstVideoTextureCacheVulkan *cache_vulkan = NULL;
+
+ if (vtdec->texture_cache)
+ cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (vtdec->texture_cache);
+
+ gst_vulkan_ensure_element_data (GST_ELEMENT (vtdec), NULL,
+ &vtdec->instance);
+
+ if (!gst_vulkan_device_run_context_query (GST_ELEMENT (vtdec),
+ &vtdec->device)) {
+ GError *error = NULL;
+ GST_DEBUG_OBJECT (vtdec, "No device retrieved from peer elements");
+ if (!(vtdec->device =
+ gst_vulkan_instance_create_device (vtdec->instance, &error))) {
+ GST_ELEMENT_ERROR (vtdec, RESOURCE, NOT_FOUND,
+ ("Failed to create vulkan device"), ("%s", error->message));
+ g_clear_error (&error);
+ return FALSE;
+ }
+ }
+
+ GST_INFO_OBJECT (vtdec, "pushing vulkan images, device %" GST_PTR_FORMAT
+ " old device %" GST_PTR_FORMAT, vtdec->device,
+ cache_vulkan ? cache_vulkan->device : NULL);
+
+ if (cache_vulkan && cache_vulkan->device != vtdec->device) {
+ g_object_unref (vtdec->texture_cache);
+ vtdec->texture_cache = NULL;
+ }
+ if (!vtdec->texture_cache) {
+ vtdec->texture_cache =
+ gst_video_texture_cache_vulkan_new (vtdec->device);
+ setup_texture_cache (vtdec);
+ }
}
- if (!vtdec->texture_cache)
- setup_texture_cache (vtdec, vtdec->ctxh->context);
+#endif
}
if (prevcaps)
vtdec->ctxh = gst_gl_context_helper_new (element);
gst_gl_handle_set_context (element, context,
&vtdec->ctxh->display, &vtdec->ctxh->other_context);
+
+#if defined (APPLEMEDIA_MOLTENVK)
+ gst_vulkan_handle_set_context (element, context, NULL, &vtdec->instance);
+#endif
+
GST_ELEMENT_CLASS (gst_vtdec_parent_class)->set_context (element, context);
}
#include <VideoToolbox/VideoToolbox.h>
#include "videotexturecache.h"
#include "glcontexthelper.h"
+#if defined(APPLEMEDIA_MOLTENVK)
+#include <gst/vulkan/vulkan.h>
+#endif
G_BEGIN_DECLS
GstVideoTextureCache *texture_cache;
GstGLContextHelper *ctxh;
+#if defined(APPLEMEDIA_MOLTENVK)
+ GstVulkanInstance *instance;
+ GstVulkanDevice *device;
+#endif
+
gboolean require_hardware;
};
projects / platform / upstream / gstreamer.git / commitdiff