2 * Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
3 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
22 * SECTION:plugin-mediafoundation
24 * Microsoft MediaFoundation plugin.
26 * This plugin consists of various hardware/software video encoders
27 * software audio encoders, and video capture (from webcam) elements.
29 * GstMediaFoundation plugin supports H.264/AVC, H.265/HEVC, VP9, codecs for
30 * hardware-accelerate encoding.
32 * However, depending on the hardware it runs on, some elements might not be
33 * registered in case that underlying hardware doesn't support the feature.
35 * Moreover, depending on hardware vendor's MediaFoundation implementation,
36 * secendary GPU may not be usable. In that case, user could use vendor
37 * specific plugins, Intel Media SDK and NVCODEC plugins for example.
39 * For a system with multiple MediaFoundation compatible hardwares (i.e., GPU),
40 * there can be multiple plugin features having the same role.
41 * Also, there would be additional software video encoder element the system
44 * The naming rule for the non-primary encoder is `mf{codec}device{index}enc`
45 * where `index` is an arbitrary index number of hardware starting from 1.
47 * To get a list of all available elements, user can run
49 * gst-inspect-1.0.exe mediafoundation
60 #include "gstmfconfig.h"
62 #include <winapifamily.h>
66 #include <gst/video/video.h>
67 #include "gstmfvideosrc.h"
68 #include "gstmfdevice.h"
69 #include "gstmfutils.h"
70 #include "gstmfh264enc.h"
71 #include "gstmfh265enc.h"
72 #include "gstmfvp9enc.h"
73 #include "gstmfaacenc.h"
74 #include "gstmfmp3enc.h"
75 #include "gstmfaacdec.h"
76 #include "gstmfmp3dec.h"
79 #include <gst/d3d11/gstd3d11.h>
80 #include <gstmfplatloader.h>
83 GST_DEBUG_CATEGORY (gst_mf_debug);
84 GST_DEBUG_CATEGORY (gst_mf_utils_debug);
85 GST_DEBUG_CATEGORY (gst_mf_source_object_debug);
86 GST_DEBUG_CATEGORY (gst_mf_transform_debug);
87 GST_DEBUG_CATEGORY (gst_mf_video_buffer_debug);
88 GST_DEBUG_CATEGORY (gst_mf_video_encoder_debug);
90 #define GST_CAT_DEFAULT gst_mf_debug
93 using namespace Microsoft::WRL;
97 plugin_deinit (gpointer data)
102 #if GST_MF_HAVE_D3D11
104 get_d3d11_devices (void)
106 GList *ret = nullptr;
109 ComPtr < IMFVideoSampleAllocatorEx > allocator;
111 /* Check whether we can use IMFVideoSampleAllocatorEx interface */
112 hr = GstMFCreateVideoSampleAllocatorEx (IID_IMFVideoSampleAllocatorEx,
114 if (!gst_mf_result (hr)) {
115 GST_DEBUG ("IMFVideoSampleAllocatorEx interface is unavailable");
119 /* AMD seems supporting up to 12 cards, and 8 for NVIDIA */
120 for (i = 0; i < 12; i++) {
121 GstD3D11Device *device;
122 gboolean is_hardware = FALSE;
123 const GstD3D11Format *d3d11_format;
124 ID3D11Device *device_handle;
125 D3D11_FEATURE_DATA_D3D11_OPTIONS4 options = { 0, };
128 device = gst_d3d11_device_new (i, D3D11_CREATE_DEVICE_VIDEO_SUPPORT);
133 g_object_get (device, "hardware", &is_hardware, nullptr);
136 GST_DEBUG_OBJECT (device, "Given d3d11 device is not for hardware");
137 gst_object_unref (device);
141 /* device can support NV12 format? */
143 gst_d3d11_device_format_from_gst (device, GST_VIDEO_FORMAT_NV12);
144 if (!d3d11_format || d3d11_format->dxgi_format != DXGI_FORMAT_NV12) {
145 GST_DEBUG_OBJECT (device,
146 "Given d3d11 device cannot support NV12 format");
147 gst_object_unref (device);
151 /* device can support ExtendedNV12SharedTextureSupported?
153 * NOTE: we will make use of per encoder object d3d11 device without
154 * sharing it in a pipeline because MF needs D3D11_CREATE_DEVICE_VIDEO_SUPPORT
155 * but the flag doesn't used for the other our use cases.
156 * So we need texture sharing feature so that we can copy d3d11 texture into
157 * MF specific texture pool without download texture */
159 device_handle = gst_d3d11_device_get_device_handle (device);
160 hr = device_handle->CheckFeatureSupport (D3D11_FEATURE_D3D11_OPTIONS4,
161 &options, sizeof (options));
162 if (!gst_d3d11_result (hr, device) ||
163 !options.ExtendedNV12SharedTextureSupported) {
164 GST_DEBUG_OBJECT (device,
165 "Given d3d11 device cannot support NV12 format for shared texture");
166 gst_object_unref (device);
170 /* can we bind NV12 texture for encoder? */
171 hr = device_handle->CheckFormatSupport (DXGI_FORMAT_NV12, &supported);
173 if (!gst_d3d11_result (hr, device)) {
174 GST_DEBUG_OBJECT (device, "Couldn't query format support");
175 gst_object_unref (device);
177 } else if ((supported & D3D11_FORMAT_SUPPORT_VIDEO_ENCODER) == 0) {
178 GST_DEBUG_OBJECT (device, "We cannot bind NV12 format for encoding");
179 gst_object_unref (device);
183 ret = g_list_append (ret, device);
191 plugin_init (GstPlugin * plugin)
194 guint rank = GST_RANK_SECONDARY;
195 GList *device_list = nullptr;
197 GST_DEBUG_CATEGORY_INIT (gst_mf_debug, "mf", 0, "media foundation");
198 GST_DEBUG_CATEGORY_INIT (gst_mf_utils_debug,
199 "mfutils", 0, "media foundation utility functions");
200 GST_DEBUG_CATEGORY_INIT (gst_mf_source_object_debug,
201 "mfsourceobject", 0, "mfsourceobject");
202 GST_DEBUG_CATEGORY_INIT (gst_mf_transform_debug,
203 "mftransform", 0, "mftransform");
204 GST_DEBUG_CATEGORY_INIT (gst_mf_video_buffer_debug,
205 "mfvideobuffer", 0, "mfvideobuffer");
206 GST_DEBUG_CATEGORY_INIT (gst_mf_video_encoder_debug,
207 "mfvideoencoder", 0, "mfvideoencoder");
209 hr = MFStartup (MF_VERSION, MFSTARTUP_NOSOCKET);
210 if (!gst_mf_result (hr)) {
211 GST_WARNING ("MFStartup failure, hr: 0x%x", hr);
215 /* mfvideosrc should be primary rank for UWP */
216 #if (GST_MF_WINAPI_APP && !GST_MF_WINAPI_DESKTOP)
217 rank = GST_RANK_PRIMARY + 1;
220 /* FIXME: In order to create MFT for a specific GPU, MFTEnum2() API is
221 * required API but it's desktop only.
222 * So, resulting MFT and D3D11 might not be compatible in case of multi-GPU
223 * environment on UWP. */
224 #if GST_MF_HAVE_D3D11
225 if (gst_mf_plat_load_library ())
226 device_list = get_d3d11_devices ();
229 gst_element_register (plugin, "mfvideosrc", rank, GST_TYPE_MF_VIDEO_SRC);
230 gst_device_provider_register (plugin, "mfdeviceprovider",
231 rank, GST_TYPE_MF_DEVICE_PROVIDER);
233 gst_mf_h264_enc_plugin_init (plugin, GST_RANK_SECONDARY, device_list);
234 gst_mf_h265_enc_plugin_init (plugin, GST_RANK_SECONDARY, device_list);
235 gst_mf_vp9_enc_plugin_init (plugin, GST_RANK_SECONDARY, device_list);
238 g_list_free_full (device_list, gst_object_unref);
240 gst_mf_aac_enc_plugin_init (plugin, GST_RANK_SECONDARY);
241 gst_mf_mp3_enc_plugin_init (plugin, GST_RANK_SECONDARY);
242 gst_mf_aac_dec_plugin_init (plugin, GST_RANK_SECONDARY);
243 gst_mf_mp3_dec_plugin_init (plugin, GST_RANK_SECONDARY);
245 /* So that call MFShutdown() when this plugin is no more used
246 * (i.e., gst_deinit). Otherwise valgrind-like tools would complain
247 * about un-released media foundation resources.
249 * NOTE: MFStartup and MFShutdown can be called multiple times, but the number
250 * of each MFStartup and MFShutdown call should be identical. This rule is
251 * simliar to that of CoInitialize/CoUninitialize pair */
252 g_object_set_data_full (G_OBJECT (plugin),
253 "plugin-mediafoundation-shutdown", (gpointer) "shutdown-data",
254 (GDestroyNotify) plugin_deinit);
259 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
262 "Microsoft Media Foundation plugin",
263 plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)