2 * Copyright (C) 2020 Seungha Yang <seungha.yang@navercorp.com>
3 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
26 #include "gstmfvideoenc.h"
28 #include "gstmfvideobuffer.h"
37 using namespace Microsoft::WRL;
41 GST_DEBUG_CATEGORY_EXTERN (gst_mf_video_enc_debug);
42 #define GST_CAT_DEFAULT gst_mf_video_enc_debug
47 #define gst_mf_video_enc_parent_class parent_class
48 G_DEFINE_ABSTRACT_TYPE (GstMFVideoEnc, gst_mf_video_enc,
49 GST_TYPE_VIDEO_ENCODER);
51 static void gst_mf_video_enc_dispose (GObject * object);
52 static void gst_mf_video_enc_set_context (GstElement * element,
53 GstContext * context);
54 static gboolean gst_mf_video_enc_open (GstVideoEncoder * enc);
55 static gboolean gst_mf_video_enc_close (GstVideoEncoder * enc);
56 static gboolean gst_mf_video_enc_start (GstVideoEncoder * enc);
57 static gboolean gst_mf_video_enc_set_format (GstVideoEncoder * enc,
58 GstVideoCodecState * state);
59 static GstFlowReturn gst_mf_video_enc_handle_frame (GstVideoEncoder * enc,
60 GstVideoCodecFrame * frame);
61 static GstFlowReturn gst_mf_video_enc_finish (GstVideoEncoder * enc);
62 static gboolean gst_mf_video_enc_flush (GstVideoEncoder * enc);
63 static gboolean gst_mf_video_enc_propose_allocation (GstVideoEncoder * enc,
65 static gboolean gst_mf_video_enc_sink_query (GstVideoEncoder * enc,
67 static gboolean gst_mf_video_enc_src_query (GstVideoEncoder * enc,
70 static HRESULT gst_mf_video_on_new_sample (GstMFTransform * object,
71 IMFSample * sample, GstMFVideoEnc * self);
74 gst_mf_video_enc_class_init (GstMFVideoEncClass * klass)
76 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
77 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
78 GstVideoEncoderClass *videoenc_class = GST_VIDEO_ENCODER_CLASS (klass);
80 gobject_class->dispose = gst_mf_video_enc_dispose;
82 element_class->set_context = GST_DEBUG_FUNCPTR (gst_mf_video_enc_set_context);
84 videoenc_class->open = GST_DEBUG_FUNCPTR (gst_mf_video_enc_open);
85 videoenc_class->close = GST_DEBUG_FUNCPTR (gst_mf_video_enc_close);
86 videoenc_class->start = GST_DEBUG_FUNCPTR (gst_mf_video_enc_start);
87 videoenc_class->set_format = GST_DEBUG_FUNCPTR (gst_mf_video_enc_set_format);
88 videoenc_class->handle_frame =
89 GST_DEBUG_FUNCPTR (gst_mf_video_enc_handle_frame);
90 videoenc_class->finish = GST_DEBUG_FUNCPTR (gst_mf_video_enc_finish);
91 videoenc_class->flush = GST_DEBUG_FUNCPTR (gst_mf_video_enc_flush);
92 videoenc_class->propose_allocation =
93 GST_DEBUG_FUNCPTR (gst_mf_video_enc_propose_allocation);
94 videoenc_class->sink_query = GST_DEBUG_FUNCPTR (gst_mf_video_enc_sink_query);
95 videoenc_class->src_query = GST_DEBUG_FUNCPTR (gst_mf_video_enc_src_query);
97 gst_type_mark_as_plugin_api (GST_TYPE_MF_VIDEO_ENC, (GstPluginAPIFlags) 0);
101 gst_mf_video_enc_init (GstMFVideoEnc * self)
106 gst_mf_video_enc_dispose (GObject * object)
108 #if GST_MF_HAVE_D3D11
109 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (object);
111 gst_clear_object (&self->d3d11_device);
112 gst_clear_object (&self->other_d3d11_device);
115 G_OBJECT_CLASS (parent_class)->dispose (object);
119 gst_mf_video_enc_set_context (GstElement * element, GstContext * context)
121 #if GST_MF_HAVE_D3D11
122 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (element);
123 GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (self);
124 GstMFVideoEncDeviceCaps *device_caps = &klass->device_caps;
126 if (device_caps->d3d11_aware) {
127 gst_d3d11_handle_set_context_for_adapter_luid (element, context,
128 device_caps->adapter_luid, &self->other_d3d11_device);
132 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
136 gst_mf_video_enc_open (GstVideoEncoder * enc)
138 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
139 GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (enc);
140 GstMFVideoEncDeviceCaps *device_caps = &klass->device_caps;
141 GstMFTransformEnumParams enum_params = { 0, };
142 MFT_REGISTER_TYPE_INFO output_type;
145 #if GST_MF_HAVE_D3D11
146 if (device_caps->d3d11_aware) {
148 ID3D11Device *device_handle;
149 ComPtr < ID3D10Multithread > multi_thread;
150 GstD3D11Device *device;
152 if (!gst_d3d11_ensure_element_data_for_adapter_luid (GST_ELEMENT (self),
153 device_caps->adapter_luid, &self->other_d3d11_device)) {
154 GST_ERROR_OBJECT (self, "Other d3d11 device is unavailable");
158 /* Create our own device with D3D11_CREATE_DEVICE_VIDEO_SUPPORT */
160 gst_d3d11_device_new_for_adapter_luid (device_caps->adapter_luid,
161 D3D11_CREATE_DEVICE_VIDEO_SUPPORT);
162 if (!self->d3d11_device) {
163 GST_ERROR_OBJECT (self, "Couldn't create internal d3d11 device");
164 gst_clear_object (&self->other_d3d11_device);
168 device = self->d3d11_device;
170 hr = MFCreateDXGIDeviceManager (&self->reset_token, &self->device_manager);
171 if (!gst_mf_result (hr)) {
172 GST_ERROR_OBJECT (self, "Couldn't create DXGI device manager");
173 gst_clear_object (&self->other_d3d11_device);
174 gst_clear_object (&self->d3d11_device);
178 device_handle = gst_d3d11_device_get_device_handle (device);
179 /* Enable multi thread protection as this device will be shared with
181 hr = device_handle->QueryInterface (IID_PPV_ARGS (&multi_thread));
182 if (!gst_d3d11_result (hr, device)) {
183 GST_WARNING_OBJECT (self,
184 "device doesn't suport ID3D10Multithread interface");
185 gst_clear_object (&self->other_d3d11_device);
186 gst_clear_object (&self->d3d11_device);
189 multi_thread->SetMultithreadProtected (TRUE);
191 hr = self->device_manager->ResetDevice ((IUnknown *) device_handle,
193 if (!gst_mf_result (hr)) {
194 GST_ERROR_OBJECT (self, "Couldn't reset device with given d3d11 device");
195 gst_clear_object (&self->other_d3d11_device);
196 gst_clear_object (&self->d3d11_device);
202 output_type.guidMajorType = MFMediaType_Video;
203 output_type.guidSubtype = klass->codec_id;
205 enum_params.category = MFT_CATEGORY_VIDEO_ENCODER;
206 enum_params.enum_flags = klass->enum_flags;
207 enum_params.output_typeinfo = &output_type;
208 enum_params.device_index = klass->device_index;
210 if (device_caps->d3d11_aware)
211 enum_params.adapter_luid = device_caps->adapter_luid;
213 GST_DEBUG_OBJECT (self,
214 "Create MFT with enum flags: 0x%x, device index: %d, d3d11 aware: %d, "
215 "adapter-luid %" G_GINT64_FORMAT, klass->enum_flags, klass->device_index,
216 device_caps->d3d11_aware, device_caps->adapter_luid);
218 self->transform = gst_mf_transform_new (&enum_params);
219 ret = !!self->transform;
222 GST_ERROR_OBJECT (self, "Cannot create MFT object");
226 /* In case of hardware MFT, it will be running on async mode.
227 * And new output sample callback will be called from Media Foundation's
228 * internal worker queue thread */
229 if (self->transform &&
230 (enum_params.enum_flags & MFT_ENUM_FLAG_HARDWARE) ==
231 MFT_ENUM_FLAG_HARDWARE) {
232 self->async_mft = TRUE;
233 gst_mf_transform_set_new_sample_callback (self->transform,
234 (GstMFTransformNewSampleCallback) gst_mf_video_on_new_sample, self);
236 self->async_mft = FALSE;
243 gst_mf_video_enc_close (GstVideoEncoder * enc)
245 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
247 gst_clear_object (&self->transform);
249 if (self->input_state) {
250 gst_video_codec_state_unref (self->input_state);
251 self->input_state = NULL;
253 #if GST_MF_HAVE_D3D11
254 if (self->device_manager) {
255 self->device_manager->Release ();
256 self->device_manager = nullptr;
259 if (self->mf_allocator) {
260 self->mf_allocator->UninitializeSampleAllocator ();
261 self->mf_allocator->Release ();
262 self->mf_allocator = NULL;
265 gst_clear_object (&self->other_d3d11_device);
266 gst_clear_object (&self->d3d11_device);
273 gst_mf_video_enc_start (GstVideoEncoder * enc)
275 /* Media Foundation Transform will shift PTS in case that B-frame is enabled.
276 * We need to adjust DTS correspondingly */
277 gst_video_encoder_set_min_pts (enc, GST_SECOND * 60 * 60 * 1000);
283 gst_mf_video_enc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state)
285 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
286 GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (enc);
287 GstVideoInfo *info = &state->info;
288 ComPtr < IMFMediaType > in_type;
289 ComPtr < IMFMediaType > out_type;
290 GList *input_types = NULL;
295 GST_DEBUG_OBJECT (self, "Set format");
297 gst_mf_video_enc_finish (enc);
299 self->mf_pts_offset = 0;
300 self->has_reorder_frame = FALSE;
301 self->last_ret = GST_FLOW_OK;
303 if (self->input_state)
304 gst_video_codec_state_unref (self->input_state);
305 self->input_state = gst_video_codec_state_ref (state);
307 if (!gst_mf_transform_open (self->transform)) {
308 GST_ERROR_OBJECT (self, "Failed to open MFT");
311 #if GST_MF_HAVE_D3D11
312 if (self->device_manager) {
313 if (!gst_mf_transform_set_device_manager (self->transform,
314 self->device_manager)) {
315 GST_ERROR_OBJECT (self, "Couldn't set device manager");
318 GST_DEBUG_OBJECT (self, "set device manager done");
323 hr = MFCreateMediaType (out_type.GetAddressOf ());
324 if (!gst_mf_result (hr))
327 hr = out_type->SetGUID (MF_MT_MAJOR_TYPE, MFMediaType_Video);
328 if (!gst_mf_result (hr))
331 if (klass->set_option) {
332 if (!klass->set_option (self, self->input_state, out_type.Get ())) {
333 GST_ERROR_OBJECT (self, "subclass failed to set option");
338 fps_n = GST_VIDEO_INFO_FPS_N (info);
339 fps_d = GST_VIDEO_INFO_FPS_D (info);
340 if (fps_n <= 0 || fps_d <= 0) {
341 /* XXX: not sure why. NVIDIA MFT accepts 0/1 framerate, but Intel or
342 * Microsoft's software MFT doesn't accept 0/1 framerate.
343 * Need to set something meaningful value here therefore */
348 hr = MFSetAttributeRatio (out_type.Get (), MF_MT_FRAME_RATE, fps_n, fps_d);
349 if (!gst_mf_result (hr)) {
350 GST_ERROR_OBJECT (self,
351 "Couldn't set framerate %d/%d, hr: 0x%x", (guint) hr);
355 hr = MFSetAttributeSize (out_type.Get (), MF_MT_FRAME_SIZE,
356 GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
357 if (!gst_mf_result (hr)) {
358 GST_ERROR_OBJECT (self,
359 "Couldn't set resolution %dx%d, hr: 0x%x", GST_VIDEO_INFO_WIDTH (info),
360 GST_VIDEO_INFO_HEIGHT (info), (guint) hr);
364 hr = MFSetAttributeRatio (out_type.Get (), MF_MT_PIXEL_ASPECT_RATIO,
365 GST_VIDEO_INFO_PAR_N (info), GST_VIDEO_INFO_PAR_D (info));
366 if (!gst_mf_result (hr)) {
367 GST_ERROR_OBJECT (self, "Couldn't set par %d/%d",
368 GST_VIDEO_INFO_PAR_N (info), GST_VIDEO_INFO_PAR_D (info));
372 hr = out_type->SetUINT32 (MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
373 if (!gst_mf_result (hr)) {
374 GST_ERROR_OBJECT (self,
375 "Couldn't set interlace mode, hr: 0x%x", (guint) hr);
379 if (!gst_mf_transform_set_output_type (self->transform, out_type.Get ())) {
380 GST_ERROR_OBJECT (self, "Couldn't set output type");
384 if (!gst_mf_transform_get_input_available_types (self->transform,
386 GST_ERROR_OBJECT (self, "Couldn't get available input types");
390 for (iter = input_types; iter; iter = g_list_next (iter)) {
391 GstVideoFormat format;
393 IMFMediaType *type = (IMFMediaType *) iter->data;
395 hr = type->GetGUID (MF_MT_SUBTYPE, &subtype);
396 if (!gst_mf_result (hr))
399 format = gst_mf_video_subtype_to_video_format (&subtype);
400 if (format != GST_VIDEO_INFO_FORMAT (info))
406 g_list_free_full (input_types, (GDestroyNotify) gst_mf_media_type_release);
409 GST_ERROR_OBJECT (self,
410 "Couldn't convert input caps %" GST_PTR_FORMAT " to media type",
415 hr = MFSetAttributeSize (in_type.Get (), MF_MT_FRAME_SIZE,
416 GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
417 if (!gst_mf_result (hr)) {
418 GST_ERROR_OBJECT (self, "Couldn't set frame size %dx%d",
419 GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
423 hr = in_type->SetUINT32 (MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
424 if (!gst_mf_result (hr)) {
425 GST_ERROR_OBJECT (self,
426 "Couldn't set interlace mode, hr: 0x%x", (guint) hr);
430 hr = MFSetAttributeRatio (in_type.Get (), MF_MT_PIXEL_ASPECT_RATIO,
431 GST_VIDEO_INFO_PAR_N (info), GST_VIDEO_INFO_PAR_D (info));
432 if (!gst_mf_result (hr)) {
433 GST_ERROR_OBJECT (self, "Couldn't set par %d/%d",
434 GST_VIDEO_INFO_PAR_N (info), GST_VIDEO_INFO_PAR_D (info));
438 hr = MFSetAttributeRatio (in_type.Get (), MF_MT_FRAME_RATE, fps_n, fps_d);
439 if (!gst_mf_result (hr)) {
440 GST_ERROR_OBJECT (self, "Couldn't set framerate ratio %d/%d", fps_n, fps_d);
444 hr = in_type->SetUINT32 (MF_MT_DEFAULT_STRIDE,
445 GST_VIDEO_INFO_PLANE_STRIDE (info, 0));
446 if (!gst_mf_result (hr)) {
447 GST_ERROR_OBJECT (self, "Couldn't set default stride");
451 if (!gst_mf_transform_set_input_type (self->transform, in_type.Get ())) {
452 GST_ERROR_OBJECT (self, "Couldn't set input media type");
456 g_assert (klass->set_src_caps != NULL);
457 if (!klass->set_src_caps (self, self->input_state, out_type.Get ())) {
458 GST_ERROR_OBJECT (self, "subclass couldn't set src caps");
461 #if GST_MF_HAVE_D3D11
462 if (self->mf_allocator) {
463 self->mf_allocator->UninitializeSampleAllocator ();
464 self->mf_allocator->Release ();
465 self->mf_allocator = NULL;
468 /* Check whether upstream is d3d11 element */
470 GstCapsFeatures *features;
471 ComPtr < IMFVideoSampleAllocatorEx > allocator;
473 features = gst_caps_get_features (state->caps, 0);
476 gst_caps_features_contains (features,
477 GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) {
478 GST_DEBUG_OBJECT (self, "found D3D11 memory feature");
480 hr = MFCreateVideoSampleAllocatorEx (IID_PPV_ARGS (&allocator));
481 if (!gst_mf_result (hr))
482 GST_WARNING_OBJECT (self,
483 "IMFVideoSampleAllocatorEx interface is unavailable");
488 ComPtr < IMFAttributes > attr;
490 hr = MFCreateAttributes (&attr, 4);
491 if (!gst_mf_result (hr))
494 /* Only one buffer per sample
495 * (multiple sample is usually for multi-view things) */
496 hr = attr->SetUINT32 (GST_GUID_MF_SA_BUFFERS_PER_SAMPLE, 1);
497 if (!gst_mf_result (hr))
500 hr = attr->SetUINT32 (GST_GUID_MF_SA_D3D11_USAGE, D3D11_USAGE_DEFAULT);
501 if (!gst_mf_result (hr))
504 /* TODO: Check if we need to use keyed-mutex */
505 hr = attr->SetUINT32 (GST_GUID_MF_SA_D3D11_SHARED_WITHOUT_MUTEX, TRUE);
506 if (!gst_mf_result (hr))
509 hr = attr->SetUINT32 (GST_GUID_MF_SA_D3D11_BINDFLAGS,
510 D3D11_BIND_VIDEO_ENCODER);
511 if (!gst_mf_result (hr))
514 hr = allocator->SetDirectXManager (self->device_manager);
515 if (!gst_mf_result (hr))
518 hr = allocator->InitializeSampleAllocatorEx (
519 /* min samples, since we are running on async mode,
520 * at least 2 samples would be required */
522 /* max samples, why 16 + 2? it's just magic number
523 * (H264 max dpb size 16 + our min sample size 2) */
524 16 + 2, attr.Get (), in_type.Get ()
527 if (!gst_mf_result (hr))
530 GST_DEBUG_OBJECT (self, "IMFVideoSampleAllocatorEx is initialized");
532 self->mf_allocator = allocator.Detach ();
542 gst_mf_video_buffer_free (GstVideoFrame * frame)
547 gst_video_frame_unmap (frame);
552 gst_mf_video_enc_frame_needs_copy (GstVideoFrame * vframe)
554 /* Single plane data can be used without copy */
555 if (GST_VIDEO_FRAME_N_PLANES (vframe) == 1)
558 switch (GST_VIDEO_FRAME_FORMAT (vframe)) {
559 case GST_VIDEO_FORMAT_I420:
561 guint8 *data, *other_data;
564 /* Unexpected stride size, Media Foundation doesn't provide API for
565 * per plane stride information */
566 if (GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 0) !=
567 2 * GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 1) ||
568 GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 1) !=
569 GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 2)) {
573 size = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 0) *
574 GST_VIDEO_FRAME_HEIGHT (vframe);
575 if (size + GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 0) !=
576 GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 1))
579 data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 0);
580 other_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 1);
581 if (data + size != other_data)
584 size = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 1) *
585 GST_VIDEO_FRAME_COMP_HEIGHT (vframe, 1);
586 if (size + GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 1) !=
587 GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 2))
590 data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 1);
591 other_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 2);
592 if (data + size != other_data)
597 case GST_VIDEO_FORMAT_NV12:
598 case GST_VIDEO_FORMAT_P010_10LE:
599 case GST_VIDEO_FORMAT_P016_LE:
601 guint8 *data, *other_data;
604 /* Unexpected stride size, Media Foundation doesn't provide API for
605 * per plane stride information */
606 if (GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 0) !=
607 GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 1)) {
611 size = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, 0) *
612 GST_VIDEO_FRAME_HEIGHT (vframe);
614 /* Unexpected padding */
615 if (size + GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 0) !=
616 GST_VIDEO_FRAME_PLANE_OFFSET (vframe, 1))
619 data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 0);
620 other_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 1);
621 if (data + size != other_data)
627 g_assert_not_reached ();
637 } GstMFVideoEncFrameData;
640 gst_mf_video_enc_process_input (GstMFVideoEnc * self,
641 GstVideoCodecFrame * frame, IMFSample * sample)
643 GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (self);
645 gboolean unset_force_keyframe = FALSE;
646 GstMFVideoEncFrameData *frame_data = NULL;
649 frame_data = g_new0 (GstMFVideoEncFrameData, 1);
650 frame_data->mf_pts = frame->pts / 100;
652 gst_video_codec_frame_set_user_data (frame,
653 frame_data, (GDestroyNotify) g_free);
655 hr = sample->SetSampleTime (frame_data->mf_pts);
656 if (!gst_mf_result (hr))
660 SetSampleDuration (GST_CLOCK_TIME_IS_VALID (frame->duration) ? frame->
662 if (!gst_mf_result (hr))
665 if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME (frame)) {
666 if (klass->device_caps.force_keyframe) {
667 unset_force_keyframe =
668 gst_mf_transform_set_codec_api_uint32 (self->transform,
669 &CODECAPI_AVEncVideoForceKeyFrame, TRUE);
671 GST_WARNING_OBJECT (self, "encoder does not support force keyframe");
675 /* Unlock temporary so that we can output frame from Media Foundation's
677 * While we are processing input, MFT might notify
678 * METransformHaveOutput event from Media Foundation's internal worker queue
679 * thread. Then we will output encoded data from the thread synchroniously,
680 * not from streaming (this) thread */
682 GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
683 res = gst_mf_transform_process_input (self->transform, sample);
685 GST_VIDEO_ENCODER_STREAM_LOCK (self);
687 if (unset_force_keyframe) {
688 gst_mf_transform_set_codec_api_uint32 (self->transform,
689 &CODECAPI_AVEncVideoForceKeyFrame, FALSE);
693 GST_ERROR_OBJECT (self, "Failed to process input");
700 static GstVideoCodecFrame *
701 gst_mf_video_enc_find_output_frame (GstMFVideoEnc * self, LONGLONG mf_pts)
703 GList *l, *walk = gst_video_encoder_get_frames (GST_VIDEO_ENCODER (self));
704 GstVideoCodecFrame *ret = NULL;
705 GstVideoCodecFrame *closest = NULL;
706 LONGLONG min_pts_abs_diff = 0;
708 for (l = walk; l; l = l->next) {
709 GstVideoCodecFrame *frame = (GstVideoCodecFrame *) l->data;
710 GstMFVideoEncFrameData *data = (GstMFVideoEncFrameData *)
711 gst_video_codec_frame_get_user_data (frame);
717 if (mf_pts == data->mf_pts) {
722 abs_diff = std::abs (mf_pts - data->mf_pts);
724 if (!closest || abs_diff < min_pts_abs_diff) {
726 min_pts_abs_diff = abs_diff;
734 gst_video_codec_frame_ref (ret);
736 /* XXX: Shouldn't happen, but possible if no GstVideoCodecFrame holds
737 * user data for some reasons */
738 GST_WARNING_OBJECT (self,
739 "Failed to find closest GstVideoCodecFrame with MF pts %"
740 G_GINT64_FORMAT, mf_pts);
741 ret = gst_video_encoder_get_oldest_frame (GST_VIDEO_ENCODER (self));
745 g_list_free_full (walk, (GDestroyNotify) gst_video_codec_frame_unref);
751 gst_mf_video_enc_finish_sample (GstMFVideoEnc * self, IMFSample * sample)
755 ComPtr < IMFMediaBuffer > media_buffer;
757 GstFlowReturn res = GST_FLOW_ERROR;
758 GstVideoCodecFrame *frame;
759 LONGLONG sample_timestamp;
760 LONGLONG sample_duration;
761 LONGLONG target_mf_pts;
763 UINT32 keyframe = FALSE;
765 GstClockTime pts, dts, duration;
767 hr = sample->GetBufferByIndex (0, media_buffer.GetAddressOf ());
768 if (!gst_mf_result (hr))
771 hr = media_buffer->Lock (&data, NULL, &buffer_len);
772 if (!gst_mf_result (hr))
775 buffer = gst_buffer_new_allocate (NULL, buffer_len, NULL);
776 gst_buffer_fill (buffer, 0, data, buffer_len);
777 media_buffer->Unlock ();
779 sample->GetSampleTime (&sample_timestamp);
780 target_mf_pts = sample_timestamp;
781 sample->GetSampleDuration (&sample_duration);
782 sample->GetUINT32 (MFSampleExtension_CleanPoint, &keyframe);
784 hr = sample->GetUINT64 (MFSampleExtension_DecodeTimestamp, &mf_dts);
786 mf_dts = sample_timestamp;
790 pts = sample_timestamp * 100;
792 duration = sample_duration * 100;
794 GST_LOG_OBJECT (self, "Finish sample, MF pts %" GST_TIME_FORMAT " MF dts %"
795 GST_TIME_FORMAT ", MF duration %" GST_TIME_FORMAT,
796 GST_TIME_ARGS (pts), GST_TIME_ARGS (dts), GST_TIME_ARGS (duration));
798 /* NOTE: When B-frame is enabled, MFT shows following pattern
799 * (input timestamp starts from 1000:00:00.000000000, and 30fps)
801 * Frame-1: MF pts 0:00.033333300 MF dts 0:00.000000000
802 * Frame-2: MF pts 0:00.133333300 MF dts 0:00.033333300
803 * Frame-3: MF pts 0:00.066666600 MF dts 0:00.066666600
804 * Frame-4: MF pts 0:00.099999900 MF dts 0:00.100000000
806 * - Sounds MFT doesn't support negative timestamp, so PTS of each frame seems
808 * - DTS is likely based on timestamp we've set to input sample,
809 * but some frames has (especially Frame-4 case) unexpected PTS and
810 * even PTS < DTS. That would be the result of PTS shifting
812 * To handle this case,
813 * - Calculate timestamp offset "Frame-1 PTS" - "Frame-1 DTS" (== duration),
814 * and compensate PTS/DTS of each frame
815 * - Needs additional offset for DTS to compenstate GST/MF timescale difference
816 * (MF uses 100ns timescale). So DTS offset should be "PTS offset + 100ns"
817 * - Find corresponding GstVideoCodecFrame by using compensated PTS.
818 * Note that MFT doesn't support user-data for tracing input/output sample
819 * pair. So, timestamp based lookup is the only way to map MF sample
820 * and our GstVideoCodecFrame
822 if (self->has_reorder_frame) {
823 /* This would be the first frame */
824 if (self->mf_pts_offset == 0) {
825 LONGLONG mf_pts_offset = -1;
826 if (sample_timestamp > mf_dts) {
827 mf_pts_offset = sample_timestamp - mf_dts;
828 GST_DEBUG_OBJECT (self, "Calculates PTS offset using \"PTS - DTS\": %"
829 G_GINT64_FORMAT, mf_pts_offset);
830 } else if (sample_duration > 0) {
831 mf_pts_offset = sample_duration;
832 GST_DEBUG_OBJECT (self, "Calculates PTS offset using duration: %"
833 G_GINT64_FORMAT, mf_pts_offset);
835 GST_WARNING_OBJECT (self, "Cannot calculate PTS offset");
838 self->mf_pts_offset = mf_pts_offset;
841 if (self->mf_pts_offset > 0) {
842 target_mf_pts -= self->mf_pts_offset;
844 pts -= (self->mf_pts_offset * 100);
845 /* +1 to compensate timescale difference */
846 dts -= ((self->mf_pts_offset + 1) * 100);
850 frame = gst_mf_video_enc_find_output_frame (self, target_mf_pts);
854 GST_DEBUG_OBJECT (self, "Keyframe pts %" GST_TIME_FORMAT,
855 GST_TIME_ARGS (frame->pts));
856 GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame);
859 frame->output_buffer = buffer;
861 /* Update DTS only if B-frame was enabled, but use input frame pts as-is.
862 * Otherwise we will lost at most 100ns precision */
863 if (self->has_reorder_frame) {
866 frame->dts = frame->pts;
869 /* make sure PTS > DTS */
870 if (GST_CLOCK_TIME_IS_VALID (frame->pts) &&
871 GST_CLOCK_TIME_IS_VALID (frame->dts) && frame->pts < frame->dts) {
872 GST_WARNING_OBJECT (self, "Calculated DTS %" GST_TIME_FORMAT
873 " is larger than PTS %" GST_TIME_FORMAT, GST_TIME_ARGS (frame->pts),
874 GST_TIME_ARGS (frame->dts));
876 /* XXX: just set clock-time-none? */
877 frame->dts = frame->pts;
880 GST_LOG_OBJECT (self, "Frame pts %" GST_TIME_FORMAT ", Frame DTS %"
881 GST_TIME_FORMAT, GST_TIME_ARGS (frame->pts),
882 GST_TIME_ARGS (frame->dts));
884 res = gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (self), frame);
886 GST_BUFFER_PTS (buffer) = pts;
887 GST_BUFFER_DTS (buffer) = dts;
888 GST_BUFFER_DURATION (buffer) = duration;
891 GST_DEBUG_OBJECT (self, "Keyframe pts %" GST_TIME_FORMAT,
892 GST_BUFFER_PTS (buffer));
893 GST_BUFFER_FLAG_UNSET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
895 GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
898 GST_LOG_OBJECT (self, "Buffer pts %" GST_TIME_FORMAT ", Buffer DTS %"
899 GST_TIME_FORMAT, GST_TIME_ARGS (pts), GST_TIME_ARGS (dts));
901 res = gst_pad_push (GST_VIDEO_ENCODER_SRC_PAD (self), buffer);
905 self->last_ret = res;
911 gst_mf_video_enc_process_output (GstMFVideoEnc * self)
913 ComPtr < IMFSample > sample;
914 GstFlowReturn res = GST_FLOW_ERROR;
916 res = gst_mf_transform_get_output (self->transform, &sample);
918 if (res != GST_FLOW_OK)
921 gst_mf_video_enc_finish_sample (self, sample.Get ());
923 return self->last_ret;
927 gst_mf_video_enc_create_input_sample (GstMFVideoEnc * self,
928 GstVideoCodecFrame * frame, IMFSample ** sample)
931 ComPtr < IMFSample > new_sample;
932 ComPtr < IMFMediaBuffer > media_buffer;
933 ComPtr < IGstMFVideoBuffer > video_buffer;
934 GstVideoInfo *info = &self->input_state->info;
936 GstVideoFrame *vframe = NULL;
940 vframe = g_new0 (GstVideoFrame, 1);
942 if (!gst_video_frame_map (vframe, info, frame->input_buffer, GST_MAP_READ)) {
943 GST_ERROR_OBJECT (self, "Couldn't map input frame");
948 hr = MFCreateSample (&new_sample);
949 if (!gst_mf_result (hr))
952 /* Check if we can forward this memory to Media Foundation without copy */
953 need_copy = gst_mf_video_enc_frame_needs_copy (vframe);
955 GST_TRACE_OBJECT (self, "Copy input buffer into Media Foundation memory");
956 hr = MFCreateMemoryBuffer (GST_VIDEO_INFO_SIZE (info), &media_buffer);
958 GST_TRACE_OBJECT (self, "Can use input buffer without copy");
959 hr = IGstMFVideoBuffer::CreateInstanceWrapped (&vframe->info,
960 (BYTE *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 0),
961 GST_VIDEO_INFO_SIZE (&vframe->info), &media_buffer);
964 if (!gst_mf_result (hr))
968 hr = media_buffer.As (&video_buffer);
969 if (!gst_mf_result (hr))
972 hr = media_buffer->Lock (&data, NULL, NULL);
973 if (!gst_mf_result (hr))
976 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) {
978 gint src_stride, dst_stride;
981 src = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, i);
982 dst = data + GST_VIDEO_INFO_PLANE_OFFSET (info, i);
984 src_stride = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, i);
985 dst_stride = GST_VIDEO_INFO_PLANE_STRIDE (info, i);
987 width = GST_VIDEO_INFO_COMP_WIDTH (info, i)
988 * GST_VIDEO_INFO_COMP_PSTRIDE (info, i);
990 for (j = 0; j < GST_VIDEO_INFO_COMP_HEIGHT (info, i); j++) {
991 memcpy (dst, src, width);
997 media_buffer->Unlock ();
1000 hr = media_buffer->SetCurrentLength (GST_VIDEO_INFO_SIZE (info));
1001 if (!gst_mf_result (hr))
1004 hr = new_sample->AddBuffer (media_buffer.Get ());
1005 if (!gst_mf_result (hr))
1009 /* IGstMFVideoBuffer will hold GstVideoFrame (+ GstBuffer), then it will be
1010 * cleared when it's no more referenced by Media Foundation internals */
1011 hr = video_buffer->SetUserData ((gpointer) vframe,
1012 (GDestroyNotify) gst_mf_video_buffer_free);
1013 if (!gst_mf_result (hr))
1016 gst_video_frame_unmap (vframe);
1021 *sample = new_sample.Detach ();
1027 gst_video_frame_unmap (vframe);
1034 #if GST_MF_HAVE_D3D11
1036 gst_mf_video_enc_create_input_sample_d3d11 (GstMFVideoEnc * self,
1037 GstVideoCodecFrame * frame, IMFSample ** sample)
1040 ComPtr < IMFSample > new_sample;
1041 ComPtr < IMFMediaBuffer > mf_buffer;
1042 ComPtr < IMFDXGIBuffer > dxgi_buffer;
1043 ComPtr < ID3D11Texture2D > mf_texture;
1044 ComPtr < IDXGIResource > dxgi_resource;
1045 ComPtr < ID3D11Texture2D > shared_texture;
1046 ComPtr < ID3D11Query > query;
1047 D3D11_QUERY_DESC query_desc;
1048 BOOL sync_done = FALSE;
1049 HANDLE shared_handle;
1051 GstD3D11Memory *dmem;
1052 ID3D11Texture2D *texture;
1053 ID3D11Device *device_handle;
1054 ID3D11DeviceContext *context_handle;
1056 D3D11_BOX src_box = { 0, };
1057 D3D11_TEXTURE2D_DESC dst_desc, src_desc;
1060 if (!self->mf_allocator) {
1061 GST_WARNING_OBJECT (self, "IMFVideoSampleAllocatorEx was configured");
1065 mem = gst_buffer_peek_memory (frame->input_buffer, 0);
1066 if (!gst_is_d3d11_memory (mem)) {
1067 GST_WARNING_OBJECT (self, "Non-d3d11 memory");
1071 dmem = (GstD3D11Memory *) mem;
1072 device_handle = gst_d3d11_device_get_device_handle (dmem->device);
1073 context_handle = gst_d3d11_device_get_device_context_handle (dmem->device);
1075 /* 1) Allocate new encoding surface */
1076 hr = self->mf_allocator->AllocateSample (&new_sample);
1077 if (!gst_mf_result (hr)) {
1078 GST_WARNING_OBJECT (self,
1079 "Couldn't allocate new sample via IMFVideoSampleAllocatorEx");
1083 hr = new_sample->GetBufferByIndex (0, &mf_buffer);
1084 if (!gst_mf_result (hr)) {
1085 GST_WARNING_OBJECT (self, "Couldn't get IMFMediaBuffer from sample");
1089 hr = mf_buffer.As (&dxgi_buffer);
1090 if (!gst_mf_result (hr)) {
1091 GST_WARNING_OBJECT (self, "Couldn't get IMFDXGIBuffer from IMFMediaBuffer");
1095 hr = dxgi_buffer->GetResource (IID_PPV_ARGS (&mf_texture));
1096 if (!gst_mf_result (hr)) {
1097 GST_WARNING_OBJECT (self,
1098 "Couldn't get ID3D11Texture2D from IMFDXGIBuffer");
1102 hr = mf_texture.As (&dxgi_resource);
1103 if (!gst_mf_result (hr)) {
1104 GST_WARNING_OBJECT (self,
1105 "Couldn't get IDXGIResource from ID3D11Texture2D");
1109 hr = dxgi_resource->GetSharedHandle (&shared_handle);
1110 if (!gst_mf_result (hr)) {
1111 GST_WARNING_OBJECT (self, "Couldn't get shared handle from IDXGIResource");
1115 /* Allocation succeeded. Now open shared texture to access it from
1117 hr = device_handle->OpenSharedResource (shared_handle,
1118 IID_PPV_ARGS (&shared_texture));
1119 if (!gst_mf_result (hr)) {
1120 GST_WARNING_OBJECT (self, "Couldn't open shared resource");
1124 /* 2) Copy upstream texture to mf's texture */
1125 /* Map memory so that ensure pending upload from staging texture */
1126 if (!gst_memory_map (mem, &info,
1127 (GstMapFlags) (GST_MAP_READ | GST_MAP_D3D11))) {
1128 GST_ERROR_OBJECT (self, "Couldn't map d3d11 memory");
1132 texture = (ID3D11Texture2D *) info.data;
1133 texture->GetDesc (&src_desc);
1134 shared_texture->GetDesc (&dst_desc);
1135 subidx = gst_d3d11_memory_get_subresource_index (dmem);
1137 /* src/dst texture size might be different if padding was used.
1138 * select smaller size */
1143 src_box.right = MIN (src_desc.Width, dst_desc.Width);
1144 src_box.bottom = MIN (src_desc.Height, dst_desc.Height);
1146 /* CopySubresourceRegion() might not be able to guarantee
1147 * copying. To ensure it, we will make use of d3d11 query */
1148 query_desc.Query = D3D11_QUERY_EVENT;
1149 query_desc.MiscFlags = 0;
1151 hr = device_handle->CreateQuery (&query_desc, &query);
1152 if (!gst_d3d11_result (hr, dmem->device)) {
1153 GST_ERROR_OBJECT (self, "Couldn't Create event query");
1157 gst_d3d11_device_lock (dmem->device);
1158 context_handle->CopySubresourceRegion (shared_texture.Get (), 0, 0, 0, 0,
1159 texture, subidx, &src_box);
1160 context_handle->End (query.Get ());
1162 /* Wait until all issued GPU commands are finished */
1164 context_handle->GetData (query.Get (), &sync_done, sizeof (BOOL), 0);
1165 } while (!sync_done && (hr == S_OK || hr == S_FALSE));
1167 if (!gst_d3d11_result (hr, dmem->device)) {
1168 GST_ERROR_OBJECT (self, "Couldn't sync GPU operation");
1169 gst_d3d11_device_unlock (dmem->device);
1170 gst_memory_unmap (mem, &info);
1175 gst_d3d11_device_unlock (dmem->device);
1176 gst_memory_unmap (mem, &info);
1178 *sample = new_sample.Detach ();
1184 static GstFlowReturn
1185 gst_mf_video_enc_handle_frame (GstVideoEncoder * enc,
1186 GstVideoCodecFrame * frame)
1188 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1189 GstFlowReturn ret = GST_FLOW_OK;
1190 ComPtr < IMFSample > sample;
1192 if (self->last_ret != GST_FLOW_OK) {
1193 GST_DEBUG_OBJECT (self, "Last return was %s", gst_flow_get_name (ret));
1194 ret = self->last_ret;
1197 #if GST_MF_HAVE_D3D11
1198 if (self->mf_allocator &&
1199 !gst_mf_video_enc_create_input_sample_d3d11 (self, frame, &sample)) {
1200 GST_WARNING_OBJECT (self, "Failed to create IMFSample for D3D11");
1205 if (!sample && !gst_mf_video_enc_create_input_sample (self, frame, &sample)) {
1206 GST_ERROR_OBJECT (self, "Failed to create IMFSample");
1207 ret = GST_FLOW_ERROR;
1211 if (!gst_mf_video_enc_process_input (self, frame, sample.Get ())) {
1212 GST_ERROR_OBJECT (self, "Failed to process input");
1213 ret = GST_FLOW_ERROR;
1217 /* Don't call process_output for async (hardware) MFT. We will output
1218 * encoded data from gst_mf_video_on_new_sample() callback which is called
1219 * from Media Foundation's internal worker queue thread */
1220 if (!self->async_mft) {
1222 ret = gst_mf_video_enc_process_output (self);
1223 } while (ret == GST_FLOW_OK);
1226 if (ret == GST_MF_TRANSFORM_FLOW_NEED_DATA)
1230 gst_video_codec_frame_unref (frame);
1235 static GstFlowReturn
1236 gst_mf_video_enc_finish (GstVideoEncoder * enc)
1238 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1239 GstFlowReturn ret = GST_FLOW_OK;
1241 if (!self->transform)
1244 /* Unlock temporary so that we can output frame from Media Foundation's
1246 if (self->async_mft)
1247 GST_VIDEO_ENCODER_STREAM_UNLOCK (enc);
1249 gst_mf_transform_drain (self->transform);
1251 if (self->async_mft)
1252 GST_VIDEO_ENCODER_STREAM_LOCK (enc);
1254 if (!self->async_mft) {
1256 ret = gst_mf_video_enc_process_output (self);
1257 } while (ret == GST_FLOW_OK);
1260 if (ret == GST_MF_TRANSFORM_FLOW_NEED_DATA)
1267 gst_mf_video_enc_flush (GstVideoEncoder * enc)
1269 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1271 if (!self->transform)
1274 /* Unlock while flushing, while flushing, new sample callback might happen */
1275 if (self->async_mft)
1276 GST_VIDEO_ENCODER_STREAM_UNLOCK (enc);
1278 gst_mf_transform_flush (self->transform);
1280 if (self->async_mft)
1281 GST_VIDEO_ENCODER_STREAM_LOCK (enc);
1284 self->last_ret = GST_FLOW_OK;
1290 gst_mf_video_enc_propose_allocation (GstVideoEncoder * enc, GstQuery * query)
1292 #if GST_MF_HAVE_D3D11
1293 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1295 GstBufferPool *pool = NULL;
1298 GstD3D11Device *device = self->other_d3d11_device;
1300 gst_query_parse_allocation (query, &caps, NULL);
1305 if (!gst_video_info_from_caps (&info, caps))
1308 if (gst_query_get_n_allocation_pools (query) == 0) {
1309 GstCapsFeatures *features;
1310 GstStructure *config;
1311 gboolean is_d3d11 = FALSE;
1313 features = gst_caps_get_features (caps, 0);
1315 if (features && gst_caps_features_contains (features,
1316 GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) {
1317 GST_DEBUG_OBJECT (self, "Allocation caps supports d3d11 memory");
1318 pool = gst_d3d11_buffer_pool_new (device);
1321 pool = gst_video_buffer_pool_new ();
1324 config = gst_buffer_pool_get_config (pool);
1326 gst_buffer_pool_config_add_option (config,
1327 GST_BUFFER_POOL_OPTION_VIDEO_META);
1329 /* d3d11 pool does not support video alignment */
1331 gst_buffer_pool_config_add_option (config,
1332 GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
1334 GstD3D11AllocationParams *d3d11_params;
1335 guint misc_flags = 0;
1336 gboolean is_hardware = FALSE;
1339 g_object_get (device, "hardware", &is_hardware, NULL);
1341 /* In case of hardware, set D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag
1342 * so that it can be shared with other d3d11 devices */
1344 misc_flags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
1347 gst_buffer_pool_config_get_d3d11_allocation_params (config);
1348 if (!d3d11_params) {
1349 d3d11_params = gst_d3d11_allocation_params_new (device, &info,
1350 (GstD3D11AllocationFlags) 0, 0);
1352 for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++)
1353 d3d11_params->desc[i].MiscFlags |= misc_flags;
1356 gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
1357 gst_d3d11_allocation_params_free (d3d11_params);
1360 size = GST_VIDEO_INFO_SIZE (&info);
1361 gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
1363 if (!gst_buffer_pool_set_config (pool, config))
1366 /* d3d11 buffer pool will update buffer size based on allocated texture,
1367 * get size from config again */
1369 config = gst_buffer_pool_get_config (pool);
1370 gst_buffer_pool_config_get_params (config,
1371 nullptr, &size, nullptr, nullptr);
1372 gst_structure_free (config);
1375 gst_query_add_allocation_pool (query, pool, size, 0, 0);
1376 gst_object_unref (pool);
1379 gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
1386 GST_ERROR_OBJECT (self, "failed to set config");
1387 gst_object_unref (pool);
1392 return GST_VIDEO_ENCODER_CLASS (parent_class)->propose_allocation (enc,
1398 gst_mf_video_enc_sink_query (GstVideoEncoder * enc, GstQuery * query)
1400 #if GST_MF_HAVE_D3D11
1401 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1403 switch (GST_QUERY_TYPE (query)) {
1404 case GST_QUERY_CONTEXT:
1405 if (gst_d3d11_handle_context_query (GST_ELEMENT (self),
1406 query, self->other_d3d11_device)) {
1415 return GST_VIDEO_ENCODER_CLASS (parent_class)->sink_query (enc, query);
1419 gst_mf_video_enc_src_query (GstVideoEncoder * enc, GstQuery * query)
1421 #if GST_MF_HAVE_D3D11
1422 GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc);
1424 switch (GST_QUERY_TYPE (query)) {
1425 case GST_QUERY_CONTEXT:
1426 if (gst_d3d11_handle_context_query (GST_ELEMENT (self),
1427 query, self->other_d3d11_device)) {
1436 return GST_VIDEO_ENCODER_CLASS (parent_class)->src_query (enc, query);
1440 gst_mf_video_on_new_sample (GstMFTransform * object,
1441 IMFSample * sample, GstMFVideoEnc * self)
1443 GST_LOG_OBJECT (self, "New Sample callback");
1445 /* NOTE: this callback will be called from Media Foundation's internal
1446 * worker queue thread */
1447 GST_VIDEO_ENCODER_STREAM_LOCK (self);
1448 gst_mf_video_enc_finish_sample (self, sample);
1449 GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
1457 const gchar *profile_str;
1458 } GstMFVideoEncProfileMap;
1461 gst_mf_video_enc_enum_internal (GstMFTransform * transform, GUID & subtype,
1462 GstObject * d3d11_device, GstMFVideoEncDeviceCaps * device_caps,
1463 GstCaps ** sink_template, GstCaps ** src_template)
1466 MFT_REGISTER_TYPE_INFO *infos;
1469 GstCaps *src_caps = NULL;
1470 GstCaps *sink_caps = NULL;
1471 GstCaps *d3d11_caps = NULL;
1472 GValue *supported_formats = NULL;
1473 GValue *profiles = NULL;
1474 gboolean have_I420 = FALSE;
1475 gboolean have_NV12 = FALSE;
1476 gboolean have_P010 = FALSE;
1477 gboolean d3d11_aware = FALSE;
1478 gchar *device_name = NULL;
1479 IMFActivate *activate;
1480 IMFTransform *encoder;
1481 ICodecAPI *codec_api;
1482 ComPtr < IMFMediaType > out_type;
1483 GstMFVideoEncProfileMap h264_profile_map[] = {
1484 {eAVEncH264VProfile_High, "high"},
1485 {eAVEncH264VProfile_Main, "main"},
1486 {eAVEncH264VProfile_Base, "baseline"},
1489 GstMFVideoEncProfileMap hevc_profile_map[] = {
1490 {eAVEncH265VProfile_Main_420_8, "main"},
1491 {eAVEncH265VProfile_Main_420_10, "main-10"},
1494 GstMFVideoEncProfileMap *profile_to_check = NULL;
1495 static gchar *h264_caps_str =
1496 "video/x-h264, stream-format=(string) byte-stream, alignment=(string) au";
1497 static gchar *hevc_caps_str =
1498 "video/x-h265, stream-format=(string) byte-stream, alignment=(string) au";
1499 static gchar *vp9_caps_str = "video/x-vp9";
1500 static gchar *codec_caps_str = NULL;
1502 /* NOTE: depending on environment,
1503 * some enumerated h/w MFT might not be usable (e.g., multiple GPU case) */
1504 if (!gst_mf_transform_open (transform))
1507 activate = gst_mf_transform_get_activate_handle (transform);
1509 GST_WARNING_OBJECT (transform, "No IMFActivate interface available");
1513 encoder = gst_mf_transform_get_transform_handle (transform);
1515 GST_WARNING_OBJECT (transform, "No IMFTransform interface available");
1519 codec_api = gst_mf_transform_get_codec_api_handle (transform);
1521 GST_WARNING_OBJECT (transform, "No ICodecAPI interface available");
1525 g_object_get (transform, "device-name", &device_name, NULL);
1527 GST_WARNING_OBJECT (transform, "Unknown device name");
1530 g_free (device_name);
1532 hr = activate->GetAllocatedBlob (MFT_INPUT_TYPES_Attributes,
1533 (UINT8 **) & infos, &info_size);
1534 if (!gst_mf_result (hr))
1537 for (i = 0; i < info_size / sizeof (MFT_REGISTER_TYPE_INFO); i++) {
1538 GstVideoFormat format;
1539 const GstVideoFormatInfo *format_info;
1540 GValue val = G_VALUE_INIT;
1542 format = gst_mf_video_subtype_to_video_format (&infos[i].guidSubtype);
1543 if (format == GST_VIDEO_FORMAT_UNKNOWN)
1546 format_info = gst_video_format_get_info (format);
1547 if (GST_VIDEO_FORMAT_INFO_IS_RGB (format_info)) {
1548 GST_DEBUG_OBJECT (transform, "Skip %s format",
1549 GST_VIDEO_FORMAT_INFO_NAME (format_info));
1553 if (!supported_formats) {
1554 supported_formats = g_new0 (GValue, 1);
1555 g_value_init (supported_formats, GST_TYPE_LIST);
1559 /* media foundation has duplicated formats IYUV and I420 */
1560 case GST_VIDEO_FORMAT_I420:
1566 case GST_VIDEO_FORMAT_NV12:
1569 case GST_VIDEO_FORMAT_P010_10LE:
1576 g_value_init (&val, G_TYPE_STRING);
1577 g_value_set_static_string (&val, gst_video_format_to_string (format));
1578 gst_value_list_append_and_take_value (supported_formats, &val);
1580 CoTaskMemFree (infos);
1582 if (!supported_formats) {
1583 GST_WARNING_OBJECT (transform, "Couldn't figure out supported format");
1587 if (IsEqualGUID (MFVideoFormat_H264, subtype)) {
1588 profile_to_check = h264_profile_map;
1589 codec_caps_str = h264_caps_str;
1590 } else if (IsEqualGUID (MFVideoFormat_HEVC, subtype)) {
1591 profile_to_check = hevc_profile_map;
1592 codec_caps_str = hevc_caps_str;
1593 } else if (IsEqualGUID (MFVideoFormat_VP90, subtype)) {
1594 codec_caps_str = vp9_caps_str;
1596 g_assert_not_reached ();
1600 if (profile_to_check) {
1601 hr = MFCreateMediaType (&out_type);
1602 if (!gst_mf_result (hr))
1605 hr = out_type->SetGUID (MF_MT_MAJOR_TYPE, MFMediaType_Video);
1606 if (!gst_mf_result (hr))
1609 hr = out_type->SetGUID (MF_MT_SUBTYPE, subtype);
1610 if (!gst_mf_result (hr))
1613 hr = out_type->SetUINT32 (MF_MT_AVG_BITRATE, 2048000);
1614 if (!gst_mf_result (hr))
1617 hr = MFSetAttributeRatio (out_type.Get (), MF_MT_FRAME_RATE, 30, 1);
1618 if (!gst_mf_result (hr))
1621 hr = out_type->SetUINT32 (MF_MT_INTERLACE_MODE,
1622 MFVideoInterlace_Progressive);
1623 if (!gst_mf_result (hr))
1626 hr = MFSetAttributeSize (out_type.Get (), MF_MT_FRAME_SIZE, 1920, 1080);
1627 if (!gst_mf_result (hr))
1632 GValue profile_val = G_VALUE_INIT;
1633 guint mf_profile = profile_to_check[i].profile;
1634 const gchar *profile_str = profile_to_check[i].profile_str;
1638 if (mf_profile == 0)
1641 g_assert (profile_str != NULL);
1643 hr = out_type->SetUINT32 (MF_MT_MPEG2_PROFILE, mf_profile);
1644 if (!gst_mf_result (hr))
1647 if (!gst_mf_transform_set_output_type (transform, out_type.Get ()))
1651 profiles = g_new0 (GValue, 1);
1652 g_value_init (profiles, GST_TYPE_LIST);
1655 /* Add "constrained-baseline" in addition to "baseline" */
1656 if (profile_str == "baseline") {
1657 g_value_init (&profile_val, G_TYPE_STRING);
1658 g_value_set_static_string (&profile_val, "constrained-baseline");
1659 gst_value_list_append_and_take_value (profiles, &profile_val);
1662 g_value_init (&profile_val, G_TYPE_STRING);
1663 g_value_set_static_string (&profile_val, profile_str);
1664 gst_value_list_append_and_take_value (profiles, &profile_val);
1668 GST_WARNING_OBJECT (transform, "Couldn't query supported profile");
1673 src_caps = gst_caps_from_string (codec_caps_str);
1675 gst_caps_set_value (src_caps, "profile", profiles);
1676 g_value_unset (profiles);
1680 sink_caps = gst_caps_new_empty_simple ("video/x-raw");
1681 /* FIXME: don't hardcode max resolution, but MF doesn't provide
1682 * API for querying supported max resolution... */
1683 gst_caps_set_simple (sink_caps,
1684 "width", GST_TYPE_INT_RANGE, 64, 8192,
1685 "height", GST_TYPE_INT_RANGE, 64, 8192, NULL);
1686 gst_caps_set_simple (src_caps,
1687 "width", GST_TYPE_INT_RANGE, 64, 8192,
1688 "height", GST_TYPE_INT_RANGE, 64, 8192, NULL);
1690 #if GST_MF_HAVE_D3D11
1691 /* Check whether this MFT can support D3D11 */
1692 if (d3d11_device && (have_NV12 || have_P010)) {
1693 g_object_get (transform, "d3d11-aware", &d3d11_aware, NULL);
1694 GST_DEBUG_OBJECT (transform, "d3d11 aware %d", d3d11_aware);
1697 if (d3d11_device && (have_NV12 || have_P010) && d3d11_aware) {
1698 gint64 adapter_luid = 0;
1699 GValue d3d11_formats = G_VALUE_INIT;
1701 g_object_get (d3d11_device, "adapter-luid", &adapter_luid, NULL);
1703 d3d11_caps = gst_caps_copy (sink_caps);
1705 g_value_init (&d3d11_formats, GST_TYPE_LIST);
1707 GValue val = G_VALUE_INIT;
1708 g_value_init (&val, G_TYPE_STRING);
1709 g_value_set_static_string (&val, "NV12");
1710 gst_value_list_append_and_take_value (&d3d11_formats, &val);
1714 GValue val = G_VALUE_INIT;
1715 g_value_init (&val, G_TYPE_STRING);
1716 g_value_set_static_string (&val, "P010_10LE");
1717 gst_value_list_append_and_take_value (&d3d11_formats, &val);
1720 gst_caps_set_value (d3d11_caps, "format", &d3d11_formats);
1721 g_value_unset (&d3d11_formats);
1722 gst_caps_set_features_simple (d3d11_caps,
1723 gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY));
1724 device_caps->d3d11_aware = TRUE;
1725 device_caps->adapter_luid = adapter_luid;
1729 gst_caps_set_value (sink_caps, "format", supported_formats);
1730 g_value_unset (supported_formats);
1731 g_free (supported_formats);
1734 gst_caps_append (sink_caps, d3d11_caps);
1736 *sink_template = sink_caps;
1737 *src_template = src_caps;
1739 #define CHECK_DEVICE_CAPS(codec_obj,api,val) \
1740 if (SUCCEEDED((codec_obj)->IsSupported(&(api)))) {\
1741 (device_caps)->val = TRUE; \
1744 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncCommonRateControlMode, rc_mode);
1745 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncCommonQuality, quality);
1746 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncAdaptiveMode, adaptive_mode);
1747 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncCommonBufferSize, buffer_size);
1748 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncCommonMaxBitRate, max_bitrate);
1749 CHECK_DEVICE_CAPS (codec_api,
1750 CODECAPI_AVEncCommonQualityVsSpeed, quality_vs_speed);
1751 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncH264CABACEnable, cabac);
1752 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncH264SPSID, sps_id);
1753 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncH264PPSID, pps_id);
1754 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncMPVDefaultBPictureCount, bframes);
1755 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncMPVGOPSize, gop_size);
1756 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncNumWorkerThreads, threads);
1757 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoContentType, content_type);
1758 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoEncodeQP, qp);
1759 CHECK_DEVICE_CAPS (codec_api,
1760 CODECAPI_AVEncVideoForceKeyFrame, force_keyframe);
1761 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVLowLatencyMode, low_latency);
1762 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoMinQP, min_qp);
1763 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoMaxQP, max_qp);
1764 CHECK_DEVICE_CAPS (codec_api,
1765 CODECAPI_AVEncVideoEncodeFrameTypeQP, frame_type_qp);
1766 CHECK_DEVICE_CAPS (codec_api, CODECAPI_AVEncVideoMaxNumRefFrame, max_num_ref);
1767 if (device_caps->max_num_ref) {
1772 hr = codec_api->GetParameterRange (&CODECAPI_AVEncVideoMaxNumRefFrame,
1774 if (SUCCEEDED (hr)) {
1775 device_caps->max_num_ref_high = max.uiVal;
1776 device_caps->max_num_ref_low = min.uiVal;
1777 VariantClear (&min);
1778 VariantClear (&max);
1779 VariantClear (&step);
1781 device_caps->max_num_ref = FALSE;
1784 #undef CHECK_DEVICE_CAPS
1789 static GstMFTransform *
1790 gst_mf_video_enc_enum (guint enum_flags, GUID * subtype, guint device_index,
1791 GstMFVideoEncDeviceCaps * device_caps, GstObject * d3d11_device,
1792 GstCaps ** sink_template, GstCaps ** src_template)
1794 GstMFTransformEnumParams enum_params = { 0, };
1795 MFT_REGISTER_TYPE_INFO output_type;
1796 GstMFTransform *transform;
1797 gint64 adapter_luid = 0;
1799 *sink_template = NULL;
1800 *src_template = NULL;
1801 memset (device_caps, 0, sizeof (GstMFVideoEncDeviceCaps));
1803 if (!IsEqualGUID (MFVideoFormat_H264, *subtype) &&
1804 !IsEqualGUID (MFVideoFormat_HEVC, *subtype) &&
1805 !IsEqualGUID (MFVideoFormat_VP90, *subtype)) {
1806 GST_ERROR ("Unknown subtype GUID");
1812 g_object_get (d3d11_device, "adapter-luid", &adapter_luid, NULL);
1813 if (!adapter_luid) {
1814 GST_ERROR ("Couldn't get adapter LUID");
1819 output_type.guidMajorType = MFMediaType_Video;
1820 output_type.guidSubtype = *subtype;
1822 enum_params.category = MFT_CATEGORY_VIDEO_ENCODER;
1823 enum_params.output_typeinfo = &output_type;
1824 enum_params.device_index = device_index;
1825 enum_params.enum_flags = enum_flags;
1826 enum_params.adapter_luid = adapter_luid;
1828 transform = gst_mf_transform_new (&enum_params);
1832 gst_mf_video_enc_enum_internal (transform, output_type.guidSubtype,
1833 d3d11_device, device_caps, sink_template, src_template);
1839 gst_mf_video_enc_register_internal (GstPlugin * plugin, guint rank,
1840 GUID * subtype, GTypeInfo * type_info,
1841 const GstMFVideoEncDeviceCaps * device_caps,
1842 guint32 enum_flags, guint device_index, GstMFTransform * transform,
1843 GstCaps * sink_caps, GstCaps * src_caps)
1846 GTypeInfo local_type_info;
1848 gchar *feature_name;
1850 GstMFVideoEncClassData *cdata;
1851 gboolean is_default = TRUE;
1852 gchar *device_name = NULL;
1853 static gchar *type_name_prefix = NULL;
1854 static gchar *feature_name_prefix = NULL;
1856 if (IsEqualGUID (MFVideoFormat_H264, *subtype)) {
1857 type_name_prefix = "H264";
1858 feature_name_prefix = "h264";
1859 } else if (IsEqualGUID (MFVideoFormat_HEVC, *subtype)) {
1860 type_name_prefix = "H265";
1861 feature_name_prefix = "h265";
1862 } else if (IsEqualGUID (MFVideoFormat_VP90, *subtype)) {
1863 type_name_prefix = "VP9";
1864 feature_name_prefix = "vp9";
1866 g_assert_not_reached ();
1870 /* Must be checked already */
1871 g_object_get (transform, "device-name", &device_name, NULL);
1872 g_assert (device_name != NULL);
1874 cdata = g_new0 (GstMFVideoEncClassData, 1);
1875 cdata->sink_caps = gst_caps_copy (sink_caps);
1876 cdata->src_caps = gst_caps_copy (src_caps);
1877 cdata->device_name = device_name;
1878 cdata->device_caps = *device_caps;
1879 cdata->enum_flags = enum_flags;
1880 cdata->device_index = device_index;
1882 local_type_info = *type_info;
1883 local_type_info.class_data = cdata;
1885 GST_MINI_OBJECT_FLAG_SET (cdata->sink_caps,
1886 GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
1887 GST_MINI_OBJECT_FLAG_SET (cdata->src_caps,
1888 GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
1890 type_name = g_strdup_printf ("GstMF%sEnc", type_name_prefix);
1891 feature_name = g_strdup_printf ("mf%senc", feature_name_prefix);
1894 while (g_type_from_name (type_name) != 0) {
1896 g_free (feature_name);
1897 type_name = g_strdup_printf ("GstMF%sDevice%dEnc", type_name_prefix, i);
1898 feature_name = g_strdup_printf ("mf%sdevice%denc", feature_name_prefix, i);
1903 cdata->is_default = is_default;
1906 g_type_register_static (GST_TYPE_MF_VIDEO_ENC, type_name,
1907 &local_type_info, (GTypeFlags) 0);
1909 /* make lower rank than default device */
1910 if (rank > 0 && !is_default)
1913 if (!gst_element_register (plugin, feature_name, rank, type))
1914 GST_WARNING ("Failed to register plugin '%s'", type_name);
1917 g_free (feature_name);
1921 gst_mf_video_enc_register (GstPlugin * plugin, guint rank, GUID * subtype,
1922 GTypeInfo * type_info, GList * d3d11_device)
1924 GstMFTransform *transform = NULL;
1925 GstCaps *sink_template = NULL;
1926 GstCaps *src_template = NULL;
1928 GstMFVideoEncDeviceCaps device_caps;
1931 /* register hardware encoders first */
1932 enum_flags = (MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_ASYNCMFT |
1933 MFT_ENUM_FLAG_SORTANDFILTER | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY);
1937 for (iter = d3d11_device; iter; iter = g_list_next (iter)) {
1938 GstObject *device = (GstObject *) iter->data;
1940 transform = gst_mf_video_enc_enum (enum_flags, subtype, 0, &device_caps,
1941 device, &sink_template, &src_template);
1943 /* No more MFT to enumerate */
1947 /* Failed to open MFT */
1948 if (!sink_template) {
1949 gst_clear_object (&transform);
1953 gst_mf_video_enc_register_internal (plugin, rank, subtype, type_info,
1954 &device_caps, enum_flags, 0, transform, sink_template, src_template);
1955 gst_clear_object (&transform);
1956 gst_clear_caps (&sink_template);
1957 gst_clear_caps (&src_template);
1960 /* AMD seems to be able to support up to 12 GPUs */
1961 for (i = 0; i < 12; i++) {
1962 transform = gst_mf_video_enc_enum (enum_flags, subtype, i, &device_caps,
1963 NULL, &sink_template, &src_template);
1965 /* No more MFT to enumerate */
1969 /* Failed to open MFT */
1970 if (!sink_template) {
1971 gst_clear_object (&transform);
1975 gst_mf_video_enc_register_internal (plugin, rank, subtype, type_info,
1976 &device_caps, enum_flags, i, transform, sink_template, src_template);
1977 gst_clear_object (&transform);
1978 gst_clear_caps (&sink_template);
1979 gst_clear_caps (&src_template);
1983 /* register software encoders */
1984 enum_flags = (MFT_ENUM_FLAG_SYNCMFT |
1985 MFT_ENUM_FLAG_SORTANDFILTER | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY);
1987 transform = gst_mf_video_enc_enum (enum_flags, subtype, 0, &device_caps,
1988 NULL, &sink_template, &src_template);
1996 gst_mf_video_enc_register_internal (plugin, rank, subtype, type_info,
1997 &device_caps, enum_flags, 0, transform, sink_template, src_template);
2000 gst_clear_object (&transform);
2001 gst_clear_caps (&sink_template);
2002 gst_clear_caps (&src_template);