2 * Copyright (C) 2022 Seungha Yang <seungha@centricular.com>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
24 #include "gstqsvjpegenc.h"
31 #include <gst/d3d11/gstd3d11.h>
33 #include <gst/va/gstva.h>
36 GST_DEBUG_CATEGORY_STATIC (gst_qsv_jpeg_enc_debug);
37 #define GST_CAT_DEFAULT gst_qsv_jpeg_enc_debug
45 #define DEFAULT_JPEG_QUALITY 85
47 typedef struct _GstQsvJpegEncClassData
56 } GstQsvJpegEncClassData;
58 typedef struct _GstQsvJpegEnc
63 /* protected by prop_lock */
64 gboolean property_updated;
70 typedef struct _GstQsvJpegEncClass
72 GstQsvEncoderClass parent_class;
76 static GstElementClass *parent_class = nullptr;
78 #define GST_QSV_JPEG_ENC(object) ((GstQsvJpegEnc *) (object))
79 #define GST_QSV_JPEG_ENC_GET_CLASS(object) \
80 (G_TYPE_INSTANCE_GET_CLASS ((object),G_TYPE_FROM_INSTANCE (object),GstQsvJpegEncClass))
82 static void gst_qsv_jpeg_enc_finalize (GObject * object);
83 static void gst_qsv_jpeg_enc_set_property (GObject * object, guint prop_id,
84 const GValue * value, GParamSpec * pspec);
85 static void gst_qsv_jpeg_enc_get_property (GObject * object, guint prop_id,
86 GValue * value, GParamSpec * pspec);
88 static gboolean gst_qsv_jpeg_enc_set_format (GstQsvEncoder * encoder,
89 GstVideoCodecState * state, mfxVideoParam * param,
90 GPtrArray * extra_params);
91 static gboolean gst_qsv_jpeg_enc_set_output_state (GstQsvEncoder * encoder,
92 GstVideoCodecState * state, mfxSession session);
93 static GstQsvEncoderReconfigure
94 gst_qsv_jpeg_enc_check_reconfigure (GstQsvEncoder * encoder, mfxSession session,
95 mfxVideoParam * param, GPtrArray * extra_params);
98 gst_qsv_jpeg_enc_class_init (GstQsvJpegEncClass * klass, gpointer data)
100 GObjectClass *object_class = G_OBJECT_CLASS (klass);
101 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
102 GstQsvEncoderClass *qsvenc_class = GST_QSV_ENCODER_CLASS (klass);
103 GstQsvJpegEncClassData *cdata = (GstQsvJpegEncClassData *) data;
105 qsvenc_class->codec_id = MFX_CODEC_JPEG;
106 qsvenc_class->impl_index = cdata->impl_index;
107 qsvenc_class->adapter_luid = cdata->adapter_luid;
108 qsvenc_class->display_path = cdata->display_path;
110 object_class->finalize = gst_qsv_jpeg_enc_finalize;
111 object_class->set_property = gst_qsv_jpeg_enc_set_property;
112 object_class->get_property = gst_qsv_jpeg_enc_get_property;
114 g_object_class_install_property (object_class, PROP_QUALITY,
115 g_param_spec_uint ("quality", "Quality",
116 "Encoding quality, 100 for best quality",
117 1, 100, DEFAULT_JPEG_QUALITY, (GParamFlags)
118 (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
120 parent_class = (GstElementClass *) g_type_class_peek_parent (klass);
123 std::string long_name = "Intel Quick Sync Video " +
124 std::string (cdata->description) + " JPEG Encoder";
126 gst_element_class_set_metadata (element_class, long_name.c_str (),
127 "Codec/Encoder/Video/Hardware",
128 "Intel Quick Sync Video JPEG Encoder",
129 "Seungha Yang <seungha@centricular.com>");
131 gst_element_class_set_static_metadata (element_class,
132 "Intel Quick Sync Video JPEG Encoder",
133 "Codec/Encoder/Video/Hardware",
134 "Intel Quick Sync Video JPEG Encoder",
135 "Seungha Yang <seungha@centricular.com>");
138 gst_element_class_add_pad_template (element_class,
139 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
141 gst_element_class_add_pad_template (element_class,
142 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
145 qsvenc_class->set_format = GST_DEBUG_FUNCPTR (gst_qsv_jpeg_enc_set_format);
146 qsvenc_class->set_output_state =
147 GST_DEBUG_FUNCPTR (gst_qsv_jpeg_enc_set_output_state);
148 qsvenc_class->check_reconfigure =
149 GST_DEBUG_FUNCPTR (gst_qsv_jpeg_enc_check_reconfigure);
151 klass->interlaved = cdata->interlaved;
153 gst_caps_unref (cdata->sink_caps);
154 gst_caps_unref (cdata->src_caps);
155 g_free (cdata->description);
160 gst_qsv_jpeg_enc_init (GstQsvJpegEnc * self)
162 self->quality = DEFAULT_JPEG_QUALITY;
164 g_mutex_init (&self->prop_lock);
168 gst_qsv_jpeg_enc_finalize (GObject * object)
170 GstQsvJpegEnc *self = GST_QSV_JPEG_ENC (object);
172 g_mutex_clear (&self->prop_lock);
174 G_OBJECT_CLASS (parent_class)->finalize (object);
178 gst_qsv_jpeg_enc_check_update_uint (GstQsvJpegEnc * self, guint * old_val,
181 if (*old_val == new_val)
184 g_mutex_lock (&self->prop_lock);
186 self->property_updated = TRUE;
187 g_mutex_unlock (&self->prop_lock);
191 gst_qsv_jpeg_enc_set_property (GObject * object, guint prop_id,
192 const GValue * value, GParamSpec * pspec)
194 GstQsvJpegEnc *self = GST_QSV_JPEG_ENC (object);
198 gst_qsv_jpeg_enc_check_update_uint (self, &self->quality,
199 g_value_get_uint (value));
202 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
208 gst_qsv_jpeg_enc_get_property (GObject * object, guint prop_id, GValue * value,
211 GstQsvJpegEnc *self = GST_QSV_JPEG_ENC (object);
215 g_value_set_uint (value, self->quality);
218 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
224 gst_qsv_jpeg_enc_set_format (GstQsvEncoder * encoder,
225 GstVideoCodecState * state, mfxVideoParam * param, GPtrArray * extra_params)
227 GstQsvJpegEnc *self = GST_QSV_JPEG_ENC (encoder);
228 GstQsvJpegEncClass *klass = GST_QSV_JPEG_ENC_GET_CLASS (self);
229 GstVideoInfo *info = &state->info;
230 mfxFrameInfo *frame_info;
232 frame_info = ¶m->mfx.FrameInfo;
234 frame_info->Width = frame_info->CropW = info->width;
235 frame_info->Height = frame_info->CropH = info->height;
237 frame_info->PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
239 if (GST_VIDEO_INFO_FPS_N (info) > 0 && GST_VIDEO_INFO_FPS_D (info) > 0) {
240 frame_info->FrameRateExtN = GST_VIDEO_INFO_FPS_N (info);
241 frame_info->FrameRateExtD = GST_VIDEO_INFO_FPS_D (info);
243 /* HACK: Same as x264enc */
244 frame_info->FrameRateExtN = 25;
245 frame_info->FrameRateExtD = 1;
248 frame_info->AspectRatioW = GST_VIDEO_INFO_PAR_N (info);
249 frame_info->AspectRatioH = GST_VIDEO_INFO_PAR_D (info);
251 switch (GST_VIDEO_INFO_FORMAT (info)) {
252 case GST_VIDEO_FORMAT_NV12:
253 frame_info->ChromaFormat = MFX_CHROMAFORMAT_YUV420;
254 frame_info->FourCC = MFX_FOURCC_NV12;
255 frame_info->BitDepthLuma = 8;
256 frame_info->BitDepthChroma = 8;
258 case GST_VIDEO_FORMAT_BGRA:
259 frame_info->ChromaFormat = MFX_CHROMAFORMAT_YUV444;
260 frame_info->FourCC = MFX_FOURCC_RGB4;
263 GST_ERROR_OBJECT (self, "Unexpected format %s",
264 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (info)));
268 g_mutex_lock (&self->prop_lock);
269 param->mfx.CodecId = MFX_CODEC_JPEG;
270 param->mfx.CodecProfile = MFX_PROFILE_JPEG_BASELINE;
271 param->mfx.Quality = self->quality;
272 if (klass->interlaved)
273 param->mfx.Interleaved = 1;
275 param->mfx.Interleaved = 0;
276 param->mfx.RestartInterval = 0;
277 param->ExtParam = (mfxExtBuffer **) extra_params->pdata;
278 param->NumExtParam = extra_params->len;
280 self->property_updated = FALSE;
281 g_mutex_unlock (&self->prop_lock);
287 gst_qsv_jpeg_enc_set_output_state (GstQsvEncoder * encoder,
288 GstVideoCodecState * state, mfxSession session)
292 GstVideoCodecState *out_state;
294 caps = gst_caps_from_string ("image/jpeg");
295 out_state = gst_video_encoder_set_output_state (GST_VIDEO_ENCODER (encoder),
297 gst_video_codec_state_unref (out_state);
299 tags = gst_tag_list_new_empty ();
300 gst_tag_list_add (tags, GST_TAG_MERGE_REPLACE, GST_TAG_ENCODER, "qsvjpegenc",
303 gst_video_encoder_merge_tags (GST_VIDEO_ENCODER (encoder),
304 tags, GST_TAG_MERGE_REPLACE);
305 gst_tag_list_unref (tags);
310 static GstQsvEncoderReconfigure
311 gst_qsv_jpeg_enc_check_reconfigure (GstQsvEncoder * encoder, mfxSession session,
312 mfxVideoParam * param, GPtrArray * extra_params)
314 GstQsvJpegEnc *self = GST_QSV_JPEG_ENC (encoder);
315 GstQsvEncoderReconfigure ret = GST_QSV_ENCODER_RECONFIGURE_NONE;
317 g_mutex_lock (&self->prop_lock);
318 if (self->property_updated)
319 ret = GST_QSV_ENCODER_RECONFIGURE_FULL;
321 self->property_updated = FALSE;
322 g_mutex_unlock (&self->prop_lock);
334 gst_qsv_jpeg_enc_register (GstPlugin * plugin, guint rank, guint impl_index,
335 GstObject * device, mfxSession session)
339 static const Resolution resolutions_to_check[] = {
340 {1280, 720}, {1920, 1088}, {2560, 1440}, {3840, 2160}, {4096, 2160},
341 {7680, 4320}, {8192, 4320}
343 std::vector < std::string > supported_formats;
344 Resolution max_resolution;
346 gboolean interlaved = TRUE;
348 GST_DEBUG_CATEGORY_INIT (gst_qsv_jpeg_enc_debug,
349 "qsvjpegenc", 0, "qsvjpegenc");
351 memset (¶m, 0, sizeof (mfxVideoParam));
352 memset (&max_resolution, 0, sizeof (Resolution));
354 param.AsyncDepth = 4;
355 param.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY;
358 mfx->LowPower = MFX_CODINGOPTION_UNKNOWN;
359 mfx->CodecId = MFX_CODEC_JPEG;
360 mfx->CodecProfile = MFX_PROFILE_JPEG_BASELINE;
361 mfx->Quality = DEFAULT_JPEG_QUALITY;
362 mfx->Interleaved = 1;
363 mfx->RestartInterval = 0;
365 mfx->FrameInfo.Width = mfx->FrameInfo.CropW = GST_ROUND_UP_16 (320);
366 mfx->FrameInfo.Height = mfx->FrameInfo.CropH = GST_ROUND_UP_16 (240);
368 mfx->FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420;
369 mfx->FrameInfo.FourCC = MFX_FOURCC_NV12;
370 mfx->FrameInfo.FrameRateExtN = 30;
371 mfx->FrameInfo.FrameRateExtD = 1;
372 mfx->FrameInfo.AspectRatioW = 1;
373 mfx->FrameInfo.AspectRatioH = 1;
374 mfx->FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
376 status = MFXVideoENCODE_Query (session, ¶m, ¶m);
377 if (status == MFX_WRN_PARTIAL_ACCELERATION) {
378 mfx->Interleaved = 0;
381 status = MFXVideoENCODE_Query (session, ¶m, ¶m);
384 if (status != MFX_ERR_NONE)
387 supported_formats.push_back ("NV12");
389 mfx->FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV444;
390 mfx->FrameInfo.FourCC = MFX_FOURCC_RGB4;
391 status = MFXVideoENCODE_Query (session, ¶m, ¶m);
393 /* TODO: Add YUY2 support, d3d11 doesn't support the format yet */
394 if (status == MFX_ERR_NONE)
395 supported_formats.push_back ("BGRA");
397 mfx->FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420;
398 mfx->FrameInfo.FourCC = MFX_FOURCC_NV12;
400 /* Check max-resolution */
401 for (guint i = 0; i < G_N_ELEMENTS (resolutions_to_check); i++) {
402 mfx->FrameInfo.Width = mfx->FrameInfo.CropW = resolutions_to_check[i].width;
403 mfx->FrameInfo.Height = mfx->FrameInfo.CropH =
404 resolutions_to_check[i].height;
406 if (MFXVideoENCODE_Query (session, ¶m, ¶m) != MFX_ERR_NONE)
409 max_resolution.width = resolutions_to_check[i].width;
410 max_resolution.height = resolutions_to_check[i].height;
413 GST_INFO ("Maximum supported resolution: %dx%d",
414 max_resolution.width, max_resolution.height);
416 /* To cover both landscape and portrait,
417 * select max value (width in this case) */
418 guint resolution = MAX (max_resolution.width, max_resolution.height);
419 std::string sink_caps_str = "video/x-raw";
421 sink_caps_str += ", width=(int) [ 16, " + std::to_string (resolution) + " ]";
422 sink_caps_str += ", height=(int) [ 16, " + std::to_string (resolution) + " ]";
425 if (supported_formats.size () > 1) {
426 sink_caps_str += ", format=(string) { ";
428 for (const auto &iter: supported_formats) {
430 sink_caps_str += ", ";
433 sink_caps_str += iter;
436 sink_caps_str += " }";
438 sink_caps_str += ", format=(string) " + supported_formats[0];
442 GstCaps *sink_caps = gst_caps_from_string (sink_caps_str.c_str ());
445 GstCaps *d3d11_caps = gst_caps_copy (sink_caps);
446 GstCapsFeatures *caps_features =
447 gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, nullptr);
448 gst_caps_set_features_simple (d3d11_caps, caps_features);
449 gst_caps_append (d3d11_caps, sink_caps);
450 sink_caps = d3d11_caps;
452 GstCaps *va_caps = gst_caps_copy (sink_caps);
453 GstCapsFeatures *caps_features =
454 gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_VA, nullptr);
455 gst_caps_set_features_simple (va_caps, caps_features);
456 gst_caps_append (va_caps, sink_caps);
460 std::string src_caps_str = "image/jpeg";
461 src_caps_str += ", width=(int) [ 16, " + std::to_string (resolution) + " ]";
462 src_caps_str += ", height=(int) [ 16, " + std::to_string (resolution) + " ]";
464 GstCaps *src_caps = gst_caps_from_string (src_caps_str.c_str ());
466 GST_MINI_OBJECT_FLAG_SET (sink_caps, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
467 GST_MINI_OBJECT_FLAG_SET (src_caps, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
469 GstQsvJpegEncClassData *cdata = g_new0 (GstQsvJpegEncClassData, 1);
470 cdata->sink_caps = sink_caps;
471 cdata->src_caps = src_caps;
472 cdata->impl_index = impl_index;
473 cdata->interlaved = interlaved;
476 g_object_get (device, "adapter-luid", &cdata->adapter_luid,
477 "description", &cdata->description, nullptr);
479 g_object_get (device, "path", &cdata->display_path, nullptr);
485 GTypeInfo type_info = {
486 sizeof (GstQsvJpegEncClass),
489 (GClassInitFunc) gst_qsv_jpeg_enc_class_init,
492 sizeof (GstQsvJpegEnc),
494 (GInstanceInitFunc) gst_qsv_jpeg_enc_init,
497 type_name = g_strdup ("GstQsvJpegEnc");
498 feature_name = g_strdup ("qsvjpegenc");
501 while (g_type_from_name (type_name)) {
504 g_free (feature_name);
505 type_name = g_strdup_printf ("GstQsvJpegDevice%dEnc", index);
506 feature_name = g_strdup_printf ("qsvjpegdevice%denc", index);
509 type = g_type_register_static (GST_TYPE_QSV_ENCODER, type_name, &type_info,
512 if (rank > 0 && index != 0)
515 if (!gst_element_register (plugin, feature_name, rank, type))
516 GST_WARNING ("Failed to register plugin '%s'", type_name);
519 g_free (feature_name);