2 * gstmsdkvpputil.c - MediaSDK video post processing utilities
4 * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
5 * Copyright (C) 2005-2012 David Schleef <ds@schleef.org>
6 * Copyright (C) 2016 Intel Corporation
7 * Copyright (C) 2018 Intel Corporation
8 * Author: Gwenole Beauchesne <gwenole.beauchesne@intel.com>
9 * Author: Victor Jaquez <victorx.jaquez@intel.com>
10 * Author: Sreerenj Balachandran <sreerenj.balachandran@intel.com>
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU Lesser General Public License
14 * as published by the Free Software Foundation; either version 2.1
15 * of the License, or (at your option) any later version.
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * Lesser General Public License for more details.
22 * You should have received a copy of the GNU Lesser General Public
23 * License along with this program; if not, write to the Free
24 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
25 * Boston, MA 02110-1301 USA
28 #include "gstmsdkvpputil.h"
29 #include "msdk-enums.h"
31 #define SWAP_GINT(a, b) do { \
32 const gint t = a; a = b; b = t; \
36 gst_msdkvpp_is_deinterlace_enabled (GstMsdkVPP * msdkvpp, GstVideoInfo * vip)
40 switch (msdkvpp->deinterlace_mode) {
41 case GST_MSDKVPP_DEINTERLACE_MODE_AUTO:
42 deinterlace = GST_VIDEO_INFO_IS_INTERLACED (vip);
44 case GST_MSDKVPP_DEINTERLACE_MODE_INTERLACED:
55 fixate_output_frame_size (GstMsdkVPP * thiz, GstVideoInfo * vinfo,
59 GValue tpar = G_VALUE_INIT;
63 to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
65 g_value_init (&tpar, GST_TYPE_FRACTION_RANGE);
66 gst_value_set_fraction_range_full (&tpar, 1, G_MAXINT, G_MAXINT, 1);
70 /* we have both PAR but they might not be fixated */
72 gint from_w, from_h, from_par_n, from_par_d, to_par_n, to_par_d;
74 gint from_dar_n, from_dar_d;
77 from_par_n = GST_VIDEO_INFO_PAR_N (vinfo);
78 from_par_d = GST_VIDEO_INFO_PAR_D (vinfo);
79 from_w = GST_VIDEO_INFO_WIDTH (vinfo);
80 from_h = GST_VIDEO_INFO_HEIGHT (vinfo);
82 /* adjust for crop settings (NOTE: msdk min frame size is 2x2) */
83 if ((thiz->crop_left + thiz->crop_right >= from_w - 1)
84 || (thiz->crop_top + thiz->crop_bottom >= from_h - 1)) {
85 GST_WARNING_OBJECT (thiz, "ignoring crop... cropping too much!");
87 from_w -= thiz->crop_left + thiz->crop_right;
88 from_h -= thiz->crop_top + thiz->crop_bottom;
91 /* compensate for rotation if needed */
92 if (thiz->rotation == 90 || thiz->rotation == 270) {
93 SWAP_GINT (from_w, from_h);
94 SWAP_GINT (from_par_n, from_par_d);
97 gst_structure_get_int (outs, "width", &w);
98 gst_structure_get_int (outs, "height", &h);
100 /* if both width and height are already fixed, we can't do anything
101 * about it anymore */
105 GST_DEBUG_OBJECT (thiz,
106 "dimensions already set to %dx%d, not fixating", w, h);
108 if (!gst_value_is_fixed (to_par)) {
109 if (gst_video_calculate_display_ratio (&n, &d, from_w, from_h,
110 from_par_n, from_par_d, w, h)) {
111 GST_DEBUG_OBJECT (thiz, "fixating to_par to %dx%d", n, d);
112 if (gst_structure_has_field (outs, "pixel-aspect-ratio"))
113 gst_structure_fixate_field_nearest_fraction (outs,
114 "pixel-aspect-ratio", n, d);
116 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
124 /* Calculate input DAR */
125 if (!gst_util_fraction_multiply (from_w, from_h, from_par_n, from_par_d,
126 &from_dar_n, &from_dar_d))
129 GST_DEBUG_OBJECT (thiz, "Input DAR is %d/%d", from_dar_n, from_dar_d);
131 /* If either width or height are fixed there's not much we
132 * can do either except choosing a height or width and PAR
133 * that matches the DAR as good as possible
137 gint set_w, set_par_n, set_par_d;
139 GST_DEBUG_OBJECT (thiz, "height is fixed (%d)", h);
141 /* If the PAR is fixed too, there's not much to do
142 * except choosing the width that is nearest to the
143 * width with the same DAR */
144 if (gst_value_is_fixed (to_par)) {
145 to_par_n = gst_value_get_fraction_numerator (to_par);
146 to_par_d = gst_value_get_fraction_denominator (to_par);
148 GST_DEBUG_OBJECT (thiz, "PAR is fixed %d/%d", to_par_n, to_par_d);
150 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
151 to_par_n, &num, &den))
154 w = (guint) gst_util_uint64_scale_int (h, num, den);
155 gst_structure_fixate_field_nearest_int (outs, "width", w);
160 /* The PAR is not fixed and it's quite likely that we can set
161 * an arbitrary PAR. */
163 /* Check if we can keep the input width */
164 tmp = gst_structure_copy (outs);
165 gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
166 gst_structure_get_int (tmp, "width", &set_w);
168 /* Might have failed but try to keep the DAR nonetheless by
169 * adjusting the PAR */
170 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, h, set_w,
171 &to_par_n, &to_par_d)) {
172 gst_structure_free (tmp);
176 if (!gst_structure_has_field (tmp, "pixel-aspect-ratio"))
177 gst_structure_set_value (tmp, "pixel-aspect-ratio", to_par);
178 gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio",
180 gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n,
182 gst_structure_free (tmp);
184 /* Check if the adjusted PAR is accepted */
185 if (set_par_n == to_par_n && set_par_d == to_par_d) {
186 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
187 set_par_n != set_par_d)
188 gst_structure_set (outs, "width", G_TYPE_INT, set_w,
189 "pixel-aspect-ratio", GST_TYPE_FRACTION, set_par_n, set_par_d,
194 /* Otherwise scale the width to the new PAR and check if the
195 * adjusted with is accepted. If all that fails we can't keep
197 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d,
198 set_par_n, &num, &den))
201 w = (guint) gst_util_uint64_scale_int (h, num, den);
202 gst_structure_fixate_field_nearest_int (outs, "width", w);
203 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
204 set_par_n != set_par_d)
205 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
206 set_par_n, set_par_d, NULL);
211 gint set_h, set_par_n, set_par_d;
213 GST_DEBUG_OBJECT (thiz, "width is fixed (%d)", w);
215 /* If the PAR is fixed too, there's not much to do
216 * except choosing the height that is nearest to the
217 * height with the same DAR */
218 if (gst_value_is_fixed (to_par)) {
219 to_par_n = gst_value_get_fraction_numerator (to_par);
220 to_par_d = gst_value_get_fraction_denominator (to_par);
222 GST_DEBUG_OBJECT (thiz, "PAR is fixed %d/%d", to_par_n, to_par_d);
224 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
225 to_par_n, &num, &den))
228 h = (guint) gst_util_uint64_scale_int (w, den, num);
229 gst_structure_fixate_field_nearest_int (outs, "height", h);
234 /* The PAR is not fixed and it's quite likely that we can set
235 * an arbitrary PAR. */
237 /* Check if we can keep the input height */
238 tmp = gst_structure_copy (outs);
239 gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
240 gst_structure_get_int (tmp, "height", &set_h);
242 /* Might have failed but try to keep the DAR nonetheless by
243 * adjusting the PAR */
244 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_h, w,
245 &to_par_n, &to_par_d)) {
246 gst_structure_free (tmp);
250 if (!gst_structure_has_field (tmp, "pixel-aspect-ratio"))
251 gst_structure_set_value (tmp, "pixel-aspect-ratio", to_par);
252 gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio",
254 gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n,
256 gst_structure_free (tmp);
258 /* Check if the adjusted PAR is accepted */
259 if (set_par_n == to_par_n && set_par_d == to_par_d) {
260 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
261 set_par_n != set_par_d)
262 gst_structure_set (outs, "height", G_TYPE_INT, set_h,
263 "pixel-aspect-ratio", GST_TYPE_FRACTION, set_par_n, set_par_d,
268 /* Otherwise scale the height to the new PAR and check if the
269 * adjusted with is accepted. If all that fails we can't keep
271 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d,
272 set_par_n, &num, &den))
275 h = (guint) gst_util_uint64_scale_int (w, den, num);
276 gst_structure_fixate_field_nearest_int (outs, "height", h);
277 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
278 set_par_n != set_par_d)
279 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
280 set_par_n, set_par_d, NULL);
283 } else if (gst_value_is_fixed (to_par)) {
285 gint set_h, set_w, f_h, f_w;
287 to_par_n = gst_value_get_fraction_numerator (to_par);
288 to_par_d = gst_value_get_fraction_denominator (to_par);
290 /* Calculate scale factor for the PAR change */
291 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_n,
292 to_par_d, &num, &den))
295 /* Try to keep the input height (because of interlacing) */
296 tmp = gst_structure_copy (outs);
297 gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
298 gst_structure_get_int (tmp, "height", &set_h);
300 /* This might have failed but try to scale the width
301 * to keep the DAR nonetheless */
302 w = (guint) gst_util_uint64_scale_int (set_h, num, den);
303 gst_structure_fixate_field_nearest_int (tmp, "width", w);
304 gst_structure_get_int (tmp, "width", &set_w);
305 gst_structure_free (tmp);
307 /* We kept the DAR and the height is nearest to the original height */
309 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
310 G_TYPE_INT, set_h, NULL);
317 /* If the former failed, try to keep the input width at least */
318 tmp = gst_structure_copy (outs);
319 gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
320 gst_structure_get_int (tmp, "width", &set_w);
322 /* This might have failed but try to scale the width
323 * to keep the DAR nonetheless */
324 h = (guint) gst_util_uint64_scale_int (set_w, den, num);
325 gst_structure_fixate_field_nearest_int (tmp, "height", h);
326 gst_structure_get_int (tmp, "height", &set_h);
327 gst_structure_free (tmp);
329 /* We kept the DAR and the width is nearest to the original width */
331 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
332 G_TYPE_INT, set_h, NULL);
336 /* If all this failed, keep the height that was nearest to the orignal
337 * height and the nearest possible width. This changes the DAR but
338 * there's not much else to do here.
340 gst_structure_set (outs, "width", G_TYPE_INT, f_w, "height", G_TYPE_INT,
345 gint set_h, set_w, set_par_n, set_par_d, tmp2;
347 /* width, height and PAR are not fixed but passthrough is not possible */
349 /* First try to keep the height and width as good as possible
351 tmp = gst_structure_copy (outs);
352 gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
353 gst_structure_get_int (tmp, "height", &set_h);
354 gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
355 gst_structure_get_int (tmp, "width", &set_w);
357 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_h, set_w,
358 &to_par_n, &to_par_d)) {
359 gst_structure_free (tmp);
363 if (!gst_structure_has_field (tmp, "pixel-aspect-ratio"))
364 gst_structure_set_value (tmp, "pixel-aspect-ratio", to_par);
365 gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio",
367 gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n,
369 gst_structure_free (tmp);
371 if (set_par_n == to_par_n && set_par_d == to_par_d) {
372 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
373 G_TYPE_INT, set_h, NULL);
375 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
376 set_par_n != set_par_d)
377 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
378 set_par_n, set_par_d, NULL);
382 /* Otherwise try to scale width to keep the DAR with the set
384 if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d,
385 set_par_n, &num, &den))
388 w = (guint) gst_util_uint64_scale_int (set_h, num, den);
389 tmp = gst_structure_copy (outs);
390 gst_structure_fixate_field_nearest_int (tmp, "width", w);
391 gst_structure_get_int (tmp, "width", &tmp2);
392 gst_structure_free (tmp);
395 gst_structure_set (outs, "width", G_TYPE_INT, tmp2, "height",
396 G_TYPE_INT, set_h, NULL);
397 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
398 set_par_n != set_par_d)
399 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
400 set_par_n, set_par_d, NULL);
404 /* ... or try the same with the height */
405 h = (guint) gst_util_uint64_scale_int (set_w, den, num);
406 tmp = gst_structure_copy (outs);
407 gst_structure_fixate_field_nearest_int (tmp, "height", h);
408 gst_structure_get_int (tmp, "height", &tmp2);
409 gst_structure_free (tmp);
412 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
413 G_TYPE_INT, tmp2, NULL);
414 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
415 set_par_n != set_par_d)
416 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
417 set_par_n, set_par_d, NULL);
421 /* If all fails we can't keep the DAR and take the nearest values
422 * for everything from the first try */
423 gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
424 G_TYPE_INT, set_h, NULL);
425 if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
426 set_par_n != set_par_d)
427 gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
428 set_par_n, set_par_d, NULL);
434 g_value_unset (&tpar);
442 GST_ELEMENT_ERROR (thiz, CORE, NEGOTIATION, (NULL),
443 ("Error calculating the output scaled size - integer overflow"));
449 fixate_frame_rate (GstMsdkVPP * thiz, GstVideoInfo * vinfo, GstStructure * outs)
451 gint fps_n = 0, fps_d;
453 /* fixate the srcpad fps */
454 if (gst_structure_fixate_field (outs, "framerate"))
455 gst_structure_get (outs, "framerate", GST_TYPE_FRACTION, &fps_n, &fps_d,
458 /* if we don't have a fixed non-zero fps_n, use the sinkpad fps */
460 fps_n = GST_VIDEO_INFO_FPS_N (vinfo);
461 fps_d = GST_VIDEO_INFO_FPS_D (vinfo);
464 if (gst_msdkvpp_is_deinterlace_enabled (thiz, vinfo)) {
465 /* Fixme: set double framerate?:
466 * msdk is not outputting double framerate for bob or adv deinterlace */
467 if (!gst_util_fraction_multiply (fps_n, fps_d, 1, 1, &fps_n, &fps_d))
470 gst_structure_set (outs, "framerate", GST_TYPE_FRACTION, fps_n, fps_d, NULL);
476 GST_ELEMENT_ERROR (thiz, CORE, NEGOTIATION, (NULL),
477 ("Error calculating the output framerate - integer overflow"));
483 set_multiview_mode (GstMsdkVPP * thiz, GstVideoInfo * vinfo,
486 const gchar *caps_str;
489 gst_video_multiview_mode_to_caps_string (GST_VIDEO_INFO_MULTIVIEW_MODE
494 gst_structure_set (outs, "multiview-mode", G_TYPE_STRING, caps_str,
495 "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
496 GST_VIDEO_INFO_MULTIVIEW_FLAGS (vinfo), GST_FLAG_SET_MASK_EXACT, NULL);
498 if (GST_VIDEO_INFO_VIEWS (vinfo) > 1) {
499 gst_structure_set (outs, "views", G_TYPE_INT, GST_VIDEO_INFO_VIEWS (vinfo),
506 set_interlace_mode (GstMsdkVPP * thiz, GstVideoInfo * vinfo,
509 const gchar *interlace_mode = NULL;
511 if (gst_msdkvpp_is_deinterlace_enabled (thiz, vinfo)) {
512 interlace_mode = "progressive";
515 gst_video_interlace_mode_to_string (GST_VIDEO_INFO_INTERLACE_MODE
522 gst_structure_set (outs, "interlace-mode", G_TYPE_STRING, interlace_mode,
528 _get_preferred_src_caps (GstMsdkVPP * thiz, GstVideoInfo * vinfo,
531 GstStructure *structure;
534 structure = gst_caps_get_structure (srccaps, 0);
537 structure = gst_structure_copy (structure);
539 if (thiz->keep_aspect)
540 gst_structure_set (structure, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1,
543 /* Fixate the format */
544 if (!gst_structure_fixate_field (structure, "format"))
547 /* Fixate the frame size */
548 if (!fixate_output_frame_size (thiz, vinfo, structure))
551 /* Fixate the framerate */
552 if (!fixate_frame_rate (thiz, vinfo, structure))
555 /* set multiview mode based on input caps */
556 if (!set_multiview_mode (thiz, vinfo, structure))
559 /*Fixme: Set colorimetry */
561 /* set interlace mode */
562 if (!set_interlace_mode (thiz, vinfo, structure))
563 goto interlace_mode_failed;
565 outcaps = gst_caps_new_empty ();
566 gst_caps_append_structure (outcaps, structure);
573 GST_WARNING_OBJECT (thiz, "Could not fixate src caps");
574 gst_structure_free (structure);
577 interlace_mode_failed:
579 GST_WARNING_OBJECT (thiz, "Invalid sink caps interlace mode");
585 * gst_msdkvpp_fixate_srccaps:
586 * @vpp: a #GstMsdkVPP instance
587 * @sinkcaps: fixed #GstCaps from sink pad
588 * @srccaps: #GstCaps from src pad to fixate
590 * Given @srccaps and @sinkcaps returns a new allocated #GstCaps with
591 * the fixated caps for the src pad.
593 * Returns: A new allocated #GstCaps
596 gst_msdkvpp_fixate_srccaps (GstMsdkVPP * msdkvpp,
597 GstCaps * sinkcaps, GstCaps * srccaps)
600 if (!gst_video_info_from_caps (&vi, sinkcaps))
602 return _get_preferred_src_caps (msdkvpp, &vi, srccaps);