From 058f1e92b73bedf65d5b81520695d80e0b5403de Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Wed, 19 May 2021 16:22:46 +0900 Subject: [PATCH] compositor: Add scaling policy to support PAR-aware scaling Adding "sizing-policy" property for user to be able to specify scaling policy (aspect-ratio for example). At the moment, supported mode is only keep-aspect-ratio, but we might be able to add more policies such as cropping, etc. Fixes: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/issues/696 Part-of: --- docs/plugins/gst_plugins_cache.json | 27 ++++++ gst/compositor/compositor.c | 184 +++++++++++++++++++++++++++++++----- gst/compositor/compositor.h | 23 +++++ 3 files changed, 209 insertions(+), 25 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 86fd32c..2831b2f 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -1977,6 +1977,18 @@ "type": "GstCompositorOperator", "writable": true }, + "sizing-policy": { + "blurb": "Sizing policy to use for image scaling", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": true, + "default": "none (0)", + "mutable": "null", + "readable": true, + "type": "GstCompositorSizingPolicy", + "writable": true + }, "width": { "blurb": "Width of the picture", "conditionally-available": false, @@ -2021,6 +2033,21 @@ } }, "signals": {} + }, + "GstCompositorSizingPolicy": { + "kind": "enum", + "values": [ + { + "desc": "None: Image is scaled to fill configured destination rectangle without padding or keeping the aspect ratio", + "name": "none", + "value": "0" + }, + { + "desc": "Keep Aspect Ratio: Image is scaled to fit destination rectangle specified by GstCompositorPad:{xpos, ypos, width, height} with preserved aspect ratio. Resulting image will be centered in the destination rectangle with padding if necessary", + "name": "keep-aspect-ratio", + "value": "1" + } + ] } }, "package": "GStreamer Base Plug-ins", diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 1fa5a79..2b5ccee 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -165,12 +165,40 @@ gst_compositor_background_get_type (void) return compositor_background_type; } +#define GST_TYPE_COMPOSITOR_SIZING_POLICY (gst_compositor_sizing_policy_get_type()) +static GType +gst_compositor_sizing_policy_get_type (void) +{ + static GType sizing_policy_type = 0; + + static const GEnumValue sizing_polices[] = { + {COMPOSITOR_SIZING_POLICY_NONE, + "None: Image is scaled to fill configured destination rectangle without " + "padding or keeping the aspect ratio", "none"}, + {COMPOSITOR_SIZING_POLICY_KEEP_ASPECT_RATIO, + "Keep Aspect Ratio: Image is scaled to fit destination rectangle " + "specified by GstCompositorPad:{xpos, ypos, width, height} " + "with preserved aspect ratio. Resulting image will be centered in " + "the destination rectangle with padding if necessary", + "keep-aspect-ratio"}, + {0, NULL, NULL}, + }; + + if (!sizing_policy_type) { + sizing_policy_type = + g_enum_register_static ("GstCompositorSizingPolicy", sizing_polices); + } + return sizing_policy_type; +} + #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 #define DEFAULT_PAD_WIDTH -1 #define DEFAULT_PAD_HEIGHT -1 #define DEFAULT_PAD_ALPHA 1.0 #define DEFAULT_PAD_OPERATOR COMPOSITOR_OPERATOR_OVER +#define DEFAULT_PAD_SIZING_POLICY COMPOSITOR_SIZING_POLICY_NONE + enum { PROP_PAD_0, @@ -180,6 +208,7 @@ enum PROP_PAD_HEIGHT, PROP_PAD_ALPHA, PROP_PAD_OPERATOR, + PROP_PAD_SIZING_POLICY, }; G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad, @@ -210,6 +239,9 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_OPERATOR: g_value_set_enum (value, pad->op); break; + case PROP_PAD_SIZING_POLICY: + g_value_set_enum (value, pad->sizing_policy); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -247,6 +279,11 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, gst_video_aggregator_pad_set_needs_alpha (GST_VIDEO_AGGREGATOR_PAD (pad), pad->op == COMPOSITOR_OPERATOR_ADD); break; + case PROP_PAD_SIZING_POLICY: + pad->sizing_policy = g_value_get_enum (value); + gst_video_aggregator_convert_pad_update_conversion_info + (GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad)); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -255,18 +292,22 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, static void _mixer_pad_get_output_size (GstCompositor * comp, GstCompositorPad * comp_pad, - gint out_par_n, gint out_par_d, gint * width, gint * height) + gint out_par_n, gint out_par_d, gint * width, gint * height, + gint * x_offset, gint * y_offset) { GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (comp_pad); gint pad_width, pad_height; guint dar_n, dar_d; + *x_offset = 0; + *y_offset = 0; + *width = 0; + *height = 0; + /* FIXME: Anything better we can do here? */ if (!vagg_pad->info.finfo || vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) { GST_DEBUG_OBJECT (comp_pad, "Have no caps yet"); - *width = 0; - *height = 0; return; } @@ -286,32 +327,95 @@ _mixer_pad_get_output_size (GstCompositor * comp, GstCompositorPad * comp_pad, 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height; } - if (pad_width == 0 || pad_height == 0) { - *width = 0; - *height = 0; + if (pad_width == 0 || pad_height == 0) return; - } if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) { GST_WARNING_OBJECT (comp_pad, "Cannot calculate display aspect ratio"); - *width = *height = 0; return; } + GST_LOG_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); - /* Pick either height or width, whichever is an integer multiple of the - * display aspect ratio. However, prefer preserving the height to account - * for interlaced video. */ - if (pad_height % dar_n == 0) { - pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); - } else if (pad_width % dar_d == 0) { - pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n); - } else { - pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); + switch (comp_pad->sizing_policy) { + case COMPOSITOR_SIZING_POLICY_NONE: + /* Pick either height or width, whichever is an integer multiple of the + * display aspect ratio. However, prefer preserving the height to account + * for interlaced video. */ + if (pad_height % dar_n == 0) { + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); + } else if (pad_width % dar_d == 0) { + pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n); + } else { + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); + } + break; + case COMPOSITOR_SIZING_POLICY_KEEP_ASPECT_RATIO: + { + gint from_dar_n, from_dar_d, to_dar_n, to_dar_d, num, den; + + /* Calculate DAR again with actual video size */ + if (!gst_util_fraction_multiply (GST_VIDEO_INFO_WIDTH (&vagg_pad->info), + GST_VIDEO_INFO_HEIGHT (&vagg_pad->info), + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), &from_dar_n, + &from_dar_d)) { + from_dar_n = from_dar_d = -1; + } + + if (!gst_util_fraction_multiply (pad_width, pad_height, + out_par_n, out_par_d, &to_dar_n, &to_dar_d)) { + to_dar_n = to_dar_d = -1; + } + + if (from_dar_n != to_dar_n || from_dar_d != to_dar_d) { + /* Calculate new output resolution */ + if (from_dar_n != -1 && from_dar_d != -1 + && gst_util_fraction_multiply (from_dar_n, from_dar_d, + out_par_d, out_par_n, &num, &den)) { + GstVideoRectangle src_rect, dst_rect, rst_rect; + + src_rect.h = gst_util_uint64_scale_int (pad_width, den, num); + if (src_rect.h == 0) { + pad_width = 0; + pad_height = 0; + break; + } + + src_rect.x = src_rect.y = 0; + src_rect.w = pad_width; + + dst_rect.x = dst_rect.y = 0; + dst_rect.w = pad_width; + dst_rect.h = pad_height; + + /* Scale rect to be centered in destination rect */ + gst_video_center_rect (&src_rect, &dst_rect, &rst_rect, TRUE); + + GST_LOG_OBJECT (comp_pad, + "Re-calculated size %dx%d -> %dx%d (x-offset %d, y-offset %d)", + pad_width, pad_height, rst_rect.w, rst_rect.h, rst_rect.x, + rst_rect.h); + + *x_offset = rst_rect.x; + *y_offset = rst_rect.y; + pad_width = rst_rect.w; + pad_height = rst_rect.h; + } else { + GST_WARNING_OBJECT (comp_pad, "Failed to calculate output size"); + + *x_offset = 0; + *y_offset = 0; + pad_width = 0; + pad_height = 0; + } + } + break; + } } *width = pad_width; @@ -360,6 +464,7 @@ _pad_obscures_rectangle (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad, GstStructure *converter_config = NULL; gboolean fill_border = TRUE; guint32 border_argb = 0xff000000; + gint x_offset, y_offset; /* No buffer to obscure the rectangle with */ if (!gst_video_aggregator_pad_has_current_buffer (pad)) @@ -391,7 +496,9 @@ _pad_obscures_rectangle (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad, /* Handle pixel and display aspect ratios to find the actual size */ _mixer_pad_get_output_size (GST_COMPOSITOR (vagg), cpad, GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info), - &(pad_rect.w), &(pad_rect.h)); + &(pad_rect.w), &(pad_rect.h), &x_offset, &y_offset); + pad_rect.x += x_offset; + pad_rect.y += y_offset; if (!is_rectangle_contained (rect, pad_rect)) return FALSE; @@ -430,14 +537,15 @@ gst_compositor_pad_prepare_frame_start (GstVideoAggregatorPad * pad, _mixer_pad_get_output_size (GST_COMPOSITOR (vagg), cpad, GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info), - &width, &height); + &width, &height, &cpad->x_offset, &cpad->y_offset); if (cpad->alpha == 0.0) { GST_DEBUG_OBJECT (pad, "Pad has alpha 0.0, not converting frame"); return; } - frame_rect = clamp_rectangle (cpad->xpos, cpad->ypos, width, height, + frame_rect = clamp_rectangle (cpad->xpos + cpad->x_offset, + cpad->ypos + cpad->y_offset, width, height, GST_VIDEO_INFO_WIDTH (&vagg->info), GST_VIDEO_INFO_HEIGHT (&vagg->info)); if (frame_rect.w == 0 || frame_rect.h == 0) { @@ -476,6 +584,7 @@ gst_compositor_pad_create_conversion_info (GstVideoAggregatorConvertPad * pad, { GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); gint width, height; + gint x_offset, y_offset; GST_VIDEO_AGGREGATOR_CONVERT_PAD_CLASS (gst_compositor_pad_parent_class)->create_conversion_info (pad, vagg, @@ -485,7 +594,7 @@ gst_compositor_pad_create_conversion_info (GstVideoAggregatorConvertPad * pad, _mixer_pad_get_output_size (GST_COMPOSITOR (vagg), cpad, GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info), - &width, &height); + &width, &height, &x_offset, &y_offset); /* The only thing that can change here is the width * and height, otherwise set_info would've been called */ @@ -551,11 +660,29 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass) GST_TYPE_COMPOSITOR_OPERATOR, DEFAULT_PAD_OPERATOR, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + /** + * GstCompositorPad:sizing-policy: + * + * Specifies sizing policy to use. Depending on selected sizing policy, + * scaled image might not fully cover the configured target rectangle area + * (e.g., "keep-aspect-ratio"). In that case, any uncovered area will be + * filled with background unless the uncovered area is drawn by other image. + * + * Since: 1.20 + */ + g_object_class_install_property (gobject_class, PROP_PAD_SIZING_POLICY, + g_param_spec_enum ("sizing-policy", "Sizing policy", + "Sizing policy to use for image scaling", + GST_TYPE_COMPOSITOR_SIZING_POLICY, DEFAULT_PAD_SIZING_POLICY, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + vaggpadclass->prepare_frame_start = GST_DEBUG_FUNCPTR (gst_compositor_pad_prepare_frame_start); vaggcpadclass->create_conversion_info = GST_DEBUG_FUNCPTR (gst_compositor_pad_create_conversion_info); + + gst_type_mark_as_plugin_api (GST_TYPE_COMPOSITOR_SIZING_POLICY, 0); } static void @@ -567,6 +694,7 @@ gst_compositor_pad_init (GstCompositorPad * compo_pad) compo_pad->op = DEFAULT_PAD_OPERATOR; compo_pad->width = DEFAULT_PAD_WIDTH; compo_pad->height = DEFAULT_PAD_HEIGHT; + compo_pad->sizing_policy = DEFAULT_PAD_SIZING_POLICY; } @@ -828,17 +956,22 @@ _fixate_caps (GstAggregator * agg, GstCaps * caps) gint width, height; gint fps_n, fps_d; gdouble cur_fps; + gint x_offset; + gint y_offset; fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info); fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info); _mixer_pad_get_output_size (GST_COMPOSITOR (vagg), compositor_pad, par_n, - par_d, &width, &height); + par_d, &width, &height, &x_offset, &y_offset); if (width == 0 || height == 0) continue; - this_width = width + MAX (compositor_pad->xpos, 0); - this_height = height + MAX (compositor_pad->ypos, 0); + /* {x,y}_offset represent padding size of each top and left area. + * To calculate total resolution, count bottom and right padding area + * as well here */ + this_width = width + MAX (compositor_pad->xpos + 2 * x_offset, 0); + this_height = height + MAX (compositor_pad->ypos + 2 * y_offset, 0); if (best_width < this_width) best_width = this_width; @@ -1178,7 +1311,8 @@ blend_pads (struct CompositeTask *comp) for (i = 0; i < comp->n_pads; i++) { composite (comp->pads_info[i].prepared_frame, - comp->pads_info[i].pad->xpos, comp->pads_info[i].pad->ypos, + comp->pads_info[i].pad->xpos + comp->pads_info[i].pad->x_offset, + comp->pads_info[i].pad->ypos + comp->pads_info[i].pad->y_offset, comp->pads_info[i].pad->alpha, comp->out_frame, comp->dst_line_start, comp->dst_line_end, comp->pads_info[i].blend_mode); } diff --git a/gst/compositor/compositor.h b/gst/compositor/compositor.h index 38e0693..2a6f38b 100644 --- a/gst/compositor/compositor.h +++ b/gst/compositor/compositor.h @@ -77,6 +77,23 @@ typedef enum COMPOSITOR_OPERATOR_ADD, } GstCompositorOperator; +/** + * GstCompositorSizingPolicy: + * @COMPOSITOR_SIZING_POLICY_NONE: Scaling image without padding + * @COMPOSITOR_SIZING_POLICY_KEEP_ASPECT_RATIO: Scaling image to fit destination + * resolution with preserving aspect ratio. Resulting image will be centered + * in the configured destination rectangle and it might have padding area + * if aspect ratio of destination rectangle is different from that of + * input image. + * + * Since: 1.20 + */ +typedef enum +{ + COMPOSITOR_SIZING_POLICY_NONE, + COMPOSITOR_SIZING_POLICY_KEEP_ASPECT_RATIO, +} GstCompositorSizingPolicy; + /* copied from video-converter.c */ typedef void (*GstParallelizedTaskFunc) (gpointer user_data); @@ -140,8 +157,14 @@ struct _GstCompositorPad gint xpos, ypos; gint width, height; gdouble alpha; + GstCompositorSizingPolicy sizing_policy; GstCompositorOperator op; + + /* offset values for xpos and ypos used when sizing-policy is equal to + * keep-aspect-ratio */ + gint x_offset; + gint y_offset; }; GST_ELEMENT_REGISTER_DECLARE (compositor); -- 2.7.4