From aae25e003280897b05594aaa63d5101c972c7369 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 27 Oct 2018 16:44:27 +0100 Subject: [PATCH] compositor: Implement different operators via per-pad property This removes the crossfade-ratio property and replaces it with an operator property. Currently this implements the following operators: - SOURCE: Copy over the source and don't look at the destination - OVER: Default blending of the source over the destination - ADD: Like OVER but simply adding the alpha instead See the example for how to implement crossfading with this. https://bugzilla.gnome.org/show_bug.cgi?id=797169 --- gst/compositor/blend.c | 64 +++++++-- gst/compositor/blend.h | 15 +- gst/compositor/compositor.c | 251 +++++++++++----------------------- gst/compositor/compositor.h | 27 +++- gst/compositor/compositororc.orc | 60 ++++++++ tests/examples/compositor/crossfade.c | 38 +++-- 6 files changed, 237 insertions(+), 218 deletions(-) diff --git a/gst/compositor/blend.c b/gst/compositor/blend.c index 416f78c..c54d898 100644 --- a/gst/compositor/blend.c +++ b/gst/compositor/blend.c @@ -100,32 +100,46 @@ _overlay_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ { \ s_alpha = MIN (255, s_alpha); \ switch (mode) { \ - case COMPOSITOR_BLEND_MODE_NORMAL:\ + case COMPOSITOR_BLEND_MODE_SOURCE:\ + compositor_orc_source_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + case COMPOSITOR_BLEND_MODE_OVER:\ compositor_orc_overlay_##name (dest, dest_stride, src, src_stride, \ s_alpha, src_width, src_height); \ break;\ - case COMPOSITOR_BLEND_MODE_ADDITIVE:\ + case COMPOSITOR_BLEND_MODE_ADD:\ compositor_orc_overlay_##name##_addition (dest, dest_stride, src, src_stride, \ s_alpha, src_width, src_height); \ break;\ }\ } -#define BLEND_A32_LOOP_WITH_MODE(name) \ +#define BLEND_A32_LOOP(name) \ static inline void \ _blend_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ gint src_width, gint src_stride, gint dest_stride, guint s_alpha, \ GstCompositorBlendMode mode) \ { \ s_alpha = MIN (255, s_alpha); \ - compositor_orc_blend_##name (dest, dest_stride, src, src_stride, \ - s_alpha, src_width, src_height); \ + switch (mode) { \ + case COMPOSITOR_BLEND_MODE_SOURCE:\ + compositor_orc_source_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + case COMPOSITOR_BLEND_MODE_OVER:\ + case COMPOSITOR_BLEND_MODE_ADD:\ + /* both modes are the same for opaque background */ \ + compositor_orc_blend_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + }\ } OVERLAY_A32_LOOP (argb); OVERLAY_A32_LOOP (bgra); -BLEND_A32_LOOP_WITH_MODE (argb); -BLEND_A32_LOOP_WITH_MODE (bgra); +BLEND_A32_LOOP (argb); +BLEND_A32_LOOP (bgra); #if G_BYTE_ORDER == G_LITTLE_ENDIAN BLEND_A32 (argb, blend, _blend_loop_argb); @@ -223,11 +237,16 @@ A32_COLOR (ayuv, FALSE, 24, 16, 8, 0); inline static void \ _blend_##format_name (const guint8 * src, guint8 * dest, \ gint src_stride, gint dest_stride, gint src_width, gint src_height, \ - gdouble src_alpha) \ + gdouble src_alpha, GstCompositorBlendMode mode) \ { \ gint i; \ gint b_alpha; \ \ + /* in source mode we just have to copy over things */ \ + if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \ + src_alpha = 1.0; \ + } \ + \ /* If it's completely transparent... we just return */ \ if (G_UNLIKELY (src_alpha == 0.0)) { \ GST_INFO ("Fast copy (alpha == 0.0)"); \ @@ -324,7 +343,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ \ b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 1); \ b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 1); \ @@ -340,7 +359,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ \ b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 2); \ b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 2); \ @@ -356,7 +375,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ } #define PLANAR_YUV_FILL_CHECKER(format_name, format_enum, MEMSET) \ @@ -468,11 +487,16 @@ PLANAR_YUV_FILL_COLOR (y41b, GST_VIDEO_FORMAT_Y41B, memset); inline static void \ _blend_##format_name (const guint8 * src, guint8 * dest, \ gint src_stride, gint dest_stride, gint src_width, gint src_height, \ - gdouble src_alpha) \ + gdouble src_alpha, GstCompositorBlendMode mode) \ { \ gint i; \ gint b_alpha; \ \ + /* in source mode we just have to copy over things */ \ + if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \ + src_alpha = 1.0; \ + } \ + \ /* If it's completely transparent... we just return */ \ if (G_UNLIKELY (src_alpha == 0.0)) { \ GST_INFO ("Fast copy (alpha == 0.0)"); \ @@ -569,7 +593,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ \ b_src = GST_VIDEO_FRAME_PLANE_DATA (srcframe, 1); \ b_dest = GST_VIDEO_FRAME_PLANE_DATA (destframe, 1); \ @@ -585,7 +609,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos * 2 + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, 2 * src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ } #define NV_YUV_FILL_CHECKER(format_name, MEMSET) \ @@ -711,6 +735,12 @@ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ } \ \ dest = dest + bpp * xpos + (ypos * dest_stride); \ + \ + /* in source mode we just have to copy over things */ \ + if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \ + src_alpha = 1.0; \ + } \ + \ /* If it's completely transparent... we just return */ \ if (G_UNLIKELY (src_alpha == 0.0)) { \ GST_INFO ("Fast copy (alpha == 0.0)"); \ @@ -880,6 +910,12 @@ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ } \ \ dest = dest + 2 * xpos + (ypos * dest_stride); \ + \ + /* in source mode we just have to copy over things */ \ + if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \ + src_alpha = 1.0; \ + } \ + \ /* If it's completely transparent... we just return */ \ if (G_UNLIKELY (src_alpha == 0.0)) { \ GST_INFO ("Fast copy (alpha == 0.0)"); \ diff --git a/gst/compositor/blend.h b/gst/compositor/blend.h index 34d5230..710ec7d 100644 --- a/gst/compositor/blend.h +++ b/gst/compositor/blend.h @@ -25,15 +25,17 @@ /** * GstCompositorBlendMode: - * @COMPOSITOR_BLEND_MODE_NORMAL: Normal blending - * @COMPOSITOR_BLEND_MODE_ADDITIVE: Alphas are simply added, + * @COMPOSITOR_BLEND_MODE_SOURCE: Copy source + * @COMPOSITOR_BLEND_MODE_OVER: Normal blending + * @COMPOSITOR_BLEND_MODE_ADD: Alphas are simply added, * * The different modes compositor can use for blending. */ typedef enum { - COMPOSITOR_BLEND_MODE_NORMAL, - COMPOSITOR_BLEND_MODE_ADDITIVE, + COMPOSITOR_BLEND_MODE_SOURCE, + COMPOSITOR_BLEND_MODE_OVER, + COMPOSITOR_BLEND_MODE_ADD, } GstCompositorBlendMode; typedef void (*BlendFunction) (GstVideoFrame *srcframe, gint xpos, gint ypos, gdouble src_alpha, GstVideoFrame * destframe, @@ -46,16 +48,11 @@ extern BlendFunction gst_compositor_blend_bgra; #define gst_compositor_blend_ayuv gst_compositor_blend_argb #define gst_compositor_blend_abgr gst_compositor_blend_argb #define gst_compositor_blend_rgba gst_compositor_blend_bgra -#define gst_compositor_blend_ayuv_addition gst_compositor_blend_argb_addition -#define gst_compositor_blend_abgr_addition gst_compositor_blend_argb_addition -#define gst_compositor_blend_rgba_addition gst_compositor_blend_bgra_addition - extern BlendFunction gst_compositor_overlay_argb; extern BlendFunction gst_compositor_overlay_bgra; #define gst_compositor_overlay_ayuv gst_compositor_overlay_argb #define gst_compositor_overlay_abgr gst_compositor_overlay_argb -#define gst_compositor_overlay_abgr gst_compositor_overlay_argb #define gst_compositor_overlay_rgba gst_compositor_overlay_bgra extern BlendFunction gst_compositor_blend_i420; #define gst_compositor_blend_yv12 gst_compositor_blend_i420 diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index d036d68..453a92e 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -122,12 +122,32 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", static void gst_compositor_child_proxy_init (gpointer g_iface, gpointer iface_data); +#define GST_TYPE_COMPOSITOR_OPERATOR (gst_compositor_operator_get_type()) +static GType +gst_compositor_operator_get_type (void) +{ + static GType compositor_operator_type = 0; + + static const GEnumValue compositor_operator[] = { + {COMPOSITOR_OPERATOR_SOURCE, "Source", "source"}, + {COMPOSITOR_OPERATOR_OVER, "Over", "over"}, + {COMPOSITOR_OPERATOR_ADD, "Add", "add"}, + {0, NULL, NULL}, + }; + + if (!compositor_operator_type) { + compositor_operator_type = + g_enum_register_static ("GstCompositorOperator", compositor_operator); + } + return compositor_operator_type; +} + #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 #define DEFAULT_PAD_WIDTH 0 #define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_ALPHA 1.0 -#define DEFAULT_PAD_CROSSFADE_RATIO 0.0 +#define DEFAULT_PAD_OPERATOR COMPOSITOR_OPERATOR_OVER enum { PROP_PAD_0, @@ -136,7 +156,7 @@ enum PROP_PAD_WIDTH, PROP_PAD_HEIGHT, PROP_PAD_ALPHA, - PROP_PAD_CROSSFADE_RATIO, + PROP_PAD_OPERATOR, }; G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad, @@ -164,8 +184,8 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: g_value_set_double (value, pad->alpha); break; - case PROP_PAD_CROSSFADE_RATIO: - g_value_set_double (value, pad->crossfade); + case PROP_PAD_OPERATOR: + g_value_set_enum (value, pad->op); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -199,10 +219,10 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: pad->alpha = g_value_get_double (value); break; - case PROP_PAD_CROSSFADE_RATIO: - pad->crossfade = g_value_get_double (value); + case PROP_PAD_OPERATOR: + pad->op = g_value_get_enum (value); gst_video_aggregator_pad_set_needs_alpha (GST_VIDEO_AGGREGATOR_PAD (pad), - pad->crossfade > 0.0); + pad->op == COMPOSITOR_OPERATOR_ADD); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -319,7 +339,7 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height); if (cpad->alpha == 0.0) { - GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); + GST_DEBUG_OBJECT (pad, "Pad has alpha 0.0, not converting frame"); goto done; } @@ -327,25 +347,16 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GST_VIDEO_INFO_WIDTH (&vagg->info), GST_VIDEO_INFO_HEIGHT (&vagg->info)); if (frame_rect.w == 0 || frame_rect.h == 0) { - GST_DEBUG_OBJECT (vagg, "Resulting frame is zero-width or zero-height " + GST_DEBUG_OBJECT (pad, "Resulting frame is zero-width or zero-height " "(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h); goto done; } GST_OBJECT_LOCK (vagg); - /* Check if we are crossfading the pad one way or another */ - l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad); - if ((l->prev && GST_COMPOSITOR_PAD (l->prev->data)->crossfade > 0.0) || - (GST_COMPOSITOR_PAD (pad)->crossfade > 0.0)) { - GST_DEBUG_OBJECT (pad, "Is being crossfaded with previous pad"); - l = NULL; - } else { - l = l->next; - } - /* Check if this frame is obscured by a higher-zorder frame * TODO: Also skip a frame if it's obscured by a combination of * higher-zorder frames */ + l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad)->next; for (; l; l = l->next) { GstVideoRectangle frame2_rect; GstVideoAggregatorPad *pad2 = l->data; @@ -468,10 +479,10 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_PAD_CROSSFADE_RATIO, - g_param_spec_double ("crossfade-ratio", "Crossfade ratio", - "The crossfade ratio to use while crossfading with the following pad", - 0.0, 1.0, DEFAULT_PAD_CROSSFADE_RATIO, + g_object_class_install_property (gobject_class, PROP_PAD_OPERATOR, + g_param_spec_enum ("operator", "Operator", + "Blending operator to use for blending this pad over the previous ones", + GST_TYPE_COMPOSITOR_OPERATOR, DEFAULT_PAD_OPERATOR, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); vaggpadclass->prepare_frame = @@ -487,7 +498,7 @@ gst_compositor_pad_init (GstCompositorPad * compo_pad) compo_pad->xpos = DEFAULT_PAD_XPOS; compo_pad->ypos = DEFAULT_PAD_YPOS; compo_pad->alpha = DEFAULT_PAD_ALPHA; - compo_pad->crossfade = DEFAULT_PAD_CROSSFADE_RATIO; + compo_pad->op = DEFAULT_PAD_OPERATOR; } @@ -818,137 +829,6 @@ _negotiated_caps (GstAggregator * agg, GstCaps * caps) return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); } -/* Fills frame with transparent pixels if @nframe is NULL otherwise copy @frame - * properties and fill @nframes with transparent pixels */ -static GstFlowReturn -gst_compositor_fill_transparent (GstCompositor * self, GstVideoFrame * frame, - GstVideoFrame * nframe) -{ - guint plane, num_planes, height, i; - - if (nframe) { - GstBuffer *cbuffer = gst_buffer_copy_deep (frame->buffer); - - if (!gst_video_frame_map (nframe, &frame->info, cbuffer, GST_MAP_WRITE)) { - GST_WARNING_OBJECT (self, "Could not map output buffer"); - gst_buffer_unref (cbuffer); - return GST_FLOW_ERROR; - } - - /* the last reference is owned by the frame and released once the frame - * is unmapped. We leak it if we don't unref here */ - gst_buffer_unref (cbuffer); - } else { - nframe = frame; - } - - num_planes = GST_VIDEO_FRAME_N_PLANES (nframe); - for (plane = 0; plane < num_planes; ++plane) { - guint8 *pdata; - gsize rowsize, plane_stride; - - pdata = GST_VIDEO_FRAME_PLANE_DATA (nframe, plane); - plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (nframe, plane); - rowsize = GST_VIDEO_FRAME_COMP_WIDTH (nframe, plane) - * GST_VIDEO_FRAME_COMP_PSTRIDE (nframe, plane); - height = GST_VIDEO_FRAME_COMP_HEIGHT (nframe, plane); - for (i = 0; i < height; ++i) { - memset (pdata, 0, rowsize); - pdata += plane_stride; - } - } - - return GST_FLOW_OK; -} - -/* WITH GST_OBJECT_LOCK !! - * Returns: %TRUE if outframe is allready ready to be used as we are using - * a transparent background and all pads have already been crossfaded - * %FALSE otherwise - */ -static gboolean -gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe) -{ - GList *l; - gboolean all_crossfading = FALSE; - GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (self); - - if (self->background == COMPOSITOR_BACKGROUND_TRANSPARENT) { - - all_crossfading = TRUE; - for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { - GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (l->data); - - if (compo_pad->crossfade == 0.0 && l->next && - GST_COMPOSITOR_PAD (l->next->data)->crossfade == 0.0) { - all_crossfading = FALSE; - - break; - } - } - } - - for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = l->data; - GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); - GstVideoAggregatorPadClass *pad_class = - GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (compo_pad); - GstVideoFrame *prepared_frame = - gst_video_aggregator_pad_get_prepared_frame (pad); - - if (compo_pad->crossfade > 0.0 && prepared_frame) { - gfloat alpha = compo_pad->crossfade * compo_pad->alpha; - GstVideoAggregatorPad *npad = l->next ? l->next->data : NULL; - GstVideoFrame *next_prepared_frame; - GstVideoFrame nframe; - - next_prepared_frame = - npad ? gst_video_aggregator_pad_get_prepared_frame (npad) : NULL; - - if (!all_crossfading) { - gst_compositor_fill_transparent (self, outframe, &nframe); - } else { - nframe = *outframe; - } - - self->overlay (prepared_frame, - compo_pad->crossfaded ? 0 : compo_pad->xpos, - compo_pad->crossfaded ? 0 : compo_pad->ypos, - alpha, &nframe, COMPOSITOR_BLEND_MODE_ADDITIVE); - - if (npad && next_prepared_frame) { - GstCompositorPad *next_compo_pad = GST_COMPOSITOR_PAD (npad); - - alpha = (1.0 - compo_pad->crossfade) * next_compo_pad->alpha; - self->overlay (next_prepared_frame, next_compo_pad->xpos, - next_compo_pad->ypos, alpha, &nframe, - COMPOSITOR_BLEND_MODE_ADDITIVE); - - /* Replace frame with current frame */ - pad_class->clean_frame (npad, vagg, next_prepared_frame); - if (!all_crossfading) - *next_prepared_frame = nframe; - next_compo_pad->crossfaded = TRUE; - - /* Frame is now consumed, clean it up */ - pad_class->clean_frame (pad, vagg, prepared_frame); - } else { - GST_LOG_OBJECT (self, "Simply fading out as no following pad found"); - pad_class->clean_frame (pad, vagg, prepared_frame); - if (!all_crossfading) - *prepared_frame = nframe; - compo_pad->crossfaded = TRUE; - } - } - } - - if (all_crossfading) - for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) - GST_COMPOSITOR_PAD (l->data)->crossfaded = FALSE; - - return all_crossfading; -} - static GstFlowReturn gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) { @@ -978,28 +858,57 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) self->fill_color (outframe, 240, 128, 128); break; case COMPOSITOR_BACKGROUND_TRANSPARENT: - gst_compositor_fill_transparent (self, outframe, NULL); + { + guint i, plane, num_planes, height; + + num_planes = GST_VIDEO_FRAME_N_PLANES (outframe); + for (plane = 0; plane < num_planes; ++plane) { + guint8 *pdata; + gsize rowsize, plane_stride; + + pdata = GST_VIDEO_FRAME_PLANE_DATA (outframe, plane); + plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (outframe, plane); + rowsize = GST_VIDEO_FRAME_COMP_WIDTH (outframe, plane) + * GST_VIDEO_FRAME_COMP_PSTRIDE (outframe, plane); + height = GST_VIDEO_FRAME_COMP_HEIGHT (outframe, plane); + for (i = 0; i < height; ++i) { + memset (pdata, 0, rowsize); + pdata += plane_stride; + } + } /* use overlay to keep background transparent */ composite = self->overlay; break; + } } GST_OBJECT_LOCK (vagg); - /* First mix the crossfade frames as required */ - if (!gst_compositor_crossfade_frames (self, outframe)) { - for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = l->data; - GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); - GstVideoFrame *prepared_frame = - gst_video_aggregator_pad_get_prepared_frame (pad); - - if (prepared_frame != NULL) { - composite (prepared_frame, - compo_pad->crossfaded ? 0 : compo_pad->xpos, - compo_pad->crossfaded ? 0 : compo_pad->ypos, compo_pad->alpha, - outframe, COMPOSITOR_BLEND_MODE_NORMAL); - compo_pad->crossfaded = FALSE; - } + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + GstVideoFrame *prepared_frame = + gst_video_aggregator_pad_get_prepared_frame (pad); + GstCompositorBlendMode blend_mode = COMPOSITOR_BLEND_MODE_OVER; + + switch (compo_pad->op) { + case COMPOSITOR_OPERATOR_SOURCE: + blend_mode = COMPOSITOR_OPERATOR_SOURCE; + break; + case COMPOSITOR_OPERATOR_OVER: + blend_mode = COMPOSITOR_OPERATOR_OVER; + break; + case COMPOSITOR_OPERATOR_ADD: + blend_mode = COMPOSITOR_OPERATOR_ADD; + break; + default: + g_assert_not_reached (); + break; + } + + if (prepared_frame != NULL) { + composite (prepared_frame, + compo_pad->xpos, + compo_pad->ypos, compo_pad->alpha, outframe, blend_mode); } } GST_OBJECT_UNLOCK (vagg); diff --git a/gst/compositor/compositor.h b/gst/compositor/compositor.h index 37ffcb5..5e8d90c 100644 --- a/gst/compositor/compositor.h +++ b/gst/compositor/compositor.h @@ -56,7 +56,7 @@ typedef struct _GstCompositorPad GstCompositorPad; typedef struct _GstCompositorPadClass GstCompositorPadClass; /** - * GstcompositorBackground: + * GstCompositorBackground: * @COMPOSITOR_BACKGROUND_CHECKER: checker pattern background * @COMPOSITOR_BACKGROUND_BLACK: solid color black background * @COMPOSITOR_BACKGROUND_WHITE: solid color white background @@ -73,6 +73,28 @@ typedef enum } GstCompositorBackground; /** + * GstCompositorOperator: + * @COMPOSITOR_OPERATOR_SOURCE: Copy the source over the destination, + * without the destination pixels. + * @COMPOSITOR_OPERATOR_OVER: Blend the source over the destination. + * @COMPOSITOR_OPERATOR_ADD: Similar to over but add the source and + * destination alpha. Requires output with alpha + * channel. + * + * The different blending operators that can be used by compositor. + * + * See https://www.cairographics.org/operators/ for some explanation and + * visualizations. + * + */ +typedef enum +{ + COMPOSITOR_OPERATOR_SOURCE, + COMPOSITOR_OPERATOR_OVER, + COMPOSITOR_OPERATOR_ADD, +} GstCompositorOperator; + +/** * GstCompositor: * * The opaque #GstCompositor structure. @@ -105,9 +127,8 @@ struct _GstCompositorPad gint xpos, ypos; gint width, height; gdouble alpha; - gdouble crossfade; - gboolean crossfaded; + GstCompositorOperator op; }; struct _GstCompositorPadClass diff --git a/gst/compositor/compositororc.orc b/gst/compositor/compositororc.orc index 66c4dcc..fea01a4 100644 --- a/gst/compositor/compositororc.orc +++ b/gst/compositor/compositororc.orc @@ -62,6 +62,36 @@ x4 convwb t, d_wide orl t, t, a_alpha storel d, t +.function compositor_orc_source_argb +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 4 t2 +.temp 2 tw +.temp 1 tb +.temp 4 a +.temp 8 a_wide +.const 4 a_alpha 0x000000ff +.const 4 a_not_alpha 0xffffff00 + +loadl t, s +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw a_wide, a +x4 mullw a_wide, a_wide, alpha +x4 div255w a_wide, a_wide + +andl t, t, a_not_alpha +x4 convwb t2, a_wide +andl t2, t2, a_alpha +orl t, t, t2 + +storel d, t + .function compositor_orc_blend_bgra .flags 2d .dest 4 d guint8 @@ -98,6 +128,36 @@ x4 convwb t, d_wide orl t, t, a_alpha storel d, t +.function compositor_orc_source_bgra +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 4 t2 +.temp 2 tw +.temp 1 tb +.temp 4 a +.temp 8 a_wide +.const 4 a_alpha 0xff000000 +.const 4 a_not_alpha 0x00ffffff + +loadl t, s +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw a_wide, a +x4 mullw a_wide, a_wide, alpha +x4 div255w a_wide, a_wide + +andl t, t, a_not_alpha +x4 convwb t2, a_wide +andl t2, t2, a_alpha +orl t, t, t2 + +storel d, t + .function compositor_orc_overlay_argb .flags 2d .dest 4 d guint8 diff --git a/tests/examples/compositor/crossfade.c b/tests/examples/compositor/crossfade.c index 44090ff..503503c 100644 --- a/tests/examples/compositor/crossfade.c +++ b/tests/examples/compositor/crossfade.c @@ -21,8 +21,7 @@ /** * Simple crossfade example using the compositor element. * - * Takes a list of uri/path to video files and crossfade them - * for 10 seconds and returns. + * Takes two video files and crossfades them for 10 seconds and returns. */ #include @@ -34,7 +33,6 @@ typedef struct { GstElement *compositor; guint z_order; - gboolean is_last; } VideoInfo; static gchar * @@ -51,24 +49,23 @@ _pad_added_cb (GstElement * decodebin, GstPad * pad, VideoInfo * info) { GstPad *sinkpad = gst_element_get_request_pad (GST_ELEMENT (info->compositor), "sink_%u"); + GstControlSource *control_source; + gboolean is_last = info->z_order == 1; - if (!info->is_last) { - GstControlSource *control_source; + control_source = gst_interpolation_control_source_new (); - control_source = gst_interpolation_control_source_new (); + gst_util_set_object_arg (G_OBJECT (sinkpad), "operator", + info->z_order == 0 ? "source" : "add"); + gst_object_add_control_binding (GST_OBJECT (sinkpad), + gst_direct_control_binding_new_absolute (GST_OBJECT (sinkpad), "alpha", + control_source)); - g_object_set (sinkpad, "crossfade-ratio", 1.0, NULL); - gst_object_add_control_binding (GST_OBJECT (sinkpad), - gst_direct_control_binding_new_absolute (GST_OBJECT (sinkpad), - "crossfade-ratio", control_source)); + g_object_set (control_source, "mode", GST_INTERPOLATION_MODE_LINEAR, NULL); - g_object_set (control_source, "mode", GST_INTERPOLATION_MODE_LINEAR, NULL); - - gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE - (control_source), 0, 1.0); - gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE - (control_source), 10 * GST_SECOND, 0.0); - } + gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE + (control_source), 0, is_last ? 0.0 : 1.0); + gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE + (control_source), 10 * GST_SECOND, is_last ? 1.0 : 0.0); g_object_set (sinkpad, "zorder", info->z_order, NULL); gst_pad_link (pad, sinkpad); @@ -84,8 +81,8 @@ main (int argc, char *argv[]) GstElement *compositor, *sink, *pipeline; GstBus *bus; - if (argc < 2) { - g_error ("At least 1 valid video file paths/urls need to " "be provided"); + if (argc != 3) { + g_error ("Need to provide 2 input videos"); return -1; } @@ -101,7 +98,7 @@ main (int argc, char *argv[]) gst_bin_add_many (GST_BIN (pipeline), compositor, sink, NULL); g_assert (gst_element_link (compositor, sink)); - for (i = 1; i < argc; i++) { + for (i = 1; i < 3; i++) { gchar *uri = ensure_uri (argv[i]); VideoInfo *info = g_malloc0 (sizeof (VideoInfo)); GstElement *uridecodebin = gst_element_factory_make ("uridecodebin", NULL); @@ -111,7 +108,6 @@ main (int argc, char *argv[]) info->compositor = compositor; info->z_order = i - 1; - info->is_last = (i == (argc - 1)) && (argc > 2); g_signal_connect (uridecodebin, "pad-added", (GCallback) _pad_added_cb, info); -- 2.7.4