From 806ff70590fb4629681b39c3c763a85f42284868 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 6 Jul 2017 14:26:21 -0400 Subject: [PATCH] compositor: Add support for crossfade blending Crossfading is a bit more complex than just having two pads with the right keyframes as the blending is not exactly the same. The difference is in the way we compute the alpha channel, in the case of crossfading, we have to compute an additive operation between the destination and the source (factored by the alpha property of both the input pad alpha property and the crossfading ratio) basically so that the crossfade result of 2 opaque frames is also fully opaque at any time in the crossfading process, avoid bleeding through the layer blending. Some rationnal can be found in https://phabricator.freedesktop.org/T7773. https://bugzilla.gnome.org/show_bug.cgi?id=784827 --- gst-libs/gst/video/gstvideoaggregator.c | 13 +- gst-libs/gst/video/gstvideoaggregatorpad.h | 7 +- gst/compositor/blend.c | 145 ++++++++++++--------- gst/compositor/blend.h | 24 +++- gst/compositor/compositor.c | 194 ++++++++++++++++++++++++----- gst/compositor/compositor.h | 5 +- gst/compositor/compositororc.orc | 144 ++++++++++++++++++++- gst/compositor/compositorpad.h | 3 + 8 files changed, 433 insertions(+), 102 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 0de0bc6..9864612 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -521,7 +521,18 @@ gst_video_aggregator_find_best_format (GstVideoAggregator * vagg, GINT_TO_POINTER (format_number)); /* If that pad is the first with alpha, set it as the new best format */ - if (!need_alpha && (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { + if (!need_alpha && (pad->ABI.needs_alpha + && (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (pad->info.finfo)))) { + need_alpha = TRUE; + /* Just fallback to ARGB in case we require alpha but the input pad + * does not have alpha. + * Do not increment best_format_number in that case. */ + gst_video_info_set_format (best_info, + GST_VIDEO_FORMAT_ARGB, + GST_VIDEO_INFO_HEIGHT (&pad->info), + GST_VIDEO_INFO_WIDTH (&pad->info)); + } else if (!need_alpha + && (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { need_alpha = TRUE; *best_info = pad->info; best_format_number = format_number; diff --git a/gst-libs/gst/video/gstvideoaggregatorpad.h b/gst-libs/gst/video/gstvideoaggregatorpad.h index 2a3ee44..c374731 100644 --- a/gst-libs/gst/video/gstvideoaggregatorpad.h +++ b/gst-libs/gst/video/gstvideoaggregatorpad.h @@ -70,7 +70,12 @@ struct _GstVideoAggregatorPad /* < private > */ GstVideoAggregatorPadPrivate *priv; - gpointer _gst_reserved[GST_PADDING]; + union { + /* Subclasses can force an alpha channel in the (input thus output) + * colorspace format */ + gboolean needs_alpha; + gpointer _gst_reserved[GST_PADDING]; + } ABI; }; /** diff --git a/gst/compositor/blend.c b/gst/compositor/blend.c index 721c766..b36c164 100644 --- a/gst/compositor/blend.c +++ b/gst/compositor/blend.c @@ -44,7 +44,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_compositor_blend_debug); #define BLEND_A32(name, method, LOOP) \ static void \ method##_ ##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ guint s_alpha; \ gint src_stride, dest_stride; \ @@ -89,24 +89,45 @@ method##_ ##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ if (src_height > 0 && src_width > 0) { \ dest = dest + 4 * xpos + (ypos * dest_stride); \ \ - LOOP (dest, src, src_height, src_width, src_stride, dest_stride, s_alpha); \ + LOOP (dest, src, src_height, src_width, src_stride, dest_stride, s_alpha, \ + mode); \ } \ } -#define BLEND_A32_LOOP(name, method) \ +#define OVERLAY_A32_LOOP(name) \ static inline void \ -_##method##_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ - gint src_width, gint src_stride, gint dest_stride, guint s_alpha) \ +_overlay_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ + gint src_width, gint src_stride, gint dest_stride, guint s_alpha, \ + GstCompositorBlendMode mode) \ { \ s_alpha = MIN (255, s_alpha); \ - compositor_orc_##method##_##name (dest, dest_stride, src, src_stride, \ - s_alpha, src_width, src_height); \ + switch (mode) { \ + case COMPOSITOR_BLEND_MODE_NORMAL:\ + compositor_orc_overlay_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + case COMPOSITOR_BLEND_MODE_ADDITIVE:\ + compositor_orc_overlay_##name##_addition (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + }\ } -BLEND_A32_LOOP (argb, blend); -BLEND_A32_LOOP (bgra, blend); -BLEND_A32_LOOP (argb, overlay); -BLEND_A32_LOOP (bgra, overlay); +#define BLEND_A32_LOOP_WITH_MODE(name) \ +static inline void \ +_blend_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ + gint src_width, gint src_stride, gint dest_stride, guint s_alpha, \ + GstCompositorBlendMode mode) \ +{ \ + s_alpha = MIN (255, s_alpha); \ + compositor_orc_blend_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ +} + +OVERLAY_A32_LOOP (argb); +OVERLAY_A32_LOOP (bgra); +BLEND_A32_LOOP_WITH_MODE (argb); +BLEND_A32_LOOP_WITH_MODE (bgra); #if G_BYTE_ORDER == G_LITTLE_ENDIAN BLEND_A32 (argb, blend, _blend_loop_argb); @@ -228,12 +249,12 @@ _blend_##format_name (const guint8 * src, guint8 * dest, \ \ b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ \ - BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, src_width, src_height); \ + BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, src_width, src_height);\ } \ \ static void \ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ const guint8 *b_src; \ guint8 *b_dest; \ @@ -478,7 +499,7 @@ _blend_##format_name (const guint8 * src, guint8 * dest, \ \ static void \ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ const guint8 *b_src; \ guint8 *b_dest; \ @@ -649,7 +670,7 @@ NV_YUV_FILL_CHECKER (nv21, memset); #define RGB_BLEND(name, bpp, MEMCPY, BLENDLOOP) \ static void \ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ gint b_alpha; \ gint i; \ @@ -815,7 +836,7 @@ RGB_FILL_COLOR (bgrx, 4, _memset_bgrx); #define PACKED_422_BLEND(name, MEMCPY, BLENDLOOP) \ static void \ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ gint b_alpha; \ gint i; \ @@ -1010,52 +1031,52 @@ gst_compositor_init_blend (void) GST_DEBUG_CATEGORY_INIT (gst_compositor_blend_debug, "compositor_blend", 0, "video compositor blending functions"); - gst_compositor_blend_argb = blend_argb; - gst_compositor_blend_bgra = blend_bgra; - gst_compositor_overlay_argb = overlay_argb; - gst_compositor_overlay_bgra = overlay_bgra; - gst_compositor_blend_i420 = blend_i420; - gst_compositor_blend_nv12 = blend_nv12; - gst_compositor_blend_nv21 = blend_nv21; - gst_compositor_blend_y444 = blend_y444; - gst_compositor_blend_y42b = blend_y42b; - gst_compositor_blend_y41b = blend_y41b; - gst_compositor_blend_rgb = blend_rgb; - gst_compositor_blend_xrgb = blend_xrgb; - gst_compositor_blend_yuy2 = blend_yuy2; + gst_compositor_blend_argb = GST_DEBUG_FUNCPTR (blend_argb); + gst_compositor_blend_bgra = GST_DEBUG_FUNCPTR (blend_bgra); + gst_compositor_overlay_argb = GST_DEBUG_FUNCPTR (overlay_argb); + gst_compositor_overlay_bgra = GST_DEBUG_FUNCPTR (overlay_bgra); + gst_compositor_blend_i420 = GST_DEBUG_FUNCPTR (blend_i420); + gst_compositor_blend_nv12 = GST_DEBUG_FUNCPTR (blend_nv12); + gst_compositor_blend_nv21 = GST_DEBUG_FUNCPTR (blend_nv21); + gst_compositor_blend_y444 = GST_DEBUG_FUNCPTR (blend_y444); + gst_compositor_blend_y42b = GST_DEBUG_FUNCPTR (blend_y42b); + gst_compositor_blend_y41b = GST_DEBUG_FUNCPTR (blend_y41b); + gst_compositor_blend_rgb = GST_DEBUG_FUNCPTR (blend_rgb); + gst_compositor_blend_xrgb = GST_DEBUG_FUNCPTR (blend_xrgb); + gst_compositor_blend_yuy2 = GST_DEBUG_FUNCPTR (blend_yuy2); - gst_compositor_fill_checker_argb = fill_checker_argb_c; - gst_compositor_fill_checker_bgra = fill_checker_bgra_c; - gst_compositor_fill_checker_ayuv = fill_checker_ayuv_c; - gst_compositor_fill_checker_i420 = fill_checker_i420; - gst_compositor_fill_checker_nv12 = fill_checker_nv12; - gst_compositor_fill_checker_nv21 = fill_checker_nv21; - gst_compositor_fill_checker_y444 = fill_checker_y444; - gst_compositor_fill_checker_y42b = fill_checker_y42b; - gst_compositor_fill_checker_y41b = fill_checker_y41b; - gst_compositor_fill_checker_rgb = fill_checker_rgb_c; - gst_compositor_fill_checker_xrgb = fill_checker_xrgb_c; - gst_compositor_fill_checker_yuy2 = fill_checker_yuy2_c; - gst_compositor_fill_checker_uyvy = fill_checker_uyvy_c; + gst_compositor_fill_checker_argb = GST_DEBUG_FUNCPTR (fill_checker_argb_c); + gst_compositor_fill_checker_bgra = GST_DEBUG_FUNCPTR (fill_checker_bgra_c); + gst_compositor_fill_checker_ayuv = GST_DEBUG_FUNCPTR (fill_checker_ayuv_c); + gst_compositor_fill_checker_i420 = GST_DEBUG_FUNCPTR (fill_checker_i420); + gst_compositor_fill_checker_nv12 = GST_DEBUG_FUNCPTR (fill_checker_nv12); + gst_compositor_fill_checker_nv21 = GST_DEBUG_FUNCPTR (fill_checker_nv21); + gst_compositor_fill_checker_y444 = GST_DEBUG_FUNCPTR (fill_checker_y444); + gst_compositor_fill_checker_y42b = GST_DEBUG_FUNCPTR (fill_checker_y42b); + gst_compositor_fill_checker_y41b = GST_DEBUG_FUNCPTR (fill_checker_y41b); + gst_compositor_fill_checker_rgb = GST_DEBUG_FUNCPTR (fill_checker_rgb_c); + gst_compositor_fill_checker_xrgb = GST_DEBUG_FUNCPTR (fill_checker_xrgb_c); + gst_compositor_fill_checker_yuy2 = GST_DEBUG_FUNCPTR (fill_checker_yuy2_c); + gst_compositor_fill_checker_uyvy = GST_DEBUG_FUNCPTR (fill_checker_uyvy_c); - gst_compositor_fill_color_argb = fill_color_argb; - gst_compositor_fill_color_bgra = fill_color_bgra; - gst_compositor_fill_color_abgr = fill_color_abgr; - gst_compositor_fill_color_rgba = fill_color_rgba; - gst_compositor_fill_color_ayuv = fill_color_ayuv; - gst_compositor_fill_color_i420 = fill_color_i420; - gst_compositor_fill_color_yv12 = fill_color_yv12; - gst_compositor_fill_color_nv12 = fill_color_nv12; - gst_compositor_fill_color_y444 = fill_color_y444; - gst_compositor_fill_color_y42b = fill_color_y42b; - gst_compositor_fill_color_y41b = fill_color_y41b; - gst_compositor_fill_color_rgb = fill_color_rgb_c; - gst_compositor_fill_color_bgr = fill_color_bgr_c; - gst_compositor_fill_color_xrgb = fill_color_xrgb; - gst_compositor_fill_color_xbgr = fill_color_xbgr; - gst_compositor_fill_color_rgbx = fill_color_rgbx; - gst_compositor_fill_color_bgrx = fill_color_bgrx; - gst_compositor_fill_color_yuy2 = fill_color_yuy2; - gst_compositor_fill_color_yvyu = fill_color_yvyu; - gst_compositor_fill_color_uyvy = fill_color_uyvy; + gst_compositor_fill_color_argb = GST_DEBUG_FUNCPTR (fill_color_argb); + gst_compositor_fill_color_bgra = GST_DEBUG_FUNCPTR (fill_color_bgra); + gst_compositor_fill_color_abgr = GST_DEBUG_FUNCPTR (fill_color_abgr); + gst_compositor_fill_color_rgba = GST_DEBUG_FUNCPTR (fill_color_rgba); + gst_compositor_fill_color_ayuv = GST_DEBUG_FUNCPTR (fill_color_ayuv); + gst_compositor_fill_color_i420 = GST_DEBUG_FUNCPTR (fill_color_i420); + gst_compositor_fill_color_yv12 = GST_DEBUG_FUNCPTR (fill_color_yv12); + gst_compositor_fill_color_nv12 = GST_DEBUG_FUNCPTR (fill_color_nv12); + gst_compositor_fill_color_y444 = GST_DEBUG_FUNCPTR (fill_color_y444); + gst_compositor_fill_color_y42b = GST_DEBUG_FUNCPTR (fill_color_y42b); + gst_compositor_fill_color_y41b = GST_DEBUG_FUNCPTR (fill_color_y41b); + gst_compositor_fill_color_rgb = GST_DEBUG_FUNCPTR (fill_color_rgb_c); + gst_compositor_fill_color_bgr = GST_DEBUG_FUNCPTR (fill_color_bgr_c); + gst_compositor_fill_color_xrgb = GST_DEBUG_FUNCPTR (fill_color_xrgb); + gst_compositor_fill_color_xbgr = GST_DEBUG_FUNCPTR (fill_color_xbgr); + gst_compositor_fill_color_rgbx = GST_DEBUG_FUNCPTR (fill_color_rgbx); + gst_compositor_fill_color_bgrx = GST_DEBUG_FUNCPTR (fill_color_bgrx); + gst_compositor_fill_color_yuy2 = GST_DEBUG_FUNCPTR (fill_color_yuy2); + gst_compositor_fill_color_yvyu = GST_DEBUG_FUNCPTR (fill_color_yvyu); + gst_compositor_fill_color_uyvy = GST_DEBUG_FUNCPTR (fill_color_uyvy); } diff --git a/gst/compositor/blend.h b/gst/compositor/blend.h index 1cc127c..34d5230 100644 --- a/gst/compositor/blend.h +++ b/gst/compositor/blend.h @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2009 Sebastian Dröge * * This library is free software; you can redistribute it and/or @@ -23,7 +23,21 @@ #include #include -typedef void (*BlendFunction) (GstVideoFrame *srcframe, gint xpos, gint ypos, gdouble src_alpha, GstVideoFrame * destframe); +/** + * GstCompositorBlendMode: + * @COMPOSITOR_BLEND_MODE_NORMAL: Normal blending + * @COMPOSITOR_BLEND_MODE_ADDITIVE: Alphas are simply added, + * + * The different modes compositor can use for blending. + */ +typedef enum +{ + COMPOSITOR_BLEND_MODE_NORMAL, + COMPOSITOR_BLEND_MODE_ADDITIVE, +} GstCompositorBlendMode; + +typedef void (*BlendFunction) (GstVideoFrame *srcframe, gint xpos, gint ypos, gdouble src_alpha, GstVideoFrame * destframe, + GstCompositorBlendMode mode); typedef void (*FillCheckerFunction) (GstVideoFrame * frame); typedef void (*FillColorFunction) (GstVideoFrame * frame, gint c1, gint c2, gint c3); @@ -32,10 +46,16 @@ extern BlendFunction gst_compositor_blend_bgra; #define gst_compositor_blend_ayuv gst_compositor_blend_argb #define gst_compositor_blend_abgr gst_compositor_blend_argb #define gst_compositor_blend_rgba gst_compositor_blend_bgra +#define gst_compositor_blend_ayuv_addition gst_compositor_blend_argb_addition +#define gst_compositor_blend_abgr_addition gst_compositor_blend_argb_addition +#define gst_compositor_blend_rgba_addition gst_compositor_blend_bgra_addition + + extern BlendFunction gst_compositor_overlay_argb; extern BlendFunction gst_compositor_overlay_bgra; #define gst_compositor_overlay_ayuv gst_compositor_overlay_argb #define gst_compositor_overlay_abgr gst_compositor_overlay_argb +#define gst_compositor_overlay_abgr gst_compositor_overlay_argb #define gst_compositor_overlay_rgba gst_compositor_overlay_bgra extern BlendFunction gst_compositor_blend_i420; #define gst_compositor_blend_yv12 gst_compositor_blend_i420 diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 23dc20c..dceca34 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -125,6 +125,7 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", #define DEFAULT_PAD_WIDTH 0 #define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_ALPHA 1.0 +#define DEFAULT_PAD_CROSSFADE_RATIO -1.0 enum { PROP_PAD_0, @@ -132,7 +133,8 @@ enum PROP_PAD_YPOS, PROP_PAD_WIDTH, PROP_PAD_HEIGHT, - PROP_PAD_ALPHA + PROP_PAD_ALPHA, + PROP_PAD_CROSSFADE_RATIO, }; G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad, @@ -160,6 +162,9 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: g_value_set_double (value, pad->alpha); break; + case PROP_PAD_CROSSFADE_RATIO: + g_value_set_double (value, pad->crossfade); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -188,6 +193,10 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: pad->alpha = g_value_get_double (value); break; + case PROP_PAD_CROSSFADE_RATIO: + pad->crossfade = g_value_get_double (value); + GST_VIDEO_AGGREGATOR_PAD (pad)->ABI.needs_alpha = pad->crossfade >= 0.0f; + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -475,11 +484,20 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, } GST_OBJECT_LOCK (vagg); + /* Check if we are crossfading the pad one way or another */ + l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad); + if ((l->prev && GST_COMPOSITOR_PAD (l->prev->data)->crossfade >= 0.0) || + (GST_COMPOSITOR_PAD (pad)->crossfade >= 0.0)) { + GST_DEBUG_OBJECT (pad, "Is being crossfaded with previous pad"); + l = NULL; + } else { + l = l->next; + } + /* Check if this frame is obscured by a higher-zorder frame * TODO: Also skip a frame if it's obscured by a combination of * higher-zorder frames */ - for (l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad)->next; l; - l = l->next) { + for (; l; l = l->next) { GstVideoRectangle frame2_rect; GstVideoAggregatorPad *pad2 = l->data; GstCompositorPad *cpad2 = GST_COMPOSITOR_PAD (pad2); @@ -621,6 +639,12 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_CROSSFADE_RATIO, + g_param_spec_double ("crossfade-ratio", "Crossfade ratio", + "The crossfade ratio to use while crossfading with the following pad." + "A value inferior to 0 means no crossfading.", + -1.0, 1.0, DEFAULT_PAD_CROSSFADE_RATIO, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); vaggpadclass->set_info = GST_DEBUG_FUNCPTR (gst_compositor_pad_set_info); vaggpadclass->prepare_frame = @@ -635,6 +659,7 @@ gst_compositor_pad_init (GstCompositorPad * compo_pad) compo_pad->xpos = DEFAULT_PAD_XPOS; compo_pad->ypos = DEFAULT_PAD_YPOS; compo_pad->alpha = DEFAULT_PAD_ALPHA; + compo_pad->crossfade = DEFAULT_PAD_CROSSFADE_RATIO; } @@ -643,7 +668,7 @@ gst_compositor_pad_init (GstCompositorPad * compo_pad) enum { PROP_0, - PROP_BACKGROUND + PROP_BACKGROUND, }; #define GST_TYPE_COMPOSITOR_BACKGROUND (gst_compositor_background_get_type()) @@ -963,6 +988,124 @@ _negotiated_caps (GstAggregator * agg, GstCaps * caps) return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); } +/* Fills frame with transparent pixels if @nframe is NULL otherwise copy @frame + * properties and fill @nframes with transparent pixels */ +static GstFlowReturn +gst_compositor_fill_transparent (GstCompositor * self, GstVideoFrame * frame, + GstVideoFrame * nframe) +{ + guint plane, num_planes, height, i; + + if (nframe) { + GstBuffer *cbuffer = gst_buffer_copy_deep (frame->buffer); + + if (!gst_video_frame_map (nframe, &frame->info, cbuffer, GST_MAP_WRITE)) { + GST_WARNING_OBJECT (self, "Could not map output buffer"); + return GST_FLOW_ERROR; + } + } else { + nframe = frame; + } + + num_planes = GST_VIDEO_FRAME_N_PLANES (nframe); + for (plane = 0; plane < num_planes; ++plane) { + guint8 *pdata; + gsize rowsize, plane_stride; + + pdata = GST_VIDEO_FRAME_PLANE_DATA (nframe, plane); + plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (nframe, plane); + rowsize = GST_VIDEO_FRAME_COMP_WIDTH (nframe, plane) + * GST_VIDEO_FRAME_COMP_PSTRIDE (nframe, plane); + height = GST_VIDEO_FRAME_COMP_HEIGHT (nframe, plane); + for (i = 0; i < height; ++i) { + memset (pdata, 0, rowsize); + pdata += plane_stride; + } + } + + return GST_FLOW_OK; +} + +/* WITH GST_OBJECT_LOCK !! + * Returns: %TRUE if outframe is allready ready to be used as we are using + * a transparent background and all pads have already been crossfaded + * %FALSE otherwise + */ +static gboolean +gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe) +{ + GList *l; + gboolean all_crossfading = FALSE; + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (self); + + if (self->background == COMPOSITOR_BACKGROUND_TRANSPARENT) { + + all_crossfading = TRUE; + for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { + GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (l->data); + + if (compo_pad->crossfade < 0.0 && l->next && + GST_COMPOSITOR_PAD (l->next->data)->crossfade < 0) { + all_crossfading = FALSE; + + break; + } + } + } + + for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + + if (compo_pad->crossfade >= 0.0f && pad->aggregated_frame) { + gfloat alpha = compo_pad->crossfade * compo_pad->alpha; + GstVideoAggregatorPad *npad = l->next ? l->next->data : NULL; + GstVideoFrame *nframe; + + if (!all_crossfading) { + nframe = g_slice_new0 (GstVideoFrame); + gst_compositor_fill_transparent (self, outframe, nframe); + } else { + nframe = outframe; + } + + self->overlay (pad->aggregated_frame, + compo_pad->crossfaded ? 0 : compo_pad->xpos, + compo_pad->crossfaded ? 0 : compo_pad->ypos, + alpha, nframe, COMPOSITOR_BLEND_MODE_ADDITIVE); + + if (npad && npad->aggregated_frame) { + GstCompositorPad *next_compo_pad = GST_COMPOSITOR_PAD (npad); + + alpha = (1.0 - compo_pad->crossfade) * next_compo_pad->alpha; + self->overlay (npad->aggregated_frame, next_compo_pad->xpos, + next_compo_pad->ypos, alpha, nframe, + COMPOSITOR_BLEND_MODE_ADDITIVE); + + /* Replace frame with current frame */ + gst_compositor_pad_clean_frame (npad, vagg); + npad->aggregated_frame = !all_crossfading ? nframe : NULL; + next_compo_pad->crossfaded = TRUE; + + /* Frame is now consumed, clean it up */ + gst_compositor_pad_clean_frame (pad, vagg); + pad->aggregated_frame = NULL; + } else { + GST_LOG_OBJECT (self, "Simply fading out as no following pad found"); + gst_compositor_pad_clean_frame (pad, vagg); + pad->aggregated_frame = !all_crossfading ? nframe : NULL; + compo_pad->crossfaded = TRUE; + } + } + } + + if (all_crossfading) + for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) + GST_COMPOSITOR_PAD (l->data)->crossfaded = FALSE; + + return all_crossfading; +} + static GstFlowReturn gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) { @@ -992,39 +1135,26 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) self->fill_color (outframe, 240, 128, 128); break; case COMPOSITOR_BACKGROUND_TRANSPARENT: - { - guint i, plane, num_planes, height; - - num_planes = GST_VIDEO_FRAME_N_PLANES (outframe); - for (plane = 0; plane < num_planes; ++plane) { - guint8 *pdata; - gsize rowsize, plane_stride; - - pdata = GST_VIDEO_FRAME_PLANE_DATA (outframe, plane); - plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (outframe, plane); - rowsize = GST_VIDEO_FRAME_COMP_WIDTH (outframe, plane) - * GST_VIDEO_FRAME_COMP_PSTRIDE (outframe, plane); - height = GST_VIDEO_FRAME_COMP_HEIGHT (outframe, plane); - for (i = 0; i < height; ++i) { - memset (pdata, 0, rowsize); - pdata += plane_stride; - } - } - + gst_compositor_fill_transparent (self, outframe, NULL); /* use overlay to keep background transparent */ composite = self->overlay; break; - } } GST_OBJECT_LOCK (vagg); - for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = l->data; - GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); - - if (pad->aggregated_frame != NULL) { - composite (pad->aggregated_frame, compo_pad->xpos, compo_pad->ypos, - compo_pad->alpha, outframe); + /* First mix the crossfade frames as required */ + if (!gst_compositor_crossfade_frames (self, outframe)) { + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + + if (pad->aggregated_frame != NULL) { + composite (pad->aggregated_frame, + compo_pad->crossfaded ? 0 : compo_pad->xpos, + compo_pad->crossfaded ? 0 : compo_pad->ypos, compo_pad->alpha, + outframe, COMPOSITOR_BLEND_MODE_NORMAL); + compo_pad->crossfaded = FALSE; + } } } GST_OBJECT_UNLOCK (vagg); @@ -1112,8 +1242,8 @@ gst_compositor_class_init (GstCompositorClass * klass) static void gst_compositor_init (GstCompositor * self) { - self->background = DEFAULT_BACKGROUND; /* initialize variables */ + self->background = DEFAULT_BACKGROUND; } /* Element registration */ diff --git a/gst/compositor/compositor.h b/gst/compositor/compositor.h index a6b4d9f..d8cefa3 100644 --- a/gst/compositor/compositor.h +++ b/gst/compositor/compositor.h @@ -17,7 +17,7 @@ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ - + #ifndef __GST_COMPOSITOR_H__ #define __GST_COMPOSITOR_H__ @@ -57,8 +57,7 @@ typedef enum COMPOSITOR_BACKGROUND_BLACK, COMPOSITOR_BACKGROUND_WHITE, COMPOSITOR_BACKGROUND_TRANSPARENT, -} -GstCompositorBackground; +} GstCompositorBackground; /** * GstCompositor: diff --git a/gst/compositor/compositororc.orc b/gst/compositor/compositororc.orc index 234ec21..66c4dcc 100644 --- a/gst/compositor/compositororc.orc +++ b/gst/compositor/compositororc.orc @@ -98,7 +98,6 @@ x4 convwb t, d_wide orl t, t, a_alpha storel d, t - .function compositor_orc_overlay_argb .flags 2d .dest 4 d guint8 @@ -159,6 +158,76 @@ andl a, a, a_alpha orl t, t, a storel d, t + +.function compositor_orc_overlay_argb_addition +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 2 tw +.temp 1 tb +.temp 8 alpha_s +.temp 8 alpha_s_inv +.temp 8 alpha_factor +.temp 8 alpha_d +.temp 4 a +.temp 8 d_wide +.temp 8 s_wide +.const 4 xfs 0xffffffff +.const 4 a_alpha 0x000000ff +.const 4 a_alpha_inv 0xffffff00 + +# calc source alpha as alpha_s = alpha_s * alpha / 255 +loadl t, s +convlw tw, t +convwb tb, tw +splatbl a, tb +x4 convubw alpha_s, a +x4 mullw alpha_s, alpha_s, alpha +x4 div255w alpha_s, alpha_s +x4 convubw s_wide, t +x4 mullw s_wide, s_wide, alpha_s + +# calc destination alpha as alpha_factor = (255-alpha_s) * alpha_factor / factor +loadpl a, xfs +x4 convubw alpha_s_inv, a +x4 subw alpha_s_inv, alpha_s_inv, alpha_s +loadl t, d +convlw tw, t +convwb tb, tw +splatbl a, tb +x4 convubw alpha_factor, a +x4 mullw alpha_factor, alpha_factor, alpha_s_inv +x4 div255w alpha_factor, alpha_factor +x4 convubw d_wide, t +x4 mullw d_wide, d_wide, alpha_factor + +# calc final pixel as pix_d = pix_s*alpha_s + pix_d*alpha_factor*(255-alpha_s)/255 +x4 addw d_wide, d_wide, s_wide + +# calc the alpha factor alpha_factor = alpha_s + alpha_factor * (255-alpha_s)/255 +x4 addw alpha_factor, alpha_factor, alpha_s + +# now normalize the pix_d by the final alpha to make it associative +x4 divluw, d_wide, d_wide, alpha_factor + +# calc the final global alpha_d = alpha_d + (alpha_s * (alpha / 255)) +loadl t, d +convlw tw, t +convwb tb, tw +splatbl a, tb +x4 convubw alpha_d, a +x4 addw alpha_d, alpha_d, alpha_s + +# pack the new alpha into the correct spot +x4 convwb t, d_wide +andl t, t, a_alpha_inv +x4 convwb a, alpha_d +andl a, a, a_alpha +orl t, t, a +storel d, t + .function compositor_orc_overlay_bgra .flags 2d .dest 4 d guint8 @@ -221,3 +290,76 @@ x4 convwb a, alpha_d andl a, a, a_alpha orl t, t, a storel d, t + +.function compositor_orc_overlay_bgra_addition +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 4 t2 +.temp 2 tw +.temp 1 tb +.temp 8 alpha_s +.temp 8 alpha_s_inv +.temp 8 alpha_factor +.temp 8 alpha_d +.temp 4 a +.temp 8 d_wide +.temp 8 s_wide +.const 4 xfs 0xffffffff +.const 4 a_alpha 0xff000000 +.const 4 a_alpha_inv 0x00ffffff + +# calc source alpha as alpha_s = alpha_s * alpha / 255 +loadl t, s +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw alpha_s, a +x4 mullw alpha_s, alpha_s, alpha +x4 div255w alpha_s, alpha_s +x4 convubw s_wide, t +x4 mullw s_wide, s_wide, alpha_s + +# calc destination alpha as alpha_factor = (255-alpha_s) * alpha_factor / 255 +loadpl a, xfs +x4 convubw alpha_s_inv, a +x4 subw alpha_s_inv, alpha_s_inv, alpha_s +loadl t, d +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw alpha_factor, a +x4 mullw alpha_factor, alpha_factor, alpha_s_inv +x4 div255w alpha_factor, alpha_factor +x4 convubw d_wide, t +x4 mullw d_wide, d_wide, alpha_factor + +# calc final pixel as pix_d = pix_s*alpha_s + pix_d*alpha_factor*(255-alpha_s)/255 +x4 addw d_wide, d_wide, s_wide + +# calc the final destination alpha_factor = alpha_s + alpha_factor * (255-alpha_s)/255 +x4 addw alpha_factor, alpha_factor, alpha_s + +# now normalize the pix_d by the final alpha to make it associative +x4 divluw, d_wide, d_wide, alpha_factor + +# calc the final global alpha_d = alpha_d + (alpha_s * (alpha / 255)) +loadl t, d +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw alpha_d, a +x4 addw alpha_d, alpha_d, alpha_s + +# pack the new alpha into the correct spot +x4 convwb t, d_wide +andl t, t, a_alpha_inv +x4 convwb a, alpha_d +andl a, a, a_alpha +orl t, t, a +storel d, t diff --git a/gst/compositor/compositorpad.h b/gst/compositor/compositorpad.h index cb6f7c7..1c2a271 100644 --- a/gst/compositor/compositorpad.h +++ b/gst/compositor/compositorpad.h @@ -52,10 +52,13 @@ struct _GstCompositorPad gint xpos, ypos; gint width, height; gdouble alpha; + gdouble crossfade; GstVideoConverter *convert; GstVideoInfo conversion_info; GstBuffer *converted_buffer; + + gboolean crossfaded; }; struct _GstCompositorPadClass -- 2.7.4