videomixer2: Add transparent background option for alpha channel formats

This option allows the videomixer2 element to output a valid alpha
channel when the inputs contain a valid alpha channel. This allows
mixing to occur in multiple stages serially.

The following pipeline shows an example of such a pipeline:

gst-launch videotestsrc background-color=0x000000 pattern=ball ! video/x-raw-yuv,format=\(fourcc\)AYUV ! videomixer2 background=transparent name=mix1 ! videomixer2 name=mix2 ! ffmpegcolorspace ! autovideosink  videotestsrc ! video/x-raw-yuv,format=\(fourcc\)AYUV ! mix2.

The first videotestsrc in this pipeline creates a moving ball on a
transparent background. It is then passed to the first videomixer2.
Previously, this videomixer2 would have forced the alpha channel to
1.0 and given a background of checker, black, or white to the
stream. With this patch, however, you can now specify the background
as transparent, and the alpha channel of the input will be
preserved. This allows for further mixing downstream, as is shown in
the above pipeline where the a second videomixer2 is used to mix in a
background of an smpte videotestsrc. So the result is a ball hovering
over the smpte test source. This could, of course, have been
accomplished with a single mixer element, but staged mixing is useful
when it is not convenient to mix all video at once (e.g. a pipeline
where a foreground and background bin exist and are mixed at the final
output, but the foreground bin needs an internal mixer to create
transitions between clips).

Fixes bug #639994.

diff --git a/gst/videomixer/blend.c b/gst/videomixer/blend.c
index 9ca6168..e2d736f 100644 (file)
--- a/gst/videomixer/blend.c
+++ b/gst/videomixer/blend.c
@@ -41,9 +41,9 @@ GST_DEBUG_CATEGORY_STATIC (gst_videomixer_blend_debug);
 /* Below are the implementations of everything */
 
 /* A32 is for AYUV, ARGB and BGRA */
-#define BLEND_A32(name, LOOP) \
+#define BLEND_A32(name, method, LOOP)          \
 static void \
-blend_##name (const guint8 * src, gint xpos, gint ypos, \
+method##_ ##name (const guint8 * src, gint xpos, gint ypos, \
     gint src_width, gint src_height, gdouble src_alpha, \
     guint8 * dest, gint dest_width, gint dest_height) \
 { \
@@ -83,25 +83,31 @@ blend_##name (const guint8 * src, gint xpos, gint ypos, \
   LOOP (dest, src, src_height, src_width, src_stride, dest_stride, s_alpha); \
 }
 
-#define BLEND_A32_LOOP(name) \
+#define BLEND_A32_LOOP(name, method)                   \
 static inline void \
-_blend_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \
+_##method##_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \
     gint src_width, gint src_stride, gint dest_stride, guint s_alpha) \
 { \
   s_alpha = MIN (255, s_alpha); \
-  orc_blend_##name (dest, dest_stride, src, src_stride, \
+  orc_##method##_##name (dest, dest_stride, src, src_stride, \
       s_alpha, src_width, src_height); \
 }
 
-BLEND_A32_LOOP (argb);
-BLEND_A32_LOOP (bgra);
+BLEND_A32_LOOP (argb, blend);
+BLEND_A32_LOOP (bgra, blend);
+BLEND_A32_LOOP (argb, overlay);
+BLEND_A32_LOOP (bgra, overlay);
 
 #if G_BYTE_ORDER == LITTLE_ENDIAN
-BLEND_A32 (argb, _blend_loop_argb);
-BLEND_A32 (bgra, _blend_loop_bgra);
+BLEND_A32 (argb, blend, _blend_loop_argb);
+BLEND_A32 (bgra, blend, _blend_loop_bgra);
+BLEND_A32 (argb, overlay, _overlay_loop_argb);
+BLEND_A32 (bgra, overlay, _overlay_loop_bgra);
 #else
-BLEND_A32 (argb, _blend_loop_bgra);
-BLEND_A32 (bgra, _blend_loop_argb);
+BLEND_A32 (argb, blend, _blend_loop_bgra);
+BLEND_A32 (bgra, blend, _blend_loop_argb);
+BLEND_A32 (argb, overlay, _overlay_loop_bgra);
+BLEND_A32 (bgra, overlay, _overlay_loop_argb);
 #endif
 
 #define A32_CHECKER_C(name, RGB, A, C1, C2, C3) \
@@ -666,6 +672,8 @@ PACKED_422_FILL_COLOR (uyvy, 16, 24, 0, 8);
 /* Init function */
 BlendFunction gst_video_mixer_blend_argb;
 BlendFunction gst_video_mixer_blend_bgra;
+BlendFunction gst_video_mixer_overlay_argb;
+BlendFunction gst_video_mixer_overlay_bgra;
 /* AYUV/ABGR is equal to ARGB, RGBA is equal to BGRA */
 BlendFunction gst_video_mixer_blend_y444;
 BlendFunction gst_video_mixer_blend_y42b;
@@ -724,6 +732,8 @@ gst_video_mixer_init_blend (void)
 
   gst_video_mixer_blend_argb = blend_argb;
   gst_video_mixer_blend_bgra = blend_bgra;
+  gst_video_mixer_overlay_argb = overlay_argb;
+  gst_video_mixer_overlay_bgra = overlay_bgra;
   gst_video_mixer_blend_i420 = blend_i420;
   gst_video_mixer_blend_y444 = blend_y444;
   gst_video_mixer_blend_y42b = blend_y42b;

diff --git a/gst/videomixer/blend.h b/gst/videomixer/blend.h
index 4f770cb..ef60c91 100644 (file)
--- a/gst/videomixer/blend.h
+++ b/gst/videomixer/blend.h
@@ -31,6 +31,11 @@ extern BlendFunction gst_video_mixer_blend_bgra;
 #define gst_video_mixer_blend_ayuv gst_video_mixer_blend_argb
 #define gst_video_mixer_blend_abgr gst_video_mixer_blend_argb
 #define gst_video_mixer_blend_rgba gst_video_mixer_blend_bgra
+extern BlendFunction gst_video_mixer_overlay_argb;
+extern BlendFunction gst_video_mixer_overlay_bgra;
+#define gst_video_mixer_overlay_ayuv gst_video_mixer_overlay_argb
+#define gst_video_mixer_overlay_abgr gst_video_mixer_overlay_argb
+#define gst_video_mixer_overlay_rgba gst_video_mixer_overlay_bgra
 extern BlendFunction gst_video_mixer_blend_i420;
 #define gst_video_mixer_blend_yv12 gst_video_mixer_blend_i420
 extern BlendFunction gst_video_mixer_blend_y41b;

diff --git a/gst/videomixer/blendorc.orc b/gst/videomixer/blendorc.orc
index bb4601c..a284513 100644 (file)
--- a/gst/videomixer/blendorc.orc
+++ b/gst/videomixer/blendorc.orc
@@ -95,3 +95,128 @@ x4 convwb t, d_wide
 orl t, t, a_alpha
 storel d, t
 
+
+.function orc_overlay_argb
+.flags 2d
+.dest 4 d guint8
+.source 4 s guint8
+.param 2 alpha
+.temp 4 t
+.temp 2 tw
+.temp 1 tb
+.temp 8 alpha_s
+.temp 8 alpha_s_inv
+.temp 8 alpha_d
+.temp 4 a
+.temp 8 d_wide
+.temp 8 s_wide
+.const 4 xfs 0xffffffff
+.const 4 a_alpha 0x000000ff
+.const 4 a_alpha_inv 0xffffff00
+
+# calc source alpha as alpha_s = alpha_s * alpha / 256
+loadl t, s
+convlw tw, t
+convwb tb, tw
+splatbl a, tb
+x4 convubw alpha_s, a
+x4 mullw alpha_s, alpha_s, alpha
+x4 shruw alpha_s, alpha_s, 8
+x4 convubw s_wide, t
+x4 mullw s_wide, s_wide, alpha_s
+
+# calc destination alpha as alpha_d = (255-alpha_s) * alpha_d / 255
+loadpl a, xfs
+x4 convubw alpha_s_inv, a
+x4 subw alpha_s_inv, alpha_s_inv, alpha_s
+loadl t, d
+convlw tw, t
+convwb tb, tw
+convubw tw, tb
+splatbl a, tb
+x4 convubw alpha_d, a
+x4 mullw alpha_d, alpha_d, alpha_s_inv
+x4 div255w alpha_d, alpha_d
+x4 convubw d_wide, t
+x4 mullw d_wide, d_wide, alpha_d
+
+# calc final pixel as pix_d = pix_s*alpha_s + pix_d*alpha_d*(255-alpha_s)/255
+x4 addw d_wide, d_wide, s_wide
+
+# calc the final destination alpha_d = alpha_s + alpha_d * (255-alpha_s)/255
+x4 addw alpha_d, alpha_d, alpha_s
+
+# now normalize the pix_d by the final alpha to make it associative
+x4 divluw, d_wide, d_wide, alpha_d
+
+# pack the new alpha into the correct spot
+x4 convwb t, d_wide
+andl t, t, a_alpha_inv
+x4 convwb a, alpha_d
+andl a, a, a_alpha
+orl  t, t, a
+storel d, t
+
+.function orc_overlay_bgra
+.flags 2d
+.dest 4 d guint8
+.source 4 s guint8
+.param 2 alpha
+.temp 4 t
+.temp 4 t2
+.temp 2 tw
+.temp 1 tb
+.temp 8 alpha_s
+.temp 8 alpha_s_inv
+.temp 8 alpha_d
+.temp 4 a
+.temp 8 d_wide
+.temp 8 s_wide
+.const 4 xfs 0xffffffff
+.const 4 a_alpha 0xff000000
+.const 4 a_alpha_inv 0x00ffffff
+
+# calc source alpha as alpha_s = alpha_s * alpha / 256
+loadl t, s
+shrul t2, t, 24
+convlw tw, t
+convwb tb, tw
+splatbl a, tb
+x4 convubw alpha_s, a
+x4 mullw alpha_s, alpha_s, alpha
+x4 shruw alpha_s, alpha_s, 8
+x4 convubw s_wide, t
+x4 mullw s_wide, s_wide, alpha_s
+
+# calc destination alpha as alpha_d = (255-alpha_s) * alpha_d / 255
+loadpl a, xfs
+x4 convubw alpha_s_inv, a
+x4 subw alpha_s_inv, alpha_s_inv, alpha_s
+loadl t, d
+shrul t2, t, 24
+convlw tw, t
+convwb tb, tw
+convubw tw, tb
+splatbl a, tb
+x4 convubw alpha_d, a
+x4 mullw alpha_d, alpha_d, alpha_s_inv
+x4 div255w alpha_d, alpha_d
+x4 convubw d_wide, t
+x4 mullw d_wide, d_wide, alpha_d
+
+# calc final pixel as pix_d = pix_s*alpha_s + pix_d*alpha_d*(255-alpha_s)/255
+x4 addw d_wide, d_wide, s_wide
+
+# calc the final destination alpha_d = alpha_s + alpha_d * (255-alpha_s)/255
+x4 addw alpha_d, alpha_d, alpha_s
+
+# now normalize the pix_d by the final alpha to make it associative
+x4 divluw, d_wide, d_wide, alpha_d
+
+# pack the new alpha into the correct spot
+x4 convwb t, d_wide
+andl t, t, a_alpha_inv
+x4 convwb a, alpha_d
+andl a, a, a_alpha
+orl  t, t, a
+storel d, t

diff --git a/gst/videomixer/videomixer2.c b/gst/videomixer/videomixer2.c
index 16e5a35..049f0af 100644 (file)
--- a/gst/videomixer/videomixer2.c
+++ b/gst/videomixer/videomixer2.c
@@ -546,6 +546,8 @@ gst_videomixer2_background_get_type (void)
     {VIDEO_MIXER2_BACKGROUND_CHECKER, "Checker pattern", "checker"},
     {VIDEO_MIXER2_BACKGROUND_BLACK, "Black", "black"},
     {VIDEO_MIXER2_BACKGROUND_WHITE, "White", "white"},
+    {VIDEO_MIXER2_BACKGROUND_TRANSPARENT,
+        "Transparent Background to enable further mixing", "transparent"},
     {0, NULL, NULL},
   };
 
@@ -810,6 +812,7 @@ gst_videomixer2_blend_buffers (GstVideoMixer2 * mix,
   GSList *l;
   GstFlowReturn ret;
   guint outsize;
+  BlendFunction composite;
 
   outsize = gst_video_format_get_size (mix->format, mix->width, mix->height);
   ret = gst_pad_alloc_buffer_and_set_caps (mix->srcpad, GST_BUFFER_OFFSET_NONE,
@@ -820,6 +823,8 @@ gst_videomixer2_blend_buffers (GstVideoMixer2 * mix,
   GST_BUFFER_TIMESTAMP (*outbuf) = output_start_time;
   GST_BUFFER_DURATION (*outbuf) = output_end_time - output_start_time;
 
+  /* default to blending */
+  composite = mix->blend;
   switch (mix->background) {
     case VIDEO_MIXER2_BACKGROUND_CHECKER:
       mix->fill_checker (GST_BUFFER_DATA (*outbuf), mix->width, mix->height);
@@ -832,6 +837,13 @@ gst_videomixer2_blend_buffers (GstVideoMixer2 * mix,
       mix->fill_color (GST_BUFFER_DATA (*outbuf), mix->width,
           mix->height, 240, 128, 128);
       break;
+    case VIDEO_MIXER2_BACKGROUND_TRANSPARENT:
+      memset (GST_BUFFER_DATA (*outbuf), 0,
+          gst_video_format_get_row_stride (mix->format, 0,
+              mix->width) * mix->height);
+      /* use overlay to keep background transparent */
+      composite = mix->overlay;
+      break;
   }
 
   for (l = mix->sinkpads; l; l = l->next) {
@@ -854,7 +866,7 @@ gst_videomixer2_blend_buffers (GstVideoMixer2 * mix,
       if (GST_CLOCK_TIME_IS_VALID (stream_time))
         gst_object_sync_values (G_OBJECT (pad), stream_time);
 
-      mix->blend (GST_BUFFER_DATA (mixcol->buffer),
+      composite (GST_BUFFER_DATA (mixcol->buffer),
           pad->xpos, pad->ypos, pad->width, pad->height, pad->alpha,
           GST_BUFFER_DATA (*outbuf), mix->width, mix->height);
     }
@@ -1387,6 +1399,7 @@ gst_videomixer2_src_setcaps (GstPad * pad, GstCaps * caps)
   GST_INFO_OBJECT (pad, "set src caps: %" GST_PTR_FORMAT, caps);
 
   mix->blend = NULL;
+  mix->overlay = NULL;
   mix->fill_checker = NULL;
   mix->fill_color = NULL;
 
@@ -1416,114 +1429,133 @@ gst_videomixer2_src_setcaps (GstPad * pad, GstCaps * caps)
   switch (mix->format) {
     case GST_VIDEO_FORMAT_AYUV:
       mix->blend = gst_video_mixer_blend_ayuv;
+      mix->overlay = gst_video_mixer_overlay_ayuv;
       mix->fill_checker = gst_video_mixer_fill_checker_ayuv;
       mix->fill_color = gst_video_mixer_fill_color_ayuv;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_ARGB:
       mix->blend = gst_video_mixer_blend_argb;
+      mix->overlay = gst_video_mixer_overlay_argb;
       mix->fill_checker = gst_video_mixer_fill_checker_argb;
       mix->fill_color = gst_video_mixer_fill_color_argb;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_BGRA:
       mix->blend = gst_video_mixer_blend_bgra;
+      mix->overlay = gst_video_mixer_overlay_bgra;
       mix->fill_checker = gst_video_mixer_fill_checker_bgra;
       mix->fill_color = gst_video_mixer_fill_color_bgra;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_ABGR:
       mix->blend = gst_video_mixer_blend_abgr;
+      mix->overlay = gst_video_mixer_overlay_abgr;
       mix->fill_checker = gst_video_mixer_fill_checker_abgr;
       mix->fill_color = gst_video_mixer_fill_color_abgr;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_RGBA:
       mix->blend = gst_video_mixer_blend_rgba;
+      mix->overlay = gst_video_mixer_overlay_rgba;
       mix->fill_checker = gst_video_mixer_fill_checker_rgba;
       mix->fill_color = gst_video_mixer_fill_color_rgba;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_Y444:
       mix->blend = gst_video_mixer_blend_y444;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_y444;
       mix->fill_color = gst_video_mixer_fill_color_y444;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_Y42B:
       mix->blend = gst_video_mixer_blend_y42b;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_y42b;
       mix->fill_color = gst_video_mixer_fill_color_y42b;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_YUY2:
       mix->blend = gst_video_mixer_blend_yuy2;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_yuy2;
       mix->fill_color = gst_video_mixer_fill_color_yuy2;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_UYVY:
       mix->blend = gst_video_mixer_blend_uyvy;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_uyvy;
       mix->fill_color = gst_video_mixer_fill_color_uyvy;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_YVYU:
       mix->blend = gst_video_mixer_blend_yvyu;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_yvyu;
       mix->fill_color = gst_video_mixer_fill_color_yvyu;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_I420:
       mix->blend = gst_video_mixer_blend_i420;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_i420;
       mix->fill_color = gst_video_mixer_fill_color_i420;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_YV12:
       mix->blend = gst_video_mixer_blend_yv12;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_yv12;
       mix->fill_color = gst_video_mixer_fill_color_yv12;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_Y41B:
       mix->blend = gst_video_mixer_blend_y41b;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_y41b;
       mix->fill_color = gst_video_mixer_fill_color_y41b;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_RGB:
       mix->blend = gst_video_mixer_blend_rgb;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_rgb;
       mix->fill_color = gst_video_mixer_fill_color_rgb;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_BGR:
       mix->blend = gst_video_mixer_blend_bgr;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_bgr;
       mix->fill_color = gst_video_mixer_fill_color_bgr;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_xRGB:
       mix->blend = gst_video_mixer_blend_xrgb;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_xrgb;
       mix->fill_color = gst_video_mixer_fill_color_xrgb;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_xBGR:
       mix->blend = gst_video_mixer_blend_xbgr;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_xbgr;
       mix->fill_color = gst_video_mixer_fill_color_xbgr;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_RGBx:
       mix->blend = gst_video_mixer_blend_rgbx;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_rgbx;
       mix->fill_color = gst_video_mixer_fill_color_rgbx;
       ret = TRUE;
       break;
     case GST_VIDEO_FORMAT_BGRx:
       mix->blend = gst_video_mixer_blend_bgrx;
+      mix->overlay = mix->blend;
       mix->fill_checker = gst_video_mixer_fill_checker_bgrx;
       mix->fill_color = gst_video_mixer_fill_color_bgrx;
       ret = TRUE;
@@ -1599,7 +1631,7 @@ gst_videomixer2_sink_event (GstCollectPads2 * pads, GstCollectData2 * cdata,
   GST_DEBUG_OBJECT (pad, "Got %s event on pad %s:%s",
       GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad));
 
-  // return FALSE => event will be forwarded
+  /* return FALSE => event will be forwarded */
   switch (GST_EVENT_TYPE (event)) {
     case GST_EVENT_NEWSEGMENT:{
       GstFormat fmt;

diff --git a/gst/videomixer/videomixer2.h b/gst/videomixer/videomixer2.h
index 14a82fe..2f2339d 100644 (file)
--- a/gst/videomixer/videomixer2.h
+++ b/gst/videomixer/videomixer2.h
@@ -47,6 +47,7 @@ typedef struct _GstVideoMixer2Class GstVideoMixer2Class;
  * @VIDEO_MIXER2_BACKGROUND_CHECKER: checker pattern background
  * @VIDEO_MIXER2_BACKGROUND_BLACK: solid color black background
  * @VIDEO_MIXER2_BACKGROUND_WHITE: solid color white background
+ * @VIDEO_MIXER2_BACKGROUND_TRANSPARENT: background is left transparent and layers are composited using "A OVER B" composition rules. This is only applicable to AYUV and ARGB (and variants) as it preserves the alpha channel and allows for further mixing.
  *
  * The different backgrounds videomixer can blend over.
  */
@@ -54,7 +55,8 @@ typedef enum
 {
   VIDEO_MIXER2_BACKGROUND_CHECKER,
   VIDEO_MIXER2_BACKGROUND_BLACK,
-  VIDEO_MIXER2_BACKGROUND_WHITE
+  VIDEO_MIXER2_BACKGROUND_WHITE,
+  VIDEO_MIXER2_BACKGROUND_TRANSPARENT,
 }
 GstVideoMixer2Background;
 
@@ -106,7 +108,7 @@ struct _GstVideoMixer2
   GstClockTime earliest_time;
   guint64 qos_processed, qos_dropped;
 
-  BlendFunction blend;
+  BlendFunction blend, overlay;
   FillCheckerFunction fill_checker;
   FillColorFunction fill_color;
 };