compositor/blend: fix blending of subsampled components

The correct way to determine the byte offset at a certain yoffset
in a subsampled component is to shift the yoffset by the component's
hsub

This fixes out-of-bounds memory accesses and visible artefacts,
example pipeline with the samples from #802:

gst-launch-1.0 compositor name=vmixer sink_1::xpos=1910 sink_1::ypos=1080 ! \
    videoconvert ! videorate ! xvimagesink \
  filesrc location=VID_20200723_203606.mp4 ! decodebin name=demux1 ! \
    queue ! videoflip method=vertical-flip ! vmixer. \
  filesrc location=bridgeoverstubbledwater.mp4 ! decodebin name=demux2 ! \
    queue ! vmixer.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/988>

diff --git a/gst/compositor/blend.c b/gst/compositor/blend.c
index c2c0150..bd6573d 100644 (file)
--- a/gst/compositor/blend.c
+++ b/gst/compositor/blend.c
@@ -383,7 +383,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \
   comp_xpos = (xpos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 1, xpos); \
   comp_ypos = (ypos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 1, ypos); \
   comp_xoffset = (xoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 1, xoffset); \
-  comp_yoffset = (yoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 1, yoffset); \
+  comp_yoffset = (yoffset == 0) ? 0 : yoffset >> info->h_sub[1]; \
   _blend_##format_name (b_src + comp_xoffset + comp_yoffset * src_comp_rowstride, \
       b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \
       src_comp_rowstride, \
@@ -399,7 +399,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \
   comp_xpos = (xpos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 2, xpos); \
   comp_ypos = (ypos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 2, ypos); \
   comp_xoffset = (xoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 2, xoffset); \
-  comp_yoffset = (yoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 2, yoffset); \
+  comp_yoffset = (yoffset == 0) ? 0 : yoffset >> info->h_sub[2]; \
   _blend_##format_name (b_src + comp_xoffset + comp_yoffset * src_comp_rowstride, \
       b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \
       src_comp_rowstride, \