omxvideoenc: include vertical padding in nFilledLen when copying

According to the OMX spec (3.1.3.7.1) nFilledLen is meant to include any
padding. We use to include the horizontal one (stride) but not the
vertical one if nSliceHeight is bigger than the actual height.

The calculated nFilledLen was wrong as it didn't include the padding
between planes.

https://bugzilla.gnome.org/show_bug.cgi?id=796749

diff --git a/omx/gstomxvideoenc.c b/omx/gstomxvideoenc.c
index 1a630c4..c296b95 100644 (file)
--- a/omx/gstomxvideoenc.c
+++ b/omx/gstomxvideoenc.c
@@ -2366,10 +2366,18 @@ gst_omx_video_enc_semi_planar_manual_copy (GstOMXVideoEnc * self,
 
     for (j = 0; j < height; j++) {
       memcpy (dest, src, width);
-      outbuf->omx_buf->nFilledLen += dest_stride;
       src += src_stride;
       dest += dest_stride;
     }
+
+    /* nFilledLen should include the vertical padding in each slice (spec 3.1.3.7.1) */
+    if (i == 0)
+      outbuf->omx_buf->nFilledLen +=
+          port_def->format.video.nSliceHeight * port_def->format.video.nStride;
+    else
+      outbuf->omx_buf->nFilledLen +=
+          (port_def->format.video.nSliceHeight / 2) *
+          port_def->format.video.nStride;
   }
 
   gst_video_frame_unmap (&frame);
@@ -2501,10 +2509,19 @@ gst_omx_video_enc_fill_buffer (GstOMXVideoEnc * self, GstBuffer * inbuf,
 
         for (j = 0; j < height; j++) {
           memcpy (dest, src, width);
-          outbuf->omx_buf->nFilledLen += dest_stride;
           src += src_stride;
           dest += dest_stride;
         }
+
+        /* nFilledLen should include the vertical padding in each slice (spec 3.1.3.7.1) */
+        if (i == 0)
+          outbuf->omx_buf->nFilledLen +=
+              port_def->format.video.nSliceHeight *
+              port_def->format.video.nStride;
+        else
+          outbuf->omx_buf->nFilledLen +=
+              (port_def->format.video.nSliceHeight / 2) *
+              (port_def->format.video.nStride / 2);
       }
       gst_video_frame_unmap (&frame);
       ret = TRUE;