encoder: h264: fix frame cropping rectangle calculation.

Fix frame cropping rectangle calculation to handle horizontal resolutions
that don't match a multiple of 16 pixels, but also the vertical resolution
that was incorrectly computed for progressive sequences too.

https://bugzilla.gnome.org/show_bug.cgi?id=722089

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
index 4d98c55..ff16098 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
@@ -991,14 +991,17 @@ fill_va_sequence_param (GstVaapiEncoderH264 * encoder,
         sizeof (seq_param->offset_for_ref_frame));
   }
 
-  if (encoder->mb_height * 16 - GST_VAAPI_ENCODER_HEIGHT (encoder)) {
+  /* frame_cropping_flag */
+  if ((GST_VAAPI_ENCODER_WIDTH (encoder) & 15) ||
+      (GST_VAAPI_ENCODER_HEIGHT (encoder) & 15)) {
     seq_param->frame_cropping_flag = 1;
     seq_param->frame_crop_left_offset = 0;
-    seq_param->frame_crop_right_offset = 0;
+    seq_param->frame_crop_right_offset =
+        16 * encoder->mb_width - GST_VAAPI_ENCODER_WIDTH (encoder);
     seq_param->frame_crop_top_offset = 0;
     seq_param->frame_crop_bottom_offset =
-        ((encoder->mb_height * 16 - GST_VAAPI_ENCODER_HEIGHT (encoder)) /
-        (2 * (!seq_param->seq_fields.bits.frame_mbs_only_flag + 1)));
+        (16 * encoder->mb_height - GST_VAAPI_ENCODER_HEIGHT (encoder)) /
+        (2 - seq_param->seq_fields.bits.frame_mbs_only_flag);
   }
 
   /* vui not set */