libs: encoder: h264: set profile via capsfilter

Until now, the encoder ignored the profile in src caps and chose one
according with the given parameters. But the encoder must honor the
profile specifed in src caps.

This patch do that, and if the encoder needs to choose the profile,
it will do it by following these rules:

1\ If given parameters are not compatible with given profile, the
   encoder will bail out with an error.
2\ The encoder will choose the higher profile indicated in the
   src caps.

https://bugzilla.gnome.org/show_bug.cgi?id=757941

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
index 42a05db3a5942c693dcda57c961271d19e2ea4f9..d1265c17d10986d37695e74b90caa6356eb08d2c 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
@@ -1043,24 +1043,34 @@ ensure_profile_limits (GstVaapiEncoderH264 * encoder)
   GstVaapiProfile profile;
 
   if (!encoder->max_profile_idc
-      || encoder->profile_idc <= encoder->max_profile_idc)
+      || encoder->profile_idc == encoder->max_profile_idc)
     return TRUE;
 
-  GST_WARNING ("lowering coding tools to meet target decoder constraints");
+  /* Give an error if the given parameters are invalid for requested
+   * profile rather than lowering profile.
+   */
+  if (encoder->profile_idc > encoder->max_profile_idc) {
+    GST_WARNING ("Invalid parameter for maximum profile");
+    return FALSE;
+  }
 
   profile = GST_VAAPI_PROFILE_UNKNOWN;
 
-  /* Try Main profile coding tools */
-  if (encoder->max_profile_idc < GST_H264_PROFILE_HIGH) {
-    encoder->use_dct8x8 = FALSE;
-    profile = GST_VAAPI_PROFILE_H264_MAIN;
-  }
+  if (encoder->profile_idc < encoder->max_profile_idc) {
+    /* Let profile be higher to fit in the maximum profile
+     * without changing parameters */
+    if (encoder->max_profile_idc > GST_H264_PROFILE_BASELINE)
+      profile = GST_VAAPI_PROFILE_H264_MAIN;
 
-  /* Try Constrained Baseline profile coding tools */
-  if (encoder->max_profile_idc < GST_H264_PROFILE_MAIN) {
-    encoder->num_bframes = 0;
-    encoder->use_cabac = FALSE;
-    profile = GST_VAAPI_PROFILE_H264_CONSTRAINED_BASELINE;
+    if (encoder->max_profile_idc > GST_H264_PROFILE_MAIN)
+      profile = GST_VAAPI_PROFILE_H264_HIGH;
+
+    if (encoder->max_profile_idc > GST_H264_PROFILE_HIGH) {
+      if (encoder->num_views > 2)
+        profile = GST_VAAPI_PROFILE_H264_MULTIVIEW_HIGH;
+      else if (encoder->num_views == 2)
+        profile = GST_VAAPI_PROFILE_H264_STEREO_HIGH;
+    }
   }
 
   if (profile) {