libs: dec: h265: Consider chroma_bit_depth to choose chrome type

For some main-10 stream, sometime the luma is 8 bits while chrome is more
than 8 bits, which cause using the wrong NV12 surface as the render target
and decoding error.

Fix #176

diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h265.c b/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
index 6cd61f6..21dcb4c 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
@@ -1144,7 +1144,7 @@ ensure_context (GstVaapiDecoderH265 * decoder, GstH265SPS * sps)
 
   chroma_type =
       gst_vaapi_utils_h265_get_chroma_type (sps->chroma_format_idc,
-      sps->bit_depth_luma_minus8 + 8);
+      sps->bit_depth_luma_minus8 + 8, sps->bit_depth_chroma_minus8 + 8);
   if (!chroma_type) {
     GST_ERROR ("unsupported chroma_format_idc %u", sps->chroma_format_idc);
     return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;

diff --git a/gst-libs/gst/vaapi/gstvaapiutils_h265.c b/gst-libs/gst/vaapi/gstvaapiutils_h265.c
index 3e71c36..5b3bef1 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapiutils_h265.c
+++ b/gst-libs/gst/vaapi/gstvaapiutils_h265.c
@@ -326,37 +326,48 @@ gst_vaapi_utils_h265_get_level_limits_table (guint * out_length_ptr)
 /** Returns GstVaapiChromaType from H.265 chroma_format_idc value */
 GstVaapiChromaType
 gst_vaapi_utils_h265_get_chroma_type (guint chroma_format_idc,
-    guint luma_bit_depth)
+    guint luma_bit_depth, guint chroma_bit_depth)
 {
   GstVaapiChromaType chroma_type = (GstVaapiChromaType) 0;
+  guint depth = 0;
+
+  if (luma_bit_depth < 8 || chroma_bit_depth < 8 ||
+      luma_bit_depth > 16 || chroma_bit_depth > 16) {
+    GST_WARNING ("invalid luma_bit_depth or chroma_bit_depth value");
+    return chroma_type;
+  }
+
+  depth = MAX (luma_bit_depth, chroma_bit_depth);
 
   switch (chroma_format_idc) {
     case 0:
       chroma_type = GST_VAAPI_CHROMA_TYPE_YUV400;
       break;
     case 1:
-      if (luma_bit_depth == 8)
+      if (depth == 8)
         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
-      else if (luma_bit_depth > 8)
+      else if (depth > 8 && depth <= 10)
         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420_10BPP;
       break;
     case 2:
-      if (luma_bit_depth == 8)
+      if (depth == 8)
         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
-      else if (luma_bit_depth > 8)
+      else if (depth > 8 && depth <= 10)
         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422_10BPP;
       break;
     case 3:
-      if (luma_bit_depth == 8)
+      if (depth == 8)
         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
-      else if (luma_bit_depth > 8)
+      else if (depth > 8 && depth <= 10)
         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444_10BPP;
       break;
     default:
-      GST_DEBUG ("unsupported chroma_format_idc value");
-      chroma_type = (GstVaapiChromaType) 0;
       break;
   }
+
+  if (chroma_type == (GstVaapiChromaType) 0)
+    GST_DEBUG ("unsupported chroma_format_idc value");
+
   return chroma_type;
 }
 

diff --git a/gst-libs/gst/vaapi/gstvaapiutils_h265_priv.h b/gst-libs/gst/vaapi/gstvaapiutils_h265_priv.h
index 40938c7..88f7fda 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapiutils_h265_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapiutils_h265_priv.h
@@ -93,7 +93,8 @@ gst_vaapi_utils_h265_get_level_limits_table (guint * out_length_ptr);
 /* Returns GstVaapiChromaType from H.265 chroma_format_idc value */
 G_GNUC_INTERNAL
 GstVaapiChromaType
-gst_vaapi_utils_h265_get_chroma_type (guint chroma_format_idc, guint luma_bit_depth);
+gst_vaapi_utils_h265_get_chroma_type (guint chroma_format_idc,
+    guint luma_bit_depth, guint chroma_bit_depth);
 
 /* Returns H.265 chroma_format_idc value from GstVaapiChromaType */
 G_GNUC_INTERNAL