nvdecoder: Fix for HEVC 4:4:4 format decoding

Map chroma_format_idc == 3 (which means 4:4:4 subsampling) correctly,
also pass coded bitdepth for decoder initialization instead of
inferring it from output format since they can be different.

Fixes: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/949
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1612>

diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.c b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.c
index cf48ead..34c0569 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.c
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.c
@@ -162,29 +162,6 @@ chroma_format_from_video_format (GstVideoFormat format)
   return cudaVideoChromaFormat_420;
 }
 
-static guint
-bitdepth_minus8_from_video_format (GstVideoFormat format)
-{
-  switch (format) {
-    case GST_VIDEO_FORMAT_NV12:
-    case GST_VIDEO_FORMAT_Y444:
-      return 0;
-    case GST_VIDEO_FORMAT_P010_10LE:
-    case GST_VIDEO_FORMAT_P010_10BE:
-      return 2;
-    case GST_VIDEO_FORMAT_P016_LE:
-    case GST_VIDEO_FORMAT_P016_BE:
-    case GST_VIDEO_FORMAT_Y444_16LE:
-    case GST_VIDEO_FORMAT_Y444_16BE:
-      return 8;
-    default:
-      g_assert_not_reached ();
-      break;
-  }
-
-  return 0;
-}
-
 static cudaVideoSurfaceFormat
 output_format_from_video_format (GstVideoFormat format)
 {
@@ -276,7 +253,8 @@ gst_nv_decoder_reset (GstNvDecoder * self)
 
 gboolean
 gst_nv_decoder_configure (GstNvDecoder * decoder, cudaVideoCodec codec,
-    GstVideoInfo * info, gint coded_width, gint coded_height, guint pool_size)
+    GstVideoInfo * info, gint coded_width, gint coded_height,
+    guint coded_bitdepth, guint pool_size)
 {
   CUVIDDECODECREATEINFO create_info = { 0, };
   GstVideoFormat format;
@@ -287,6 +265,7 @@ gst_nv_decoder_configure (GstNvDecoder * decoder, cudaVideoCodec codec,
   g_return_val_if_fail (info != NULL, FALSE);
   g_return_val_if_fail (coded_width >= GST_VIDEO_INFO_WIDTH (info), FALSE);
   g_return_val_if_fail (coded_height >= GST_VIDEO_INFO_HEIGHT (info), FALSE);
+  g_return_val_if_fail (coded_bitdepth >= 8, FALSE);
   g_return_val_if_fail (pool_size > 0, FALSE);
 
   gst_nv_decoder_reset (decoder);
@@ -304,7 +283,7 @@ gst_nv_decoder_configure (GstNvDecoder * decoder, cudaVideoCodec codec,
   create_info.CodecType = codec;
   create_info.ChromaFormat = chroma_format_from_video_format (format);
   create_info.ulCreationFlags = cudaVideoCreate_Default;
-  create_info.bitDepthMinus8 = bitdepth_minus8_from_video_format (format);
+  create_info.bitDepthMinus8 = coded_bitdepth - 8;
   create_info.ulIntraDecodeOnly = 0;
 
   create_info.display_area.left = 0;

diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.h b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.h
index 2f78249..947ed74 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.h
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.h
@@ -55,6 +55,7 @@ gboolean       gst_nv_decoder_configure (GstNvDecoder * decoder,
                                          GstVideoInfo * info,
                                          gint coded_width,
                                          gint coded_height,
+                                         guint coded_bitdepth,
                                          guint pool_size);
 
 GstNvDecoderFrame * gst_nv_decoder_new_frame (GstNvDecoder * decoder);

diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264dec.c b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264dec.c
index c0b7c76..b17dfba 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264dec.c
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264dec.c
@@ -446,6 +446,7 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps,
     /* FIXME: add support cudaVideoCodec_H264_SVC and cudaVideoCodec_H264_MVC */
     if (!gst_nv_decoder_configure (self->decoder,
             cudaVideoCodec_H264, &info, self->coded_width, self->coded_height,
+            self->bitdepth,
             /* Additional 4 buffers for render delay */
             max_dpb_size + 4)) {
       GST_ERROR_OBJECT (self, "Failed to configure decoder");

diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265dec.c b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265dec.c
index 6b38e2e..70f665e 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265dec.c
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265dec.c
@@ -362,16 +362,20 @@ gst_nv_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
     GstVideoFormat out_format = GST_VIDEO_FORMAT_UNKNOWN;
 
     if (self->bitdepth == 8) {
-      if (self->chroma_format_idc == 1)
+      if (self->chroma_format_idc == 1) {
         out_format = GST_VIDEO_FORMAT_NV12;
-      else {
-        GST_FIXME_OBJECT (self, "Could not support 8bits non-4:2:0 format");
+      } else if (self->chroma_format_idc == 3) {
+        out_format = GST_VIDEO_FORMAT_Y444;
+      } else {
+        GST_FIXME_OBJECT (self, "8 bits supports only 4:2:0 or 4:4:4 format");
       }
     } else if (self->bitdepth == 10) {
-      if (self->chroma_format_idc == 1)
+      if (self->chroma_format_idc == 1) {
         out_format = GST_VIDEO_FORMAT_P010_10LE;
-      else {
-        GST_FIXME_OBJECT (self, "Could not support 10bits non-4:2:0 format");
+      } else if (self->chroma_format_idc == 3) {
+        out_format = GST_VIDEO_FORMAT_Y444_16LE;
+      } else {
+        GST_FIXME_OBJECT (self, "10 bits supports only 4:2:0 or 4:4:4 format");
       }
     }
 
@@ -384,6 +388,7 @@ gst_nv_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
 
     if (!gst_nv_decoder_configure (self->decoder,
             cudaVideoCodec_HEVC, &info, self->coded_width, self->coded_height,
+            self->bitdepth,
             /* Additional 2 buffers for margin */
             max_dpb_size + 2)) {
       GST_ERROR_OBJECT (self, "Failed to configure decoder");

diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvvp8dec.c b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvvp8dec.c
index 36c7826..6f06045 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvvp8dec.c
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvvp8dec.c
@@ -257,7 +257,7 @@ gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
         GST_VIDEO_FORMAT_NV12, self->width, self->height);
 
     if (!gst_nv_decoder_configure (self->decoder,
-            cudaVideoCodec_VP8, &info, self->width, self->height,
+            cudaVideoCodec_VP8, &info, self->width, self->height, 8,
             /* +4 for render delay */
             NUM_OUTPUT_VIEW + 4)) {
       GST_ERROR_OBJECT (self, "Failed to configure decoder");

diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvvp9dec.c b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvvp9dec.c
index 3ba6fce..4663691 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvvp9dec.c
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvvp9dec.c
@@ -256,10 +256,11 @@ gst_nv_vp9_dec_new_sequence (GstVp9Decoder * decoder,
   if (self->profile == GST_VP9_PROFILE_0) {
     out_format = GST_VIDEO_FORMAT_NV12;
   } else if (self->profile == GST_VP9_PROFILE_2) {
-    if (frame_hdr->bit_depth == 10)
+    if (frame_hdr->bit_depth == 10) {
       out_format = GST_VIDEO_FORMAT_P010_10LE;
-    else
+    } else {
       out_format = GST_VIDEO_FORMAT_P016_LE;
+    }
   }
 
   if (out_format == GST_VIDEO_FORMAT_UNKNOWN) {
@@ -270,6 +271,7 @@ gst_nv_vp9_dec_new_sequence (GstVp9Decoder * decoder,
   gst_video_info_set_format (&info, out_format, self->width, self->height);
   if (!gst_nv_decoder_configure (self->decoder,
           cudaVideoCodec_VP9, &info, self->width, self->height,
+          frame_hdr->bit_depth,
           /* +4 for render delay */
           NUM_OUTPUT_VIEW)) {
     GST_ERROR_OBJECT (self, "Failed to configure decoder");