codecs: h264: Fix DPB size calculation

As per specification in A.3.1 h) and A.3.2 f), the maximum size of the DPB is
16. Fix the maximum in the fine and fix the formula to use MIN instead of MAX
so that we no longer always use the maximum for the profile/level.

diff --git a/gst-libs/gst/codecs/gsth264decoder.c b/gst-libs/gst/codecs/gsth264decoder.c
index 456b8fd..26127f5 100644 (file)
--- a/gst-libs/gst/codecs/gsth264decoder.c
+++ b/gst-libs/gst/codecs/gsth264decoder.c
@@ -1684,9 +1684,12 @@ gst_h264_decoder_process_sps (GstH264Decoder * self, GstH264SPS * sps)
   max_dpb_frames = MIN (max_dpb_mbs / (width_mb * height_mb),
       GST_H264_DPB_MAX_SIZE);
 
-  max_dpb_size = MAX (max_dpb_frames,
+  max_dpb_size = MIN (max_dpb_frames,
       MAX (sps->num_ref_frames, sps->vui_parameters.max_dec_frame_buffering));
 
+  /* Safety, so that subclass don't need bound checking */
+  g_return_val_if_fail (max_dpb_size <= GST_H264_DPB_MAX_SIZE, FALSE);
+
   prev_max_dpb_size = gst_h264_dpb_get_max_num_pics (priv->dpb);
   if (priv->width != sps->width || priv->height != sps->height ||
       prev_max_dpb_size != max_dpb_size) {

diff --git a/gst-libs/gst/codecs/gsth264picture.h b/gst-libs/gst/codecs/gsth264picture.h
index 73cad13..44db365 100644 (file)
--- a/gst-libs/gst/codecs/gsth264picture.h
+++ b/gst-libs/gst/codecs/gsth264picture.h
@@ -34,7 +34,8 @@ G_BEGIN_DECLS
 typedef struct _GstH264Slice GstH264Slice;
 typedef struct _GstH264Picture GstH264Picture;
 
-#define GST_H264_DPB_MAX_SIZE 32
+/* As specified in A.3.1 h) and A.3.2 f) */
+#define GST_H264_DPB_MAX_SIZE 16
 
 struct _GstH264Slice
 {