From b79c949789628f537f68e190a2a73817b737c34b Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Mon, 6 Apr 2020 15:06:01 -0400 Subject: [PATCH] codecs: h264: Fix DPB size calculation As per specification in A.3.1 h) and A.3.2 f), the maximum size of the DPB is 16. Fix the maximum in the fine and fix the formula to use MIN instead of MAX so that we no longer always use the maximum for the profile/level. --- gst-libs/gst/codecs/gsth264decoder.c | 5 ++++- gst-libs/gst/codecs/gsth264picture.h | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/codecs/gsth264decoder.c b/gst-libs/gst/codecs/gsth264decoder.c index 456b8fd182..26127f5122 100644 --- a/gst-libs/gst/codecs/gsth264decoder.c +++ b/gst-libs/gst/codecs/gsth264decoder.c @@ -1684,9 +1684,12 @@ gst_h264_decoder_process_sps (GstH264Decoder * self, GstH264SPS * sps) max_dpb_frames = MIN (max_dpb_mbs / (width_mb * height_mb), GST_H264_DPB_MAX_SIZE); - max_dpb_size = MAX (max_dpb_frames, + max_dpb_size = MIN (max_dpb_frames, MAX (sps->num_ref_frames, sps->vui_parameters.max_dec_frame_buffering)); + /* Safety, so that subclass don't need bound checking */ + g_return_val_if_fail (max_dpb_size <= GST_H264_DPB_MAX_SIZE, FALSE); + prev_max_dpb_size = gst_h264_dpb_get_max_num_pics (priv->dpb); if (priv->width != sps->width || priv->height != sps->height || prev_max_dpb_size != max_dpb_size) { diff --git a/gst-libs/gst/codecs/gsth264picture.h b/gst-libs/gst/codecs/gsth264picture.h index 73cad13630..44db3654e7 100644 --- a/gst-libs/gst/codecs/gsth264picture.h +++ b/gst-libs/gst/codecs/gsth264picture.h @@ -34,7 +34,8 @@ G_BEGIN_DECLS typedef struct _GstH264Slice GstH264Slice; typedef struct _GstH264Picture GstH264Picture; -#define GST_H264_DPB_MAX_SIZE 32 +/* As specified in A.3.1 h) and A.3.2 f) */ +#define GST_H264_DPB_MAX_SIZE 16 struct _GstH264Slice { -- 2.34.1