decoder: h264: fix memory leak in PPS.

[platform/upstream/gstreamer-vaapi.git] / gst-libs / gst / vaapi / gstvaapidecoder_h264.c

diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
index 2db5073..7caabfb 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
@@ -104,6 +104,9 @@ gst_vaapi_parser_info_h264_finalize(GstVaapiParserInfoH264 *pi)
     case GST_H264_NAL_SUBSET_SPS:
         gst_h264_sps_clear(&pi->data.sps);
         break;
+    case GST_H264_NAL_PPS:
+        gst_h264_pps_clear(&pi->data.pps);
+        break;
     case GST_H264_NAL_SEI:
         if (pi->data.sei) {
             g_array_unref(pi->data.sei);
@@ -689,11 +692,11 @@ dpb_output(
 {
     picture->output_needed = FALSE;
 
-    if (fs) {
-        if (--fs->output_needed > 0)
-            return TRUE;
-        picture = fs->buffers[0];
-    }
+    if (--fs->output_needed > 0)
+        return TRUE;
+
+    if (!GST_VAAPI_PICTURE_IS_COMPLETE(picture))
+        return TRUE;
     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
 }
 
@@ -929,6 +932,14 @@ dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
             GST_VAAPI_PICTURE_H264(picture->base.parent_picture));
         if (found_index >= 0)
             return gst_vaapi_frame_store_add(priv->dpb[found_index], picture);
+
+        // ... also check the previous picture that was immediately output
+        fs = priv->prev_frames[picture->base.voc];
+        if (fs && &fs->buffers[0]->base == picture->base.parent_picture) {
+            if (!gst_vaapi_frame_store_add(fs, picture))
+                return FALSE;
+            return dpb_output(decoder, fs, picture);
+        }
     }
 
     // Create new frame store, and split fields if necessary
@@ -938,6 +949,11 @@ dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
     gst_vaapi_frame_store_replace(&priv->prev_frames[picture->base.voc], fs);
     gst_vaapi_frame_store_unref(fs);
 
+    if (picture->output_flag) {
+        picture->output_needed = TRUE;
+        fs->output_needed++;
+    }
+
     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
         if (!gst_vaapi_frame_store_split_fields(fs))
             return FALSE;
@@ -965,18 +981,13 @@ dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
             if (!StoreInterViewOnlyRefFlag) {
                 if (dpb_find_lowest_poc(decoder, picture, &found_picture) < 0 ||
                     found_picture->base.poc > picture->base.poc)
-                    return dpb_output(decoder, NULL, picture);
+                    return dpb_output(decoder, fs, picture);
             }
             if (!dpb_bump(decoder, picture))
                 return FALSE;
         }
     }
-
     gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
-    if (picture->output_flag) {
-        picture->output_needed = TRUE;
-        fs->output_needed++;
-    }
     return TRUE;
 }
 
@@ -1455,12 +1466,12 @@ decode_current_picture(GstVaapiDecoderH264 *decoder)
     if (!picture)
         return GST_VAAPI_DECODER_STATUS_SUCCESS;
 
+    if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
+        goto error;
     if (!exec_ref_pic_marking(decoder, picture))
         goto error;
     if (!dpb_add(decoder, picture))
         goto error;
-    if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
-        goto error;
     gst_vaapi_picture_replace(&priv->current_picture, NULL);
     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 
@@ -3879,6 +3890,7 @@ gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
     guint i, size, buf_size, nalu_size, flags;
     guint32 start_code;
     gint ofs, ofs2;
+    gboolean at_au_end = FALSE;
 
     status = ensure_decoder(decoder);
     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
@@ -3886,6 +3898,7 @@ gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
 
     switch (priv->stream_alignment) {
     case GST_VAAPI_STREAM_ALIGN_H264_NALU:
+    case GST_VAAPI_STREAM_ALIGN_H264_AU:
         size = gst_adapter_available_fast(adapter);
         break;
     default:
@@ -3908,6 +3921,8 @@ gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
         buf_size = priv->nal_length_size + nalu_size;
         if (size < buf_size)
             return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
+        else if (priv->stream_alignment == GST_VAAPI_STREAM_ALIGN_H264_AU)
+            at_au_end = (buf_size == size);
     }
     else {
         if (size < 4)
@@ -3933,7 +3948,10 @@ gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
                 scan_for_start_code(adapter, ofs2, size - ofs2, NULL);
             if (ofs < 0) {
                 // Assume the whole NAL unit is present if end-of-stream
-                if (!at_eos) {
+                // or stream buffers aligned on access unit boundaries
+                if (priv->stream_alignment == GST_VAAPI_STREAM_ALIGN_H264_AU)
+                    at_au_end = TRUE;
+                else if (!at_eos) {
                     ps->input_offset2 = size;
                     return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
                 }
@@ -3998,6 +4016,10 @@ gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
         return status;
 
     flags = 0;
+    if (at_au_end) {
+        flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_END |
+            GST_VAAPI_DECODER_UNIT_FLAG_AU_END;
+    }
     switch (pi->nalu.type) {
     case GST_H264_NAL_AU_DELIMITER:
         flags |= GST_VAAPI_DECODER_UNIT_FLAG_AU_START;
@@ -4029,7 +4051,12 @@ gst_vaapi_decoder_h264_parse(GstVaapiDecoder *base_decoder,
     case GST_H264_NAL_SLICE_IDR:
     case GST_H264_NAL_SLICE:
         flags |= GST_VAAPI_DECODER_UNIT_FLAG_SLICE;
-        if (is_new_picture(pi, priv->prev_slice_pi)) {
+        if (priv->prev_pi &&
+            (priv->prev_pi->flags & GST_VAAPI_DECODER_UNIT_FLAG_AU_END)) {
+            flags |= GST_VAAPI_DECODER_UNIT_FLAG_AU_START |
+                GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
+        }
+        else if (is_new_picture(pi, priv->prev_slice_pi)) {
             flags |= GST_VAAPI_DECODER_UNIT_FLAG_FRAME_START;
             if (is_new_access_unit(pi, priv->prev_slice_pi))
                 flags |= GST_VAAPI_DECODER_UNIT_FLAG_AU_START;