h265decoder: Do not abort when failed to prepare ref pic set

Currently the element calls abort when failed to prepare reference
picture set. This can happent when the input stream is somehow
corrupted, like a rtsp strem with lost packets. Now it will only
return with GST_FLOW_ERROR instead of terminating whole process.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3505>

diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c
index 3126517..723697c 100644 (file)
--- a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c
@@ -1553,7 +1553,8 @@ gst_h265_decoder_prepare_rps (GstH265Decoder * self, const GstH265Slice * slice,
       stRefPic =
           &sps->short_term_ref_pic_set[slice_hdr->short_term_ref_pic_set_idx];
 
-    g_assert (stRefPic != NULL);
+    if (stRefPic == NULL)
+      return FALSE;
 
     GST_LOG_OBJECT (self,
         "NumDeltaPocs: %d, NumNegativePics: %d, NumPositivePics %d",
@@ -1810,8 +1811,12 @@ gst_h265_decoder_start_current_picture (GstH265Decoder * self)
     priv->input_state_changed = FALSE;
   }
 
-  gst_h265_decoder_prepare_rps (self, &priv->current_slice,
-      priv->current_picture);
+  if (!gst_h265_decoder_prepare_rps (self, &priv->current_slice,
+          priv->current_picture)) {
+    GST_WARNING_OBJECT (self, "Failed to prepare ref pic set");
+    gst_clear_h265_picture (&priv->current_picture);
+    return GST_FLOW_ERROR;
+  }
 
   ret = gst_h265_decoder_dpb_init (self,
       &priv->current_slice, priv->current_picture);