codecs: h265decoder: Use GstFlowReturn everywhere

boolean return value is not sufficient for representing the reason
of error in most cases. For instance, any errors around new_sequence()
would mean negotiation error, not just *ERROR*.
And some subclasses will allocate buffer/memory/surface on new_picture()
but it could be failed because of expected error, likely flushing

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1019>

diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c
index 16d4122..443f924 100644 (file)
--- a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c
@@ -69,7 +69,6 @@ struct _GstH265DecoderPrivate
   GstH265DecoderAlign align;
   GstH265Parser *parser;
   GstH265Dpb *dpb;
-  GstFlowReturn last_ret;
 
   /* 0: frame or field-pair interlaced stream
    * 1: alternating, single field interlaced stream.
@@ -126,6 +125,11 @@ struct _GstH265DecoderPrivate
   GArray *ref_pic_list1;
 };
 
+#define UPDATE_FLOW_RETURN(ret,new_ret) G_STMT_START { \
+  if (*(ret) == GST_FLOW_OK) \
+    *(ret) = new_ret; \
+} G_STMT_END
+
 #define parent_class gst_h265_decoder_parent_class
 G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstH265Decoder, gst_h265_decoder,
     GST_TYPE_VIDEO_DECODER,
@@ -145,11 +149,13 @@ static GstFlowReturn gst_h265_decoder_drain (GstVideoDecoder * decoder);
 static GstFlowReturn gst_h265_decoder_handle_frame (GstVideoDecoder * decoder,
     GstVideoCodecFrame * frame);
 
-static gboolean gst_h265_decoder_finish_current_picture (GstH265Decoder * self);
+static void gst_h265_decoder_finish_current_picture (GstH265Decoder * self,
+    GstFlowReturn * ret);
 static void gst_h265_decoder_clear_ref_pic_sets (GstH265Decoder * self);
 static void gst_h265_decoder_clear_dpb (GstH265Decoder * self, gboolean flush);
-static gboolean gst_h265_decoder_drain_internal (GstH265Decoder * self);
-static gboolean gst_h265_decoder_start_current_picture (GstH265Decoder * self);
+static GstFlowReturn gst_h265_decoder_drain_internal (GstH265Decoder * self);
+static GstFlowReturn
+gst_h265_decoder_start_current_picture (GstH265Decoder * self);
 
 static void
 gst_h265_decoder_class_init (GstH265DecoderClass * klass)
@@ -243,23 +249,22 @@ gst_h265_decoder_stop (GstVideoDecoder * decoder)
   return TRUE;
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_parse_vps (GstH265Decoder * self, GstH265NalUnit * nalu)
 {
   GstH265DecoderPrivate *priv = self->priv;
   GstH265VPS vps;
   GstH265ParserResult pres;
-  gboolean ret = TRUE;
 
   pres = gst_h265_parser_parse_vps (priv->parser, nalu, &vps);
   if (pres != GST_H265_PARSER_OK) {
     GST_WARNING_OBJECT (self, "Failed to parse VPS, result %d", pres);
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   GST_LOG_OBJECT (self, "VPS parsed");
 
-  return ret;
+  return GST_FLOW_OK;
 }
 
 static gboolean
@@ -281,7 +286,7 @@ gst_h265_decoder_is_crop_rect_changed (GstH265Decoder * self, GstH265SPS * sps)
   return FALSE;
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_process_sps (GstH265Decoder * self, GstH265SPS * sps)
 {
   GstH265DecoderPrivate *priv = self->priv;
@@ -293,6 +298,7 @@ gst_h265_decoder_process_sps (GstH265Decoder * self, GstH265SPS * sps)
   guint8 field_seq_flag = 0;
   guint8 progressive_source_flag = 0;
   guint8 interlaced_source_flag = 0;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   /* A.4.1 */
   MaxLumaPS = 35651584;
@@ -334,9 +340,10 @@ gst_h265_decoder_process_sps (GstH265Decoder * self, GstH265SPS * sps)
 
     g_assert (klass->new_sequence);
 
-    if (!klass->new_sequence (self, sps, max_dpb_size)) {
-      GST_ERROR_OBJECT (self, "subclass does not want accept new sequence");
-      return FALSE;
+    ret = klass->new_sequence (self, sps, max_dpb_size);
+    if (ret != GST_FLOW_OK) {
+      GST_WARNING_OBJECT (self, "subclass does not want accept new sequence");
+      return ret;
     }
 
     priv->width = sps->width;
@@ -361,38 +368,38 @@ gst_h265_decoder_process_sps (GstH265Decoder * self, GstH265SPS * sps)
 
   GST_DEBUG_OBJECT (self, "Set DPB max size %d", max_dpb_size);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_parse_sps (GstH265Decoder * self, GstH265NalUnit * nalu)
 {
   GstH265DecoderPrivate *priv = self->priv;
   GstH265SPS sps;
   GstH265ParserResult pres;
-  gboolean ret;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   pres = gst_h265_parse_sps (priv->parser, nalu, &sps, TRUE);
   if (pres != GST_H265_PARSER_OK) {
     GST_WARNING_OBJECT (self, "Failed to parse SPS, result %d", pres);
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   GST_LOG_OBJECT (self, "SPS parsed");
 
   ret = gst_h265_decoder_process_sps (self, &sps);
-  if (!ret) {
+  if (ret != GST_FLOW_OK) {
     GST_WARNING_OBJECT (self, "Failed to process SPS");
   } else if (gst_h265_parser_update_sps (priv->parser,
           &sps) != GST_H265_PARSER_OK) {
     GST_WARNING_OBJECT (self, "Failed to update SPS");
-    ret = FALSE;
+    ret = GST_FLOW_ERROR;
   }
 
   return ret;
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_parse_pps (GstH265Decoder * self, GstH265NalUnit * nalu)
 {
   GstH265DecoderPrivate *priv = self->priv;
@@ -402,15 +409,15 @@ gst_h265_decoder_parse_pps (GstH265Decoder * self, GstH265NalUnit * nalu)
   pres = gst_h265_parser_parse_pps (priv->parser, nalu, &pps);
   if (pres != GST_H265_PARSER_OK) {
     GST_WARNING_OBJECT (self, "Failed to parse PPS, result %d", pres);
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   GST_LOG_OBJECT (self, "PPS parsed");
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_parse_sei (GstH265Decoder * self, GstH265NalUnit * nalu)
 {
   GstH265DecoderPrivate *priv = self->priv;
@@ -425,7 +432,7 @@ gst_h265_decoder_parse_sei (GstH265Decoder * self, GstH265NalUnit * nalu)
     /* XXX: Ignore error from SEI parsing, it might be malformed bitstream,
      * or our fault. But shouldn't be critical  */
     g_clear_pointer (&messages, g_array_unref);
-    return TRUE;
+    return GST_FLOW_OK;
   }
 
   for (i = 0; i < messages->len; i++) {
@@ -450,7 +457,7 @@ gst_h265_decoder_parse_sei (GstH265Decoder * self, GstH265NalUnit * nalu)
   g_array_free (messages, TRUE);
   GST_LOG_OBJECT (self, "SEI parsed");
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static void
@@ -573,7 +580,7 @@ gst_h265_decoder_process_ref_pic_lists (GstH265Decoder * self,
   g_array_set_size (tmp_refs, 0);
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_decode_slice (GstH265Decoder * self)
 {
   GstH265DecoderClass *klass = GST_H265_DECODER_GET_CLASS (self);
@@ -582,11 +589,11 @@ gst_h265_decoder_decode_slice (GstH265Decoder * self)
   GstH265Picture *picture = priv->current_picture;
   GArray *l0 = NULL;
   GArray *l1 = NULL;
-  gboolean ret;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   if (!picture) {
     GST_ERROR_OBJECT (self, "No current picture");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   g_assert (klass->decode_slice);
@@ -607,7 +614,7 @@ gst_h265_decoder_decode_slice (GstH265Decoder * self)
   return ret;
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_preprocess_slice (GstH265Decoder * self, GstH265Slice * slice)
 {
   GstH265DecoderPrivate *priv = self->priv;
@@ -617,18 +624,19 @@ gst_h265_decoder_preprocess_slice (GstH265Decoder * self, GstH265Slice * slice)
     GST_WARNING_OBJECT (self,
         "Current picture is not finished but slice header has "
         "first_slice_segment_in_pic_flag");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_parse_slice (GstH265Decoder * self, GstH265NalUnit * nalu,
     GstClockTime pts)
 {
   GstH265DecoderPrivate *priv = self->priv;
   GstH265ParserResult pres = GST_H265_PARSER_OK;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   memset (&priv->current_slice, 0, sizeof (GstH265Slice));
 
@@ -639,7 +647,7 @@ gst_h265_decoder_parse_slice (GstH265Decoder * self, GstH265NalUnit * nalu,
     GST_ERROR_OBJECT (self, "Failed to parse slice header, ret %d", pres);
     memset (&priv->current_slice, 0, sizeof (GstH265Slice));
 
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   priv->current_slice.nalu = *nalu;
@@ -656,8 +664,9 @@ gst_h265_decoder_parse_slice (GstH265Decoder * self, GstH265NalUnit * nalu,
     memset (&priv->prev_independent_slice.nalu, 0, sizeof (GstH265NalUnit));
   }
 
-  if (!gst_h265_decoder_preprocess_slice (self, &priv->current_slice))
-    return FALSE;
+  ret = gst_h265_decoder_preprocess_slice (self, &priv->current_slice);
+  if (ret != GST_FLOW_OK)
+    return ret;
 
   priv->active_pps = priv->current_slice.header.pps;
   priv->active_sps = priv->active_pps->sps;
@@ -667,32 +676,34 @@ gst_h265_decoder_parse_slice (GstH265Decoder * self, GstH265NalUnit * nalu,
     GstH265Picture *picture;
     gboolean ret = TRUE;
 
+    g_assert (priv->current_frame);
+
     picture = gst_h265_picture_new ();
     picture->pts = pts;
     /* This allows accessing the frame from the picture. */
     picture->system_frame_number = priv->current_frame->system_frame_number;
 
     priv->current_picture = picture;
-    g_assert (priv->current_frame);
 
     if (klass->new_picture)
       ret = klass->new_picture (self, priv->current_frame, picture);
 
-    if (!ret) {
-      GST_ERROR_OBJECT (self, "subclass does not want accept new picture");
+    if (ret != GST_FLOW_OK) {
+      GST_WARNING_OBJECT (self, "subclass does not want accept new picture");
       priv->current_picture = NULL;
       gst_h265_picture_unref (picture);
-      return FALSE;
+      return ret;
     }
 
-    if (!gst_h265_decoder_start_current_picture (self)) {
-      GST_ERROR_OBJECT (self, "start picture failed");
-      return FALSE;
+    ret = gst_h265_decoder_start_current_picture (self);
+    if (ret != GST_FLOW_OK) {
+      GST_WARNING_OBJECT (self, "start picture failed");
+      return ret;
     }
 
     /* this picture was dropped */
     if (!priv->current_picture)
-      return TRUE;
+      return GST_FLOW_OK;
   }
 
   return gst_h265_decoder_decode_slice (self);
@@ -703,7 +714,7 @@ gst_h265_decoder_decode_nal (GstH265Decoder * self, GstH265NalUnit * nalu,
     GstClockTime pts)
 {
   GstH265DecoderPrivate *priv = self->priv;
-  gboolean ret = TRUE;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   GST_LOG_OBJECT (self, "Parsed nal type: %d, offset %d, size %d",
       nalu->type, nalu->offset, nalu->size);
@@ -798,7 +809,7 @@ gst_h265_decoder_format_from_caps (GstH265Decoder * self, GstCaps * caps,
   }
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_parse_codec_data (GstH265Decoder * self, const guint8 * data,
     gsize size)
 {
@@ -808,16 +819,17 @@ gst_h265_decoder_parse_codec_data (GstH265Decoder * self, const guint8 * data,
   guint num_nals, i, j;
   GstH265ParserResult pres;
   GstH265NalUnit nalu;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   /* parse the hvcC data */
   if (size < 23) {
     GST_WARNING_OBJECT (self, "hvcC too small");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   /* wrong hvcC version */
   if (data[0] != 0 && data[0] != 1) {
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   priv->nal_length_size = (data[21] & 0x03) + 1;
@@ -829,7 +841,7 @@ gst_h265_decoder_parse_codec_data (GstH265Decoder * self, const guint8 * data,
   for (i = 0; i < num_nal_arrays; i++) {
     if (off + 3 >= size) {
       GST_WARNING_OBJECT (self, "hvcC too small");
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
 
     num_nals = GST_READ_UINT16_BE (data + off + 1);
@@ -840,26 +852,29 @@ gst_h265_decoder_parse_codec_data (GstH265Decoder * self, const guint8 * data,
 
       if (pres != GST_H265_PARSER_OK) {
         GST_WARNING_OBJECT (self, "hvcC too small");
-        return FALSE;
+        return GST_FLOW_ERROR;
       }
 
       switch (nalu.type) {
         case GST_H265_NAL_VPS:
-          if (!gst_h265_decoder_parse_vps (self, &nalu)) {
+          ret = gst_h265_decoder_parse_vps (self, &nalu);
+          if (ret != GST_FLOW_OK) {
             GST_WARNING_OBJECT (self, "Failed to parse VPS");
-            return FALSE;
+            return ret;
           }
           break;
         case GST_H265_NAL_SPS:
-          if (!gst_h265_decoder_parse_sps (self, &nalu)) {
+          ret = gst_h265_decoder_parse_sps (self, &nalu);
+          if (ret != GST_FLOW_OK) {
             GST_WARNING_OBJECT (self, "Failed to parse SPS");
-            return FALSE;
+            return ret;
           }
           break;
         case GST_H265_NAL_PPS:
-          if (!gst_h265_decoder_parse_pps (self, &nalu)) {
+          ret = gst_h265_decoder_parse_pps (self, &nalu);
+          if (ret != GST_FLOW_OK) {
             GST_WARNING_OBJECT (self, "Failed to parse PPS");
-            return FALSE;
+            return ret;
           }
           break;
         default:
@@ -870,7 +885,7 @@ gst_h265_decoder_parse_codec_data (GstH265Decoder * self, const guint8 * data,
     }
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static gboolean
@@ -947,7 +962,8 @@ gst_h265_decoder_set_format (GstVideoDecoder * decoder,
     GstMapInfo map;
 
     gst_buffer_map (priv->codec_data, &map, GST_MAP_READ);
-    if (!gst_h265_decoder_parse_codec_data (self, map.data, map.size)) {
+    if (gst_h265_decoder_parse_codec_data (self, map.data, map.size) !=
+        GST_FLOW_OK) {
       /* keep going without error.
        * Probably inband SPS/PPS might be valid data */
       GST_WARNING_OBJECT (self, "Failed to handle codec data");
@@ -972,13 +988,9 @@ static GstFlowReturn
 gst_h265_decoder_drain (GstVideoDecoder * decoder)
 {
   GstH265Decoder *self = GST_H265_DECODER (decoder);
-  GstH265DecoderPrivate *priv = self->priv;
 
-  priv->last_ret = GST_FLOW_OK;
   /* dpb will be cleared by this method */
-  gst_h265_decoder_drain_internal (self);
-
-  return priv->last_ret;
+  return gst_h265_decoder_drain_internal (self);
 }
 
 static GstFlowReturn
@@ -1453,11 +1465,14 @@ gst_h265_decoder_prepare_rps (GstH265Decoder * self, const GstH265Slice * slice,
 
 static void
 gst_h265_decoder_do_output_picture (GstH265Decoder * self,
-    GstH265Picture * picture)
+    GstH265Picture * picture, GstFlowReturn * ret)
 {
   GstH265DecoderPrivate *priv = self->priv;
   GstH265DecoderClass *klass;
   GstVideoCodecFrame *frame = NULL;
+  GstFlowReturn flow_ret = GST_FLOW_OK;
+
+  g_assert (ret != NULL);
 
   GST_LOG_OBJECT (self, "Output picture %p (poc %d)", picture,
       picture->pic_order_cnt);
@@ -1477,7 +1492,7 @@ gst_h265_decoder_do_output_picture (GstH265Decoder * self,
     GST_ERROR_OBJECT (self,
         "No available codec frame with frame number %d",
         picture->system_frame_number);
-    priv->last_ret = GST_FLOW_ERROR;
+    UPDATE_FLOW_RETURN (ret, GST_FLOW_ERROR);
 
     gst_h265_picture_unref (picture);
     return;
@@ -1486,7 +1501,9 @@ gst_h265_decoder_do_output_picture (GstH265Decoder * self,
   klass = GST_H265_DECODER_GET_CLASS (self);
 
   g_assert (klass->output_picture);
-  priv->last_ret = klass->output_picture (self, frame, picture);
+  flow_ret = klass->output_picture (self, frame, picture);
+
+  UPDATE_FLOW_RETURN (ret, flow_ret);
 }
 
 static void
@@ -1513,23 +1530,24 @@ gst_h265_decoder_clear_dpb (GstH265Decoder * self, gboolean flush)
   priv->last_output_poc = G_MININT32;
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_drain_internal (GstH265Decoder * self)
 {
   GstH265DecoderPrivate *priv = self->priv;
   GstH265Picture *picture;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   while ((picture = gst_h265_dpb_bump (priv->dpb, TRUE)) != NULL)
-    gst_h265_decoder_do_output_picture (self, picture);
+    gst_h265_decoder_do_output_picture (self, picture, &ret);
 
   gst_h265_dpb_clear (priv->dpb);
   priv->last_output_poc = G_MININT32;
 
-  return TRUE;
+  return ret;
 }
 
 /* C.5.2.2 */
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_dpb_init (GstH265Decoder * self, const GstH265Slice * slice,
     GstH265Picture * picture)
 {
@@ -1538,6 +1556,7 @@ gst_h265_decoder_dpb_init (GstH265Decoder * self, const GstH265Slice * slice,
   const GstH265NalUnit *nalu = &slice->nalu;
   const GstH265SPS *sps = priv->active_sps;
   GstH265Picture *to_output;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   /* C 3.2 */
   if (GST_H265_IS_NAL_TYPE_IRAP (nalu->type) && picture->NoRaslOutputFlag
@@ -1554,7 +1573,7 @@ gst_h265_decoder_dpb_init (GstH265Decoder * self, const GstH265Slice * slice,
     } else {
       gst_h265_dpb_delete_unused (priv->dpb);
       while ((to_output = gst_h265_dpb_bump (priv->dpb, FALSE)) != NULL)
-        gst_h265_decoder_do_output_picture (self, to_output);
+        gst_h265_decoder_do_output_picture (self, to_output, &ret);
 
       if (gst_h265_dpb_get_size (priv->dpb) > 0) {
         GST_WARNING_OBJECT (self, "IDR or BLA frame failed to clear the dpb, "
@@ -1579,26 +1598,26 @@ gst_h265_decoder_dpb_init (GstH265Decoder * self, const GstH265Slice * slice,
         break;
       }
 
-      gst_h265_decoder_do_output_picture (self, to_output);
+      gst_h265_decoder_do_output_picture (self, to_output, &ret);
     }
   }
 
-  return TRUE;
+  return ret;
 }
 
-static gboolean
+static GstFlowReturn
 gst_h265_decoder_start_current_picture (GstH265Decoder * self)
 {
   GstH265DecoderClass *klass;
   GstH265DecoderPrivate *priv = self->priv;
-  gboolean ret = TRUE;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   g_assert (priv->current_picture != NULL);
   g_assert (priv->active_sps != NULL);
   g_assert (priv->active_pps != NULL);
 
   if (!gst_h265_decoder_init_current_picture (self))
-    return FALSE;
+    return GST_FLOW_ERROR;
 
   /* Drop all RASL pictures having NoRaslOutputFlag is TRUE for the
    * associated IRAP picture */
@@ -1606,35 +1625,43 @@ gst_h265_decoder_start_current_picture (GstH265Decoder * self)
       priv->associated_irap_NoRaslOutputFlag) {
     GST_DEBUG_OBJECT (self, "Drop current picture");
     gst_h265_picture_replace (&priv->current_picture, NULL);
-    return TRUE;
+    return GST_FLOW_OK;
   }
 
   gst_h265_decoder_prepare_rps (self, &priv->current_slice,
       priv->current_picture);
 
-  gst_h265_decoder_dpb_init (self, &priv->current_slice, priv->current_picture);
+  ret = gst_h265_decoder_dpb_init (self,
+      &priv->current_slice, priv->current_picture);
+  if (ret != GST_FLOW_OK) {
+    GST_WARNING_OBJECT (self, "Failed to init dpb");
+    return ret;
+  }
 
   klass = GST_H265_DECODER_GET_CLASS (self);
-  if (klass->start_picture)
+  if (klass->start_picture) {
     ret = klass->start_picture (self, priv->current_picture,
         &priv->current_slice, priv->dpb);
 
-  if (!ret) {
-    GST_ERROR_OBJECT (self, "subclass does not want to start picture");
-    return FALSE;
+    if (ret != GST_FLOW_OK) {
+      GST_WARNING_OBJECT (self, "subclass does not want to start picture");
+      return ret;
+    }
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static void
 gst_h265_decoder_finish_picture (GstH265Decoder * self,
-    GstH265Picture * picture)
+    GstH265Picture * picture, GstFlowReturn * ret)
 {
   GstVideoDecoder *decoder = GST_VIDEO_DECODER (self);
   GstH265DecoderPrivate *priv = self->priv;
   const GstH265SPS *sps = priv->active_sps;
 
+  g_assert (ret != NULL);
+
   GST_LOG_OBJECT (self,
       "Finishing picture %p (poc %d), entries in DPB %d",
       picture, picture->pic_order_cnt, gst_h265_dpb_get_size (priv->dpb));
@@ -1667,37 +1694,40 @@ gst_h265_decoder_finish_picture (GstH265Decoder * self,
       break;
     }
 
-    gst_h265_decoder_do_output_picture (self, to_output);
+    gst_h265_decoder_do_output_picture (self, to_output, ret);
   }
-
-  return TRUE;
 }
 
-static gboolean
-gst_h265_decoder_finish_current_picture (GstH265Decoder * self)
+static void
+gst_h265_decoder_finish_current_picture (GstH265Decoder * self,
+    GstFlowReturn * ret)
 {
   GstH265DecoderPrivate *priv = self->priv;
   GstH265DecoderClass *klass;
-  gboolean ret = TRUE;
+  GstFlowReturn flow_ret = GST_FLOW_OK;
+
+  g_assert (ret != NULL);
 
   if (!priv->current_picture)
-    return TRUE;
+    return;
 
   klass = GST_H265_DECODER_GET_CLASS (self);
 
-  if (klass->end_picture)
-    ret = klass->end_picture (self, priv->current_picture);
+  if (klass->end_picture) {
+    flow_ret = klass->end_picture (self, priv->current_picture);
+    if (flow_ret != GST_FLOW_OK) {
+      GST_WARNING_OBJECT (self, "End picture failed");
+
+      /* continue to empty dpb */
+      UPDATE_FLOW_RETURN (ret, flow_ret);
+    }
+  }
 
   /* finish picture takes ownership of the picture */
-  ret = gst_h265_decoder_finish_picture (self, priv->current_picture);
+  gst_h265_decoder_finish_picture (self, priv->current_picture, &flow_ret);
   priv->current_picture = NULL;
 
-  if (!ret) {
-    GST_ERROR_OBJECT (self, "Failed to finish picture");
-    return FALSE;
-  }
-
-  return TRUE;
+  UPDATE_FLOW_RETURN (ret, flow_ret);
 }
 
 static void
@@ -1721,7 +1751,7 @@ gst_h265_decoder_handle_frame (GstVideoDecoder * decoder,
   GstH265NalUnit nalu;
   GstH265ParserResult pres;
   GstMapInfo map;
-  gboolean decode_ret = TRUE;
+  GstFlowReturn decode_ret = GST_FLOW_OK;
 
   GST_LOG_OBJECT (self,
       "handle frame, PTS: %" GST_TIME_FORMAT ", DTS: %"
@@ -1729,7 +1759,6 @@ gst_h265_decoder_handle_frame (GstVideoDecoder * decoder,
       GST_TIME_ARGS (GST_BUFFER_DTS (in_buf)));
 
   priv->current_frame = frame;
-  priv->last_ret = GST_FLOW_OK;
 
   gst_h265_decoder_reset_frame_state (self);
 
@@ -1744,7 +1773,7 @@ gst_h265_decoder_handle_frame (GstVideoDecoder * decoder,
     pres = gst_h265_parser_identify_nalu_hevc (priv->parser,
         map.data, 0, map.size, priv->nal_length_size, &nalu);
 
-    while (pres == GST_H265_PARSER_OK && decode_ret) {
+    while (pres == GST_H265_PARSER_OK && decode_ret == GST_FLOW_OK) {
       decode_ret = gst_h265_decoder_decode_nal (self,
           &nalu, GST_BUFFER_PTS (in_buf));
 
@@ -1759,7 +1788,7 @@ gst_h265_decoder_handle_frame (GstVideoDecoder * decoder,
     if (pres == GST_H265_PARSER_NO_NAL_END)
       pres = GST_H265_PARSER_OK;
 
-    while (pres == GST_H265_PARSER_OK && decode_ret) {
+    while (pres == GST_H265_PARSER_OK && decode_ret == GST_FLOW_OK) {
       decode_ret = gst_h265_decoder_decode_nal (self,
           &nalu, GST_BUFFER_PTS (in_buf));
 
@@ -1774,25 +1803,30 @@ gst_h265_decoder_handle_frame (GstVideoDecoder * decoder,
   gst_buffer_unmap (in_buf, &map);
   priv->current_frame = NULL;
 
-  if (!decode_ret) {
+  if (decode_ret != GST_FLOW_OK) {
     GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
-        ("Failed to decode data"), (NULL), priv->last_ret);
+        ("Failed to decode data"), (NULL), decode_ret);
     gst_video_decoder_drop_frame (decoder, frame);
 
     gst_h265_picture_clear (&priv->current_picture);
 
-    return priv->last_ret;
+    return decode_ret;
   }
 
   if (priv->current_picture) {
-    gst_h265_decoder_finish_current_picture (self);
+    gst_h265_decoder_finish_current_picture (self, &decode_ret);
     gst_video_codec_frame_unref (frame);
   } else {
     /* This picture was dropped */
     gst_video_decoder_release_frame (decoder, frame);
   }
 
-  return priv->last_ret;
+  if (decode_ret != GST_FLOW_OK) {
+    GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
+        ("Failed to decode data"), (NULL), decode_ret);
+  }
+
+  return decode_ret;
 }
 
 /**

diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.h b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.h
index a45629d..41d30bd 100644 (file)
--- a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.h
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.h
@@ -94,7 +94,7 @@ struct _GstH265DecoderClass
 {
   GstVideoDecoderClass parent_class;
 
-  gboolean      (*new_sequence)     (GstH265Decoder * decoder,
+  GstFlowReturn (*new_sequence)     (GstH265Decoder * decoder,
                                      const GstH265SPS * sps,
                                      gint max_dpb_size);
   /**
@@ -103,22 +103,22 @@ struct _GstH265DecoderClass
    * @frame: (transfer none): a #GstVideoCodecFrame
    * @picture: (transfer none): a #GstH265Picture
    */
-  gboolean      (*new_picture)      (GstH265Decoder * decoder,
+  GstFlowReturn (*new_picture)      (GstH265Decoder * decoder,
                                      GstVideoCodecFrame * frame,
                                      GstH265Picture * picture);
 
-  gboolean      (*start_picture)    (GstH265Decoder * decoder,
+  GstFlowReturn (*start_picture)    (GstH265Decoder * decoder,
                                      GstH265Picture * picture,
                                      GstH265Slice * slice,
                                      GstH265Dpb * dpb);
 
-  gboolean      (*decode_slice)     (GstH265Decoder * decoder,
+  GstFlowReturn (*decode_slice)     (GstH265Decoder * decoder,
                                      GstH265Picture * picture,
                                      GstH265Slice * slice,
                                      GArray * ref_pic_list0,
                                      GArray * ref_pic_list1);
 
-  gboolean      (*end_picture)      (GstH265Decoder * decoder,
+  GstFlowReturn (*end_picture)      (GstH265Decoder * decoder,
                                      GstH265Picture * picture);
   /**
    * GstH265Decoder:output_picture:

diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11h265dec.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11h265dec.cpp
index d594dec..91bd56d 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11h265dec.cpp
+++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11h265dec.cpp
@@ -113,19 +113,19 @@ static gboolean gst_d3d11_h265_dec_sink_event (GstVideoDecoder * decoder,
     GstEvent * event);
 
 /* GstH265Decoder */
-static gboolean gst_d3d11_h265_dec_new_sequence (GstH265Decoder * decoder,
+static GstFlowReturn gst_d3d11_h265_dec_new_sequence (GstH265Decoder * decoder,
     const GstH265SPS * sps, gint max_dpb_size);
-static gboolean gst_d3d11_h265_dec_new_picture (GstH265Decoder * decoder,
+static GstFlowReturn gst_d3d11_h265_dec_new_picture (GstH265Decoder * decoder,
     GstVideoCodecFrame * cframe, GstH265Picture * picture);
-static GstFlowReturn gst_d3d11_h265_dec_output_picture (GstH265Decoder *
-    decoder, GstVideoCodecFrame * frame, GstH265Picture * picture);
-static gboolean gst_d3d11_h265_dec_start_picture (GstH265Decoder * decoder,
+static GstFlowReturn gst_d3d11_h265_dec_start_picture (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice, GstH265Dpb * dpb);
-static gboolean gst_d3d11_h265_dec_decode_slice (GstH265Decoder * decoder,
+static GstFlowReturn gst_d3d11_h265_dec_decode_slice (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice,
     GArray * ref_pic_list0, GArray * ref_pic_list1);
-static gboolean gst_d3d11_h265_dec_end_picture (GstH265Decoder * decoder,
+static GstFlowReturn gst_d3d11_h265_dec_end_picture (GstH265Decoder * decoder,
     GstH265Picture * picture);
+static GstFlowReturn gst_d3d11_h265_dec_output_picture (GstH265Decoder *
+    decoder, GstVideoCodecFrame * frame, GstH265Picture * picture);
 
 static void
 gst_d3d11_h265_dec_class_init (GstD3D11H265DecClass * klass, gpointer data)
@@ -159,14 +159,14 @@ gst_d3d11_h265_dec_class_init (GstD3D11H265DecClass * klass, gpointer data)
       GST_DEBUG_FUNCPTR (gst_d3d11_h265_dec_new_sequence);
   h265decoder_class->new_picture =
       GST_DEBUG_FUNCPTR (gst_d3d11_h265_dec_new_picture);
-  h265decoder_class->output_picture =
-      GST_DEBUG_FUNCPTR (gst_d3d11_h265_dec_output_picture);
   h265decoder_class->start_picture =
       GST_DEBUG_FUNCPTR (gst_d3d11_h265_dec_start_picture);
   h265decoder_class->decode_slice =
       GST_DEBUG_FUNCPTR (gst_d3d11_h265_dec_decode_slice);
   h265decoder_class->end_picture =
       GST_DEBUG_FUNCPTR (gst_d3d11_h265_dec_end_picture);
+  h265decoder_class->output_picture =
+      GST_DEBUG_FUNCPTR (gst_d3d11_h265_dec_output_picture);
 }
 
 static void
@@ -316,7 +316,7 @@ gst_d3d11_h265_dec_sink_event (GstVideoDecoder * decoder, GstEvent * event)
   return GST_VIDEO_DECODER_CLASS (parent_class)->sink_event (decoder, event);
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_h265_dec_new_sequence (GstH265Decoder * decoder,
     const GstH265SPS * sps, gint max_dpb_size)
 {
@@ -399,7 +399,7 @@ gst_d3d11_h265_dec_new_sequence (GstH265Decoder * decoder,
 
     if (inner->out_format == GST_VIDEO_FORMAT_UNKNOWN) {
       GST_ERROR_OBJECT (self, "Could not support bitdepth/chroma format");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
 
     gst_video_info_set_format (&info,
@@ -412,19 +412,19 @@ gst_d3d11_h265_dec_new_sequence (GstH265Decoder * decoder,
             /* Additional 4 views margin for zero-copy rendering */
             max_dpb_size + 4)) {
       GST_ERROR_OBJECT (self, "Failed to create decoder");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
 
     if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
       GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_h265_dec_new_picture (GstH265Decoder * decoder,
     GstVideoCodecFrame * cframe, GstH265Picture * picture)
 {
@@ -436,7 +436,7 @@ gst_d3d11_h265_dec_new_picture (GstH265Decoder * decoder,
       GST_VIDEO_DECODER (decoder));
   if (!view_buffer) {
     GST_DEBUG_OBJECT (self, "No available output view buffer");
-    return FALSE;
+    return GST_FLOW_FLUSHING;
   }
 
   GST_LOG_OBJECT (self, "New output view buffer %" GST_PTR_FORMAT, view_buffer);
@@ -446,7 +446,7 @@ gst_d3d11_h265_dec_new_picture (GstH265Decoder * decoder,
 
   GST_LOG_OBJECT (self, "New h265picture %p", picture);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static void
@@ -652,7 +652,7 @@ init_pic_params (DXVA_PicParams_HEVC * params)
   }
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_h265_dec_start_picture (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice, GstH265Dpb * dpb)
 {
@@ -675,7 +675,7 @@ gst_d3d11_h265_dec_start_picture (GstH265Decoder * decoder,
       &view_id);
   if (!view) {
     GST_ERROR_OBJECT (self, "current picture does not have output view handle");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   init_pic_params (pic_params);
@@ -815,10 +815,10 @@ gst_d3d11_h265_dec_start_picture (GstH265Decoder * decoder,
   inner->slice_list.resize (0);
   inner->bitstream_buffer.resize (0);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_h265_dec_decode_slice (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice,
     GArray * ref_pic_list0, GArray * ref_pic_list1)
@@ -846,10 +846,10 @@ gst_d3d11_h265_dec_decode_slice (GstH265Decoder * decoder,
   memcpy (&inner->bitstream_buffer[0] + pos + start_code_size,
       slice->nalu.data + slice->nalu.offset, slice->nalu.size);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_h265_dec_end_picture (GstH265Decoder * decoder,
     GstH265Picture * picture)
 {
@@ -866,14 +866,14 @@ gst_d3d11_h265_dec_end_picture (GstH265Decoder * decoder,
 
   if (inner->bitstream_buffer.empty () || inner->slice_list.empty ()) {
     GST_ERROR_OBJECT (self, "No bitstream buffer to submit");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   view = gst_d3d11_h265_dec_get_output_view_from_picture (self, picture,
       &view_id);
   if (!view) {
     GST_ERROR_OBJECT (self, "current picture does not have output view handle");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   memset (&input_args, 0, sizeof (GstD3D11DecodeInputStreamArgs));
@@ -906,8 +906,10 @@ gst_d3d11_h265_dec_end_picture (GstH265Decoder * decoder,
     input_args.inverse_quantization_matrix_size = sizeof (DXVA_Qmatrix_HEVC);
   }
 
-  return gst_d3d11_decoder_decode_frame (inner->d3d11_decoder,
-      view, &input_args);
+  if (!gst_d3d11_decoder_decode_frame (inner->d3d11_decoder, view, &input_args))
+    return GST_FLOW_ERROR;
+
+  return GST_FLOW_OK;
 }
 
 static GstFlowReturn

diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265dec.c b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265dec.c
index 6c9c435..6b38e2e 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265dec.c
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265dec.c
@@ -134,18 +134,18 @@ static gboolean gst_nv_h265_dec_src_query (GstVideoDecoder * decoder,
     GstQuery * query);
 
 /* GstH265Decoder */
-static gboolean gst_nv_h265_dec_new_sequence (GstH265Decoder * decoder,
+static GstFlowReturn gst_nv_h265_dec_new_sequence (GstH265Decoder * decoder,
     const GstH265SPS * sps, gint max_dpb_size);
-static gboolean gst_nv_h265_dec_new_picture (GstH265Decoder * decoder,
+static GstFlowReturn gst_nv_h265_dec_new_picture (GstH265Decoder * decoder,
     GstVideoCodecFrame * frame, GstH265Picture * picture);
 static GstFlowReturn gst_nv_h265_dec_output_picture (GstH265Decoder *
     decoder, GstVideoCodecFrame * frame, GstH265Picture * picture);
-static gboolean gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
+static GstFlowReturn gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice, GstH265Dpb * dpb);
-static gboolean gst_nv_h265_dec_decode_slice (GstH265Decoder * decoder,
+static GstFlowReturn gst_nv_h265_dec_decode_slice (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice,
     GArray * ref_pic_list0, GArray * ref_pic_list1);
-static gboolean gst_nv_h265_dec_end_picture (GstH265Decoder * decoder,
+static GstFlowReturn gst_nv_h265_dec_end_picture (GstH265Decoder * decoder,
     GstH265Picture * picture);
 
 static void
@@ -316,7 +316,7 @@ gst_nv_h265_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
   return GST_VIDEO_DECODER_CLASS (parent_class)->src_query (decoder, query);
 }
 
-static gboolean
+static GstFlowReturn
 gst_nv_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
     gint max_dpb_size)
 {
@@ -377,7 +377,7 @@ gst_nv_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
 
     if (out_format == GST_VIDEO_FORMAT_UNKNOWN) {
       GST_ERROR_OBJECT (self, "Could not support bitdepth/chroma format");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
 
     gst_video_info_set_format (&info, out_format, self->width, self->height);
@@ -387,12 +387,12 @@ gst_nv_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
             /* Additional 2 buffers for margin */
             max_dpb_size + 2)) {
       GST_ERROR_OBJECT (self, "Failed to configure decoder");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
 
     if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
       GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
 
     self->last_sps = NULL;
@@ -400,10 +400,10 @@ gst_nv_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
     memset (&self->params, 0, sizeof (CUVIDPICPARAMS));
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_nv_h265_dec_new_picture (GstH265Decoder * decoder,
     GstVideoCodecFrame * cframe, GstH265Picture * picture)
 {
@@ -413,7 +413,7 @@ gst_nv_h265_dec_new_picture (GstH265Decoder * decoder,
   frame = gst_nv_decoder_new_frame (self->decoder);
   if (!frame) {
     GST_ERROR_OBJECT (self, "No available decoder frame");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   GST_LOG_OBJECT (self, "New decoder frame %p (index %d)", frame, frame->index);
@@ -421,7 +421,7 @@ gst_nv_h265_dec_new_picture (GstH265Decoder * decoder,
   gst_h265_picture_set_user_data (picture,
       frame, (GDestroyNotify) gst_nv_decoder_frame_unref);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static GstFlowReturn
@@ -642,7 +642,7 @@ gst_nv_h265_dec_reset_bitstream_params (GstNvH265Dec * self)
   self->params.pSliceDataOffsets = NULL;
 }
 
-static gboolean
+static GstFlowReturn
 gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice, GstH265Dpb * dpb)
 {
@@ -668,15 +668,15 @@ gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
   G_STATIC_ASSERT (sizeof (scaling_list->scaling_lists_32x32) ==
       sizeof (h265_params->ScalingList32x32));
 
-  g_return_val_if_fail (slice_header->pps != NULL, FALSE);
-  g_return_val_if_fail (slice_header->pps->sps != NULL, FALSE);
+  g_return_val_if_fail (slice_header->pps != NULL, GST_FLOW_ERROR);
+  g_return_val_if_fail (slice_header->pps->sps != NULL, GST_FLOW_ERROR);
 
   frame = gst_nv_h265_dec_get_decoder_frame_from_picture (self, picture);
 
   if (!frame) {
     GST_ERROR_OBJECT (self,
         "Couldn't get decoder frame frame picture %p", picture);
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   gst_nv_h265_dec_reset_bitstream_params (self);
@@ -702,7 +702,7 @@ gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
     gst_nv_h265_dec_picture_params_from_sps (self, sps, h265_params);
     if (!gst_nv_h265_dec_picture_params_from_pps (self, pps, h265_params)) {
       GST_ERROR_OBJECT (self, "Couldn't copy pps");
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
     self->last_sps = sps;
     self->last_pps = pps;
@@ -710,7 +710,7 @@ gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
     GST_DEBUG_OBJECT (self, "Update params from PPS");
     if (!gst_nv_h265_dec_picture_params_from_pps (self, pps, h265_params)) {
       GST_ERROR_OBJECT (self, "Couldn't copy pps");
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
     self->last_pps = pps;
   } else {
@@ -721,18 +721,18 @@ gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
   if (decoder->NumPocStCurrBefore >
       G_N_ELEMENTS (h265_params->RefPicSetStCurrBefore)) {
     GST_ERROR_OBJECT (self, "Too many RefPicSetStCurrBefore");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   if (decoder->NumPocStCurrAfter >
       G_N_ELEMENTS (h265_params->RefPicSetStCurrAfter)) {
     GST_ERROR_OBJECT (self, "Too many RefPicSetStCurrAfter");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   if (decoder->NumPocLtCurr > G_N_ELEMENTS (h265_params->RefPicSetLtCurr)) {
     GST_ERROR_OBJECT (self, "Too many RefPicSetLtCurr");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   /* Fill ref list */
@@ -759,7 +759,7 @@ gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
 
     if (num_ref_pic >= G_N_ELEMENTS (h265_params->RefPicIdx)) {
       GST_ERROR_OBJECT (self, "Too many reference frames");
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
 
     other_frame = gst_nv_h265_dec_get_decoder_frame_from_picture (self, other);
@@ -784,7 +784,7 @@ gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
 
     if (!decoder->RefPicSetStCurrBefore[i]) {
       GST_ERROR_OBJECT (self, "Empty RefPicSetStCurrBefore[%d]", i);
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
 
     other = decoder->RefPicSetStCurrBefore[i];
@@ -802,7 +802,7 @@ gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
 
     if (!decoder->RefPicSetStCurrAfter[i]) {
       GST_ERROR_OBJECT (self, "Empty RefPicSetStCurrAfter[%d]", i);
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
 
     other = decoder->RefPicSetStCurrAfter[i];
@@ -820,7 +820,7 @@ gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
 
     if (!decoder->RefPicSetLtCurr[i]) {
       GST_ERROR_OBJECT (self, "Empty RefPicSetLtCurr[%d]", i);
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
 
     other = decoder->RefPicSetLtCurr[i];
@@ -861,10 +861,10 @@ gst_nv_h265_dec_start_picture (GstH265Decoder * decoder,
         scaling_list->scaling_list_dc_coef_minus8_32x32[i] + 8;
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_nv_h265_dec_decode_slice (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice,
     GArray * ref_pic_list0, GArray * ref_pic_list1)
@@ -898,10 +898,10 @@ gst_nv_h265_dec_decode_slice (GstH265Decoder * decoder,
       slice->nalu.data + slice->nalu.offset, slice->nalu.size);
   self->bitstream_buffer_offset = new_size;
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_nv_h265_dec_end_picture (GstH265Decoder * decoder, GstH265Picture * picture)
 {
   GstNvH265Dec *self = GST_NV_H265_DEC (decoder);
@@ -918,10 +918,12 @@ gst_nv_h265_dec_end_picture (GstH265Decoder * decoder, GstH265Picture * picture)
 
   ret = gst_nv_decoder_decode_picture (self->decoder, &self->params);
 
-  if (!ret)
+  if (!ret) {
     GST_ERROR_OBJECT (self, "Failed to decode picture");
+    return GST_FLOW_ERROR;
+  }
 
-  return ret;
+  return GST_FLOW_OK;
 }
 
 typedef struct

diff --git a/subprojects/gst-plugins-bad/sys/va/gstvah265dec.c b/subprojects/gst-plugins-bad/sys/va/gstvah265dec.c
index 779d450..be10970 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/va/gstvah265dec.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvah265dec.c
@@ -475,7 +475,7 @@ _get_slice_data_byte_offset (GstH265SliceHdr * slice_hdr,
   return nal_header_bytes + (slice_hdr->header_size + 7) / 8 - epb_count;
 }
 
-static gboolean
+static GstFlowReturn
 gst_va_h265_dec_decode_slice (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice, GArray * ref_pic_list0,
     GArray * ref_pic_list1)
@@ -491,7 +491,7 @@ gst_va_h265_dec_decode_slice (GstH265Decoder * decoder,
   if (!_submit_previous_slice (base, va_pic)) {
     _replace_previous_slice (self, NULL, 0);
     GST_ERROR_OBJECT (base, "Failed to submit previous slice buffers");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   slice_param = &self->prev_slice.param;
@@ -559,7 +559,7 @@ gst_va_h265_dec_decode_slice (GstH265Decoder * decoder,
   _replace_previous_slice (self, slice->nalu.data + slice->nalu.offset,
       slice->nalu.size);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static void
@@ -649,7 +649,7 @@ _fill_screen_content_ext_parameter (GstVaH265Dec * decoder,
   }
 }
 
-static gboolean
+static GstFlowReturn
 gst_va_h265_dec_start_picture (GstH265Decoder * decoder,
     GstH265Picture * picture, GstH265Slice * slice, GstH265Dpb * dpb)
 {
@@ -798,7 +798,7 @@ gst_va_h265_dec_start_picture (GstH265Decoder * decoder,
       sizeof (*pic_param) : sizeof (pic_param->base);
   if (!gst_va_decoder_add_param_buffer (base->decoder, va_pic,
           VAPictureParameterBufferType, pic_param, pic_param_size))
-    return FALSE;
+    return GST_FLOW_ERROR;
 
   if (pps->scaling_list_data_present_flag ||
       (sps->scaling_list_enabled_flag
@@ -836,14 +836,16 @@ gst_va_h265_dec_start_picture (GstH265Decoder * decoder,
       iq_matrix.ScalingListDC32x32[i] =
           scaling_list->scaling_list_dc_coef_minus8_32x32[i] + 8;
 
-    return gst_va_decoder_add_param_buffer (base->decoder, va_pic,
-        VAIQMatrixBufferType, &iq_matrix, sizeof (iq_matrix));
+    if (!gst_va_decoder_add_param_buffer (base->decoder, va_pic,
+            VAIQMatrixBufferType, &iq_matrix, sizeof (iq_matrix))) {
+      return GST_FLOW_ERROR;
+    }
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
-static gboolean
+static GstFlowReturn
 gst_va_h265_dec_new_picture (GstH265Decoder * decoder,
     GstVideoCodecFrame * frame, GstH265Picture * picture)
 {
@@ -869,14 +871,14 @@ gst_va_h265_dec_new_picture (GstH265Decoder * decoder,
   GST_LOG_OBJECT (self, "New va decode picture %p - %#x", pic,
       gst_va_decode_picture_get_surface (pic));
 
-  return TRUE;
+  return GST_FLOW_OK;
 
 error:
   {
     GST_WARNING_OBJECT (self,
         "Failed to allocated output buffer, return %s",
         gst_flow_get_name (self->last_ret));
-    return FALSE;
+    return self->last_ret;
   }
 }
 
@@ -1033,7 +1035,7 @@ _get_profile (GstVaH265Dec * self, const GstH265SPS * sps, gint max_dpb_size)
   return VAProfileNone;
 }
 
-static gboolean
+static GstFlowReturn
 gst_va_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
     gint max_dpb_size)
 {
@@ -1064,12 +1066,12 @@ gst_va_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
 
   profile = _get_profile (self, sps, max_dpb_size);
   if (profile == VAProfileNone)
-    return FALSE;
+    return GST_FLOW_NOT_NEGOTIATED;
 
   rt_format = _get_rtformat (self, sps->bit_depth_luma_minus8 + 8,
       sps->bit_depth_chroma_minus8 + 8, sps->chroma_format_idc);
   if (rt_format == 0)
-    return FALSE;
+    return GST_FLOW_NOT_NEGOTIATED;
 
   if (!gst_va_decoder_config_is_equal (base->decoder, profile,
           rt_format, sps->width, sps->height)) {
@@ -1120,7 +1122,7 @@ gst_va_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
     self->need_negotiation = TRUE;
     if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
       GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return FALSE;
+      return GST_FLOW_NOT_NEGOTIATED;
     }
   }
 
@@ -1135,7 +1137,7 @@ gst_va_h265_dec_new_sequence (GstH265Decoder * decoder, const GstH265SPS * sps,
         1 << (high_precision_offsets_enabled_flag ? (bitdepthC - 1) : 7);
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static GstCaps *