From c6ec7f3ea4f2882e9ddb94cad46772dada92d42a Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Sat, 2 Oct 2021 20:21:41 +0900 Subject: [PATCH] codecs: vp9decoder: add support for render delay Some decoding APIs support delayed output for performance reasons. One example would be to request decoding for multiple frames and then query for the oldest frame in the output queue. This also increases throughput for transcoding and improves seek performance when supported by the underlying backend. Introduce support in the vp9 base class, so that backends that support render delays can actually implement it. Co-authored by Seungha Yang Part-of: --- .../gst-libs/gst/codecs/gstvp9decoder.c | 109 ++++++++++++++++++++- .../gst-libs/gst/codecs/gstvp9decoder.h | 16 +++ 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.c b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.c index 4e37e0b..e482009 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.c @@ -58,6 +58,7 @@ #include #endif +#include #include "gstvp9decoder.h" GST_DEBUG_CATEGORY (gst_vp9_decoder_debug); @@ -75,8 +76,19 @@ struct _GstVp9DecoderPrivate GstVp9Dpb *dpb; gboolean wait_keyframe; + /* controls how many frames to delay when calling output_picture() */ + guint preferred_output_delay; + GstQueueArray *output_queue; + gboolean is_live; }; +typedef struct +{ + GstVideoCodecFrame *frame; + GstVp9Picture *picture; + GstVp9Decoder *self; +} GstVp9DecoderOutputFrame; + #define parent_class gst_vp9_decoder_parent_class G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstVp9Decoder, gst_vp9_decoder, GST_TYPE_VIDEO_DECODER, @@ -94,6 +106,10 @@ static GstFlowReturn gst_vp9_decoder_drain (GstVideoDecoder * decoder); static GstFlowReturn gst_vp9_decoder_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame); +static void +gst_vp9_decoder_clear_output_frame (GstVp9DecoderOutputFrame * output_frame); +static void gst_vp9_decoder_drain_output_queue (GstVp9Decoder * self, + guint num, GstFlowReturn * ret); static void gst_vp9_decoder_class_init (GstVp9DecoderClass * klass) @@ -128,6 +144,11 @@ gst_vp9_decoder_start (GstVideoDecoder * decoder) priv->dpb = gst_vp9_dpb_new (); priv->wait_keyframe = TRUE; + priv->output_queue = + gst_queue_array_new_for_struct (sizeof (GstVp9DecoderOutputFrame), 1); + gst_queue_array_set_clear_func (priv->output_queue, + (GDestroyNotify) gst_vp9_decoder_clear_output_frame); + return TRUE; } @@ -140,6 +161,7 @@ gst_vp9_decoder_stop (GstVideoDecoder * decoder) g_clear_pointer (&self->input_state, gst_video_codec_state_unref); g_clear_pointer (&priv->parser, gst_vp9_stateful_parser_free); g_clear_pointer (&priv->dpb, gst_vp9_dpb_free); + gst_queue_array_free (priv->output_queue); return TRUE; } @@ -170,6 +192,14 @@ gst_vp9_decoder_check_codec_change (GstVp9Decoder * self, GstVp9DecoderClass *klass = GST_VP9_DECODER_GET_CLASS (self); priv->had_sequence = TRUE; + + if (klass->get_preferred_output_delay) { + priv->preferred_output_delay = + klass->get_preferred_output_delay (self, priv->is_live); + } else { + priv->preferred_output_delay = 0; + } + if (klass->new_sequence) ret = klass->new_sequence (self, frame_hdr); @@ -186,6 +216,7 @@ gst_vp9_decoder_set_format (GstVideoDecoder * decoder, { GstVp9Decoder *self = GST_VP9_DECODER (decoder); GstVp9DecoderPrivate *priv = self->priv; + GstQuery *query; GST_DEBUG_OBJECT (decoder, "Set format"); @@ -197,6 +228,11 @@ gst_vp9_decoder_set_format (GstVideoDecoder * decoder, priv->width = GST_VIDEO_INFO_WIDTH (&state->info); priv->height = GST_VIDEO_INFO_HEIGHT (&state->info); + query = gst_query_new_latency (); + if (gst_pad_peer_query (GST_VIDEO_DECODER_SINK_PAD (self), query)) + gst_query_parse_latency (query, &priv->is_live, NULL, NULL); + gst_query_unref (query); + return TRUE; } @@ -209,16 +245,20 @@ gst_vp9_decoder_reset (GstVp9Decoder * self) gst_vp9_dpb_clear (priv->dpb); priv->wait_keyframe = TRUE; + gst_queue_array_clear (priv->output_queue); } static GstFlowReturn gst_vp9_decoder_finish (GstVideoDecoder * decoder) { + GstFlowReturn ret = GST_FLOW_OK; + GST_DEBUG_OBJECT (decoder, "finish"); + gst_vp9_decoder_drain_output_queue (GST_VP9_DECODER (decoder), 0, &ret); gst_vp9_decoder_reset (GST_VP9_DECODER (decoder)); - return GST_FLOW_OK; + return ret; } static gboolean @@ -234,11 +274,29 @@ gst_vp9_decoder_flush (GstVideoDecoder * decoder) static GstFlowReturn gst_vp9_decoder_drain (GstVideoDecoder * decoder) { + GstFlowReturn ret = GST_FLOW_OK; + GST_DEBUG_OBJECT (decoder, "drain"); + gst_vp9_decoder_drain_output_queue (GST_VP9_DECODER (decoder), 0, &ret); gst_vp9_decoder_reset (GST_VP9_DECODER (decoder)); - return GST_FLOW_OK; + return ret; +} + +static void +gst_vp9_decoder_clear_output_frame (GstVp9DecoderOutputFrame * output_frame) +{ + if (!output_frame) + return; + + if (output_frame->frame) { + gst_video_decoder_release_frame (GST_VIDEO_DECODER (output_frame->self), + output_frame->frame); + output_frame->frame = NULL; + } + + gst_vp9_picture_clear (&output_frame->picture); } static GstFlowReturn @@ -256,6 +314,7 @@ gst_vp9_decoder_handle_frame (GstVideoDecoder * decoder, GstFlowReturn ret = GST_FLOW_OK; gboolean intra_only = FALSE; gboolean check_codec_change = FALSE; + GstVp9DecoderOutputFrame output_frame; GST_LOG_OBJECT (self, "handle frame %" GST_PTR_FORMAT, in_buf); @@ -397,10 +456,14 @@ gst_vp9_decoder_handle_frame (GstVideoDecoder * decoder, ret = gst_video_decoder_finish_frame (GST_VIDEO_DECODER (self), frame); } else { - g_assert (klass->output_picture); - ret = klass->output_picture (self, frame, picture); + output_frame.frame = frame; + output_frame.picture = picture; + output_frame.self = self; + gst_queue_array_push_tail_struct (priv->output_queue, &output_frame); } + gst_vp9_decoder_drain_output_queue (self, priv->preferred_output_delay, &ret); + if (ret == GST_FLOW_ERROR) { GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE, ("Failed to decode data"), (NULL), ret); @@ -430,3 +493,41 @@ error: return ret; } } + +static void +gst_vp9_decoder_drain_output_queue (GstVp9Decoder * self, guint num, + GstFlowReturn * ret) +{ + GstVp9DecoderPrivate *priv = self->priv; + GstVp9DecoderClass *klass = GST_VP9_DECODER_GET_CLASS (self); + + g_assert (klass->output_picture); + + while (gst_queue_array_get_length (priv->output_queue) > num) { + GstVp9DecoderOutputFrame *output_frame = (GstVp9DecoderOutputFrame *) + gst_queue_array_pop_head_struct (priv->output_queue); + /* Output queued frames whatever the return value is, in order to empty + * the queue */ + GstFlowReturn flow_ret = klass->output_picture (self, + output_frame->frame, output_frame->picture); + + /* Then, update @ret with new flow return value only if @ret was + * GST_FLOW_OK. This is to avoid pattern such that + * ```c + * GstFlowReturn my_return = GST_FLOW_OK; + * do something + * + * if (my_return == GST_FLOW_OK) { + * my_return = gst_vp9_decoder_drain_output_queue (); + * } else { + * // Ignore flow return of this method, but current `my_return` error code + * gst_vp9_decoder_drain_output_queue (); + * } + * + * return my_return; + * ``` + */ + if (*ret == GST_FLOW_OK) + *ret = flow_ret; + } +} diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.h b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.h index 4f5ccdb..76983f5 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.h +++ b/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.h @@ -171,6 +171,22 @@ struct _GstVp9DecoderClass GstVideoCodecFrame * frame, GstVp9Picture * picture); + /** + * GstVp9DecoderClass::get_preferred_output_delay: + * @decoder: a #GstVp9Decoder + * @is_live: whether upstream is live or not + * + * Optional. Retrieve the preferred output delay from child classes. + * controls how many frames to delay when calling + * GstVp9DecoderClass::output_picture + * + * Returns: the number of perferred delayed output frame + * + * Since: 1.20 + */ + guint (*get_preferred_output_delay) (GstVp9Decoder * decoder, + gboolean is_live); + /*< private >*/ gpointer padding[GST_PADDING_LARGE]; }; -- 2.7.4