From: Seungha Yang Date: Sun, 3 Oct 2021 10:14:07 +0000 (+0900) Subject: nvcodec: nvh264sldec: Add support for interlaced stream X-Git-Tag: 1.19.3~375 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=da7f4e0a69109eba2e74ea602fdd215facc2058f;p=platform%2Fupstream%2Fgstreamer.git nvcodec: nvh264sldec: Add support for interlaced stream Implement missing interlaced stream support Part-of: --- diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.c b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.c index a6cbcca..ae0ad35 100644 --- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.c +++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.c @@ -1472,8 +1472,8 @@ gst_nv_decoder_negotiate (GstNvDecoder * decoder, } info = &decoder->info; - state = gst_video_decoder_set_output_state (videodec, - GST_VIDEO_INFO_FORMAT (info), + state = gst_video_decoder_set_interlaced_output_state (videodec, + GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_INTERLACE_MODE (info), GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info), input_state); state->caps = gst_video_info_to_caps (&state->info); diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264dec.c b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264dec.c index 17f5875..f81385d 100644 --- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264dec.c +++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264dec.c @@ -89,9 +89,6 @@ struct _GstNvH264Dec GstVideoCodecState *output_state; - const GstH264SPS *last_sps; - const GstH264PPS *last_pps; - GstCudaContext *context; GstNvDecoder *decoder; CUVIDPICPARAMS params; @@ -112,6 +109,10 @@ struct _GstNvH264Dec guint bitdepth; guint chroma_format_idc; gint max_dpb_size; + + gboolean interlaced; + + GArray *ref_list; }; struct _GstNvH264DecClass @@ -123,6 +124,7 @@ struct _GstNvH264DecClass #define gst_nv_h264_dec_parent_class parent_class G_DEFINE_TYPE (GstNvH264Dec, gst_nv_h264_dec, GST_TYPE_H264_DECODER); +static void gst_nv_h264_decoder_dispose (GObject * object); static void gst_nv_h264_decoder_finalize (GObject * object); static void gst_nv_h264_dec_set_context (GstElement * element, GstContext * context); @@ -139,6 +141,8 @@ static gboolean gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps, gint max_dpb_size); static gboolean gst_nv_h264_dec_new_picture (GstH264Decoder * decoder, GstVideoCodecFrame * frame, GstH264Picture * picture); +static gboolean gst_nv_h264_dec_new_field_picture (GstH264Decoder * + decoder, const GstH264Picture * first_field, GstH264Picture * second_field); static GstFlowReturn gst_nv_h264_dec_output_picture (GstH264Decoder * decoder, GstVideoCodecFrame * frame, GstH264Picture * picture); static gboolean gst_nv_h264_dec_start_picture (GstH264Decoder * decoder, @@ -166,6 +170,7 @@ gst_nv_h264_dec_class_init (GstNvH264DecClass * klass) * Since: 1.18 */ + object_class->dispose = gst_nv_h264_decoder_dispose; object_class->finalize = gst_nv_h264_decoder_finalize; element_class->set_context = GST_DEBUG_FUNCPTR (gst_nv_h264_dec_set_context); @@ -181,6 +186,8 @@ gst_nv_h264_dec_class_init (GstNvH264DecClass * klass) GST_DEBUG_FUNCPTR (gst_nv_h264_dec_new_sequence); h264decoder_class->new_picture = GST_DEBUG_FUNCPTR (gst_nv_h264_dec_new_picture); + h264decoder_class->new_field_picture = + GST_DEBUG_FUNCPTR (gst_nv_h264_dec_new_field_picture); h264decoder_class->output_picture = GST_DEBUG_FUNCPTR (gst_nv_h264_dec_output_picture); h264decoder_class->start_picture = @@ -201,6 +208,20 @@ gst_nv_h264_dec_class_init (GstNvH264DecClass * klass) static void gst_nv_h264_dec_init (GstNvH264Dec * self) { + self->ref_list = g_array_sized_new (FALSE, TRUE, + sizeof (GstH264Picture *), 16); + g_array_set_clear_func (self->ref_list, + (GDestroyNotify) gst_h264_picture_clear); +} + +static void +gst_nv_h264_decoder_dispose (GObject * object) +{ + GstNvH264Dec *self = GST_NV_H264_DEC (object); + + g_clear_pointer (&self->ref_list, g_array_unref); + + G_OBJECT_CLASS (parent_class)->dispose (object); } static void @@ -246,6 +267,7 @@ gst_d3d11_h264_dec_reset (GstNvH264Dec * self) self->bitdepth = 0; self->chroma_format_idc = 0; self->max_dpb_size = 0; + self->interlaced = FALSE; } static gboolean @@ -344,6 +366,7 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps, GstNvH264Dec *self = GST_NV_H264_DEC (decoder); gint crop_width, crop_height; gboolean modified = FALSE; + gboolean interlaced; GST_LOG_OBJECT (self, "new sequence"); @@ -378,6 +401,13 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps, modified = TRUE; } + interlaced = !sps->frame_mbs_only_flag; + if (self->interlaced != interlaced) { + GST_INFO_OBJECT (self, "interlaced sequence changed"); + self->interlaced = interlaced; + modified = TRUE; + } + if (self->max_dpb_size < max_dpb_size) { GST_INFO_OBJECT (self, "Requires larger DPB size (%d -> %d)", self->max_dpb_size, max_dpb_size); @@ -408,6 +438,8 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps, } gst_video_info_set_format (&info, out_format, self->width, self->height); + if (self->interlaced) + GST_VIDEO_INFO_INTERLACE_MODE (&info) = GST_VIDEO_INTERLACE_MODE_MIXED; self->max_dpb_size = max_dpb_size; /* FIXME: add support cudaVideoCodec_H264_SVC and cudaVideoCodec_H264_MVC */ @@ -424,8 +456,6 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps, return FALSE; } - self->last_sps = NULL; - self->last_pps = NULL; memset (&self->params, 0, sizeof (CUVIDPICPARAMS)); } @@ -454,6 +484,27 @@ gst_nv_h264_dec_new_picture (GstH264Decoder * decoder, return TRUE; } +static gboolean +gst_nv_h264_dec_new_field_picture (GstH264Decoder * decoder, + const GstH264Picture * first_field, GstH264Picture * second_field) +{ + GstNvDecoderFrame *nv_frame; + + nv_frame = (GstNvDecoderFrame *) + gst_h264_picture_get_user_data ((GstH264Picture *) first_field); + if (!nv_frame) { + GST_ERROR_OBJECT (decoder, + "No decoder frame in the first picture %p", first_field); + return FALSE; + } + + gst_h264_picture_set_user_data (second_field, + gst_nv_decoder_frame_ref (nv_frame), + (GDestroyNotify) gst_nv_decoder_frame_unref); + + return TRUE; +} + static GstFlowReturn gst_nv_h264_dec_output_picture (GstH264Decoder * decoder, GstVideoCodecFrame * frame, GstH264Picture * picture) @@ -478,13 +529,24 @@ gst_nv_h264_dec_output_picture (GstH264Decoder * decoder, goto error; } + if (picture->buffer_flags != 0) { + gboolean interlaced = + (picture->buffer_flags & GST_VIDEO_BUFFER_FLAG_INTERLACED) != 0; + gboolean tff = (picture->buffer_flags & GST_VIDEO_BUFFER_FLAG_TFF) != 0; + + GST_TRACE_OBJECT (self, + "apply buffer flags 0x%x (interlaced %d, top-field-first %d)", + picture->buffer_flags, interlaced, tff); + GST_BUFFER_FLAG_SET (frame->output_buffer, picture->buffer_flags); + } + gst_h264_picture_unref (picture); return gst_video_decoder_finish_frame (vdec, frame); error: - gst_video_decoder_drop_frame (vdec, frame); gst_h264_picture_unref (picture); + gst_video_decoder_release_frame (vdec, frame); return GST_FLOW_ERROR; } @@ -531,7 +593,7 @@ gst_nv_h264_dec_picture_params_from_sps (GstNvH264Dec * self, const GstH264SPS * sps, gboolean field_pic, CUVIDH264PICPARAMS * params) { params->residual_colour_transform_flag = sps->separate_colour_plane_flag; - params->MbaffFrameFlag = sps->mb_adaptive_frame_field_flag && field_pic; + params->MbaffFrameFlag = sps->mb_adaptive_frame_field_flag && !field_pic; #define COPY_FIELD(f) \ (params)->f = (sps)->f @@ -591,6 +653,67 @@ gst_nv_h264_dec_reset_bitstream_params (GstNvH264Dec * self) self->params.pSliceDataOffsets = NULL; } +static void +gst_nv_h264_dec_fill_dpb (GstNvH264Dec * self, GstH264Picture * ref, + CUVIDH264DPBENTRY * dpb) +{ + GstNvDecoderFrame *frame; + + dpb->not_existing = ref->nonexisting; + dpb->PicIdx = -1; + + frame = gst_nv_h264_dec_get_decoder_frame_from_picture (self, ref); + if (!frame) { + dpb->not_existing = 1; + } else if (!dpb->not_existing) { + dpb->PicIdx = frame->index; + } + + if (dpb->not_existing) + return; + + if (GST_H264_PICTURE_IS_LONG_TERM_REF (ref)) { + dpb->FrameIdx = ref->long_term_frame_idx; + dpb->is_long_term = 1; + } else { + dpb->FrameIdx = ref->frame_num; + dpb->is_long_term = 0; + } + + switch (ref->field) { + case GST_H264_PICTURE_FIELD_FRAME: + dpb->FieldOrderCnt[0] = ref->top_field_order_cnt; + dpb->FieldOrderCnt[1] = ref->bottom_field_order_cnt; + dpb->used_for_reference = 0x3; + break; + case GST_H264_PICTURE_FIELD_TOP_FIELD: + dpb->FieldOrderCnt[0] = ref->top_field_order_cnt; + dpb->used_for_reference = 0x1; + if (ref->other_field) { + dpb->FieldOrderCnt[1] = ref->other_field->bottom_field_order_cnt; + dpb->used_for_reference |= 0x2; + } else { + dpb->FieldOrderCnt[1] = 0; + } + break; + case GST_H264_PICTURE_FIELD_BOTTOM_FIELD: + dpb->FieldOrderCnt[1] = ref->bottom_field_order_cnt; + dpb->used_for_reference = 0x2; + if (ref->other_field) { + dpb->FieldOrderCnt[0] = ref->other_field->bottom_field_order_cnt; + dpb->used_for_reference |= 0x1; + } else { + dpb->FieldOrderCnt[0] = 0; + } + break; + default: + dpb->FieldOrderCnt[0] = 0; + dpb->FieldOrderCnt[1] = 0; + dpb->used_for_reference = 0; + break; + } +} + static gboolean gst_nv_h264_dec_start_picture (GstH264Decoder * decoder, GstH264Picture * picture, GstH264Slice * slice, GstH264Dpb * dpb) @@ -602,8 +725,8 @@ gst_nv_h264_dec_start_picture (GstH264Decoder * decoder, const GstH264SPS *sps; const GstH264PPS *pps; GstNvDecoderFrame *frame; - GArray *dpb_array; - gint i; + GArray *ref_list = self->ref_list; + guint i, ref_frame_idx; g_return_val_if_fail (slice_header->pps != NULL, FALSE); g_return_val_if_fail (slice_header->pps->sequence != NULL, FALSE); @@ -623,14 +746,27 @@ gst_nv_h264_dec_start_picture (GstH264Decoder * decoder, /* FIXME: update sps/pps related params only when it's required */ params->PicWidthInMbs = sps->pic_width_in_mbs_minus1 + 1; - params->FrameHeightInMbs = sps->pic_height_in_map_units_minus1 + 1; + if (!sps->frame_mbs_only_flag) { + params->FrameHeightInMbs = (sps->pic_height_in_map_units_minus1 + 1) << 1; + } else { + params->FrameHeightInMbs = sps->pic_height_in_map_units_minus1 + 1; + } params->CurrPicIdx = frame->index; - /* TODO: verifiy interlaced */ - params->field_pic_flag = picture->field != GST_H264_PICTURE_FIELD_FRAME; + params->field_pic_flag = slice_header->field_pic_flag; params->bottom_field_flag = picture->field == GST_H264_PICTURE_FIELD_BOTTOM_FIELD; - /* TODO: set second_field here */ - params->second_field = 0; + params->second_field = picture->second_field; + + if (picture->field == GST_H264_PICTURE_FIELD_TOP_FIELD) { + h264_params->CurrFieldOrderCnt[0] = picture->top_field_order_cnt; + h264_params->CurrFieldOrderCnt[1] = 0; + } else if (picture->field == GST_H264_PICTURE_FIELD_BOTTOM_FIELD) { + h264_params->CurrFieldOrderCnt[0] = 0; + h264_params->CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt; + } else { + h264_params->CurrFieldOrderCnt[0] = picture->top_field_order_cnt; + h264_params->CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt; + } /* nBitstreamDataLen, pBitstreamData, nNumSlices and pSliceDataOffsets * will be set later */ @@ -642,68 +778,33 @@ gst_nv_h264_dec_start_picture (GstH264Decoder * decoder, h264_params->frame_num = picture->frame_num; h264_params->ref_pic_flag = GST_H264_PICTURE_IS_REF (picture); - /* FIXME: should be updated depending on field type? */ - h264_params->CurrFieldOrderCnt[0] = picture->top_field_order_cnt; - h264_params->CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt; - - if (!self->last_sps || self->last_sps != sps) { - GST_DEBUG_OBJECT (self, "Update params from SPS and PPS"); - gst_nv_h264_dec_picture_params_from_sps (self, - sps, slice_header->field_pic_flag, h264_params); - gst_nv_h264_dec_picture_params_from_pps (self, pps, h264_params); - self->last_sps = sps; - self->last_pps = pps; - } else if (!self->last_pps || self->last_pps != pps) { - GST_DEBUG_OBJECT (self, "Update params from PPS"); - gst_nv_h264_dec_picture_params_from_pps (self, pps, h264_params); - self->last_pps = pps; - } else { - GST_TRACE_OBJECT (self, "SPS and PPS were not updated"); - } - - memset (&h264_params->dpb, 0, sizeof (h264_params->dpb)); - for (i = 0; i < G_N_ELEMENTS (h264_params->dpb); i++) - h264_params->dpb[i].PicIdx = -1; - - dpb_array = gst_h264_dpb_get_pictures_all (dpb); - for (i = 0; i < dpb_array->len && i < G_N_ELEMENTS (h264_params->dpb); i++) { - GstH264Picture *other = g_array_index (dpb_array, GstH264Picture *, i); - GstNvDecoderFrame *other_frame; - gint picture_index = -1; - CUVIDH264DPBENTRY *dpb = &h264_params->dpb[i]; - - if (!other->ref) - continue; - other_frame = gst_nv_h264_dec_get_decoder_frame_from_picture (self, other); + gst_nv_h264_dec_picture_params_from_sps (self, + sps, slice_header->field_pic_flag, h264_params); + gst_nv_h264_dec_picture_params_from_pps (self, pps, h264_params); - if (other_frame) - picture_index = other_frame->index; + ref_frame_idx = 0; + g_array_set_size (ref_list, 0); - dpb->PicIdx = picture_index; - if (GST_H264_PICTURE_IS_LONG_TERM_REF (other)) { - dpb->FrameIdx = other->long_term_frame_idx; - dpb->is_long_term = 1; - } else { - dpb->FrameIdx = other->frame_num; - dpb->is_long_term = 0; - } - - dpb->not_existing = other->nonexisting; - if (dpb->not_existing && dpb->PicIdx != -1) { - GST_WARNING_OBJECT (self, - "Non-existing frame has valid picture index %d", dpb->PicIdx); - dpb->PicIdx = -1; - } - - /* FIXME: 1=top_field, 2=bottom_field, 3=both_fields */ - dpb->used_for_reference = 3; + memset (&h264_params->dpb, 0, sizeof (h264_params->dpb)); + gst_h264_dpb_get_pictures_short_term_ref (dpb, FALSE, FALSE, ref_list); + for (i = 0; ref_frame_idx < 16 && i < ref_list->len; i++) { + GstH264Picture *other = g_array_index (ref_list, GstH264Picture *, i); + gst_nv_h264_dec_fill_dpb (self, other, &h264_params->dpb[ref_frame_idx]); + ref_frame_idx++; + } + g_array_set_size (ref_list, 0); - dpb->FieldOrderCnt[0] = other->top_field_order_cnt; - dpb->FieldOrderCnt[1] = other->bottom_field_order_cnt; + gst_h264_dpb_get_pictures_long_term_ref (dpb, FALSE, ref_list); + for (i = 0; ref_frame_idx < 16 && i < ref_list->len; i++) { + GstH264Picture *other = g_array_index (ref_list, GstH264Picture *, i); + gst_nv_h264_dec_fill_dpb (self, other, &h264_params->dpb[ref_frame_idx]); + ref_frame_idx++; } + g_array_set_size (ref_list, 0); - g_array_unref (dpb_array); + for (i = ref_frame_idx; i < 16; i++) + h264_params->dpb[i].PicIdx = -1; return TRUE; }