From 39f800c44970732f5dc9cad4a6ebc0f8a8a0e6f9 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Fri, 9 Aug 2019 18:47:14 +0900 Subject: [PATCH] nvdec: Re-negotiate whenever output format is changed Input stream might be silently changed without ::set_format() call. Since nvdec has internal parser, nvdec element can figure out the format change by itself. --- sys/nvcodec/gstnvdec.c | 310 +++++++++++++++++++++++++++---------------------- sys/nvcodec/gstnvdec.h | 5 +- 2 files changed, 171 insertions(+), 144 deletions(-) diff --git a/sys/nvcodec/gstnvdec.c b/sys/nvcodec/gstnvdec.c index 6896f3c..e945bc3 100644 --- a/sys/nvcodec/gstnvdec.c +++ b/sys/nvcodec/gstnvdec.c @@ -190,20 +190,172 @@ gst_nvdec_init (GstNvDec * nvdec) gst_video_decoder_set_needs_format (GST_VIDEO_DECODER (nvdec), TRUE); } +static cudaVideoSurfaceFormat +get_cuda_surface_format_from_gst (GstVideoFormat format) +{ + switch (format) { + case GST_VIDEO_FORMAT_NV12: + return cudaVideoSurfaceFormat_NV12; + case GST_VIDEO_FORMAT_P010_10LE: + case GST_VIDEO_FORMAT_P010_10BE: + case GST_VIDEO_FORMAT_P016_LE: + case GST_VIDEO_FORMAT_P016_BE: + return cudaVideoSurfaceFormat_P016; + case GST_VIDEO_FORMAT_Y444: + return cudaVideoSurfaceFormat_YUV444; + case GST_VIDEO_FORMAT_Y444_16LE: + case GST_VIDEO_FORMAT_Y444_16BE: + return cudaVideoSurfaceFormat_YUV444_16Bit; + default: + g_assert_not_reached (); + break; + } + + return cudaVideoSurfaceFormat_NV12; +} + static gboolean parser_sequence_callback (GstNvDec * nvdec, CUVIDEOFORMAT * format) { - guint width, height, fps_n, fps_d; + guint width, height; CUVIDDECODECREATEINFO create_info = { 0, }; - GstVideoFormat out_format = GST_VIDEO_FORMAT_NV12; - GstVideoInfo *info = &nvdec->input_state->info; + GstVideoFormat out_format; + GstVideoInfo *in_info = &nvdec->input_state->info; + GstVideoInfo *out_info = &nvdec->out_info; + GstVideoInfo prev_out_info = *out_info; GstCudaContext *ctx = nvdec->cuda_ctx; + GstStructure *in_s = NULL; + gboolean updata = FALSE; width = format->display_area.right - format->display_area.left; height = format->display_area.bottom - format->display_area.top; + + switch (format->chroma_format) { + case cudaVideoChromaFormat_444: + if (format->bit_depth_luma_minus8 == 0) { + out_format = GST_VIDEO_FORMAT_Y444; + } else if (format->bit_depth_luma_minus8 == 2 || + format->bit_depth_luma_minus8 == 4) { +#if G_BYTE_ORDER == G_LITTLE_ENDIAN + out_format = GST_VIDEO_FORMAT_Y444_16LE; +#else + out_format = GST_VIDEO_FORMAT_Y444_16BE; +#endif + } else { + GST_ERROR_OBJECT (nvdec, "Unknown 4:4:4 format bitdepth %d", + format->bit_depth_luma_minus8 + 8); + + nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED; + return FALSE; + } + break; + case cudaVideoChromaFormat_420: + if (format->bit_depth_luma_minus8 == 0) { + out_format = GST_VIDEO_FORMAT_NV12; + } else if (format->bit_depth_luma_minus8 == 2) { +#if G_BYTE_ORDER == G_LITTLE_ENDIAN + out_format = GST_VIDEO_FORMAT_P010_10LE; +#else + out_format = GST_VIDEO_FORMAT_P010_10BE; +#endif + } else if (format->bit_depth_luma_minus8 == 4) { +#if G_BYTE_ORDER == G_LITTLE_ENDIAN + out_format = GST_VIDEO_FORMAT_P016_LE; +#else + out_format = GST_VIDEO_FORMAT_P016_BE; +#endif + } else { + GST_ERROR_OBJECT (nvdec, "Unknown 4:2:0 format bitdepth %d", + format->bit_depth_luma_minus8 + 8); + + nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED; + return FALSE; + } + break; + default: + GST_ERROR_OBJECT (nvdec, "unhandled chroma format %d, bitdepth %d", + format->chroma_format, format->bit_depth_luma_minus8 + 8); + + nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED; + return FALSE; + } + + GST_DEBUG_OBJECT (nvdec, + "out format: %s", gst_video_format_to_string (out_format)); + GST_DEBUG_OBJECT (nvdec, "width: %u, height: %u", width, height); - if (!nvdec->decoder || (nvdec->width != width || nvdec->height != height)) { + gst_video_info_set_format (out_info, out_format, width, height); + GST_VIDEO_INFO_FPS_N (out_info) = GST_VIDEO_INFO_FPS_N (in_info); + GST_VIDEO_INFO_FPS_D (out_info) = GST_VIDEO_INFO_FPS_D (in_info); + + if (GST_VIDEO_INFO_FPS_N (out_info) < 1 || + GST_VIDEO_INFO_FPS_D (out_info) < 1) { + GST_VIDEO_INFO_FPS_N (out_info) = format->frame_rate.numerator; + GST_VIDEO_INFO_FPS_D (out_info) = MAX (1, format->frame_rate.denominator); + } + + GST_LOG_OBJECT (nvdec, + "Reading colorimetry information full-range %d matrix %d transfer %d primaries %d", + format->video_signal_description.video_full_range_flag, + format->video_signal_description.matrix_coefficients, + format->video_signal_description.transfer_characteristics, + format->video_signal_description.color_primaries); + + if (nvdec->input_state->caps) + in_s = gst_caps_get_structure (nvdec->input_state->caps, 0); + + /* Set colorimetry when upstream did not provide it */ + if (in_s && !gst_structure_has_field (in_s, "colorimetry")) { + GstVideoColorimetry colorimetry = { 0, }; + + if (format->video_signal_description.video_full_range_flag) + colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255; + else + colorimetry.range = GST_VIDEO_COLOR_RANGE_16_235; + + colorimetry.primaries = + gst_video_color_primaries_from_iso + (format->video_signal_description.color_primaries); + + colorimetry.transfer = + gst_video_color_transfer_from_iso + (format->video_signal_description.transfer_characteristics); + + colorimetry.matrix = + gst_video_color_matrix_from_iso + (format->video_signal_description.matrix_coefficients); + + /* Use a colorimetry having at least one valid colorimetry entry, + * because we don't know whether the returned + * colorimetry (by nvdec) was actually parsed information or not. + * Otherwise let GstVideoInfo handle it with default colorimetry */ + if (colorimetry.primaries != GST_VIDEO_COLOR_PRIMARIES_UNKNOWN || + colorimetry.transfer != GST_VIDEO_TRANSFER_UNKNOWN || + colorimetry.matrix != GST_VIDEO_COLOR_MATRIX_UNKNOWN) { + GST_DEBUG_OBJECT (nvdec, + "Found valid colorimetry, update output colorimetry"); + out_info->colorimetry = colorimetry; + } + } else { + out_info->colorimetry = in_info->colorimetry; + } + + if (format->progressive_sequence) { + out_info->interlace_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE; + + /* nvdec doesn't seem to deal with interlacing with hevc so rely + * on upstream's value */ + if (format->codec == cudaVideoCodec_HEVC) { + out_info->interlace_mode = in_info->interlace_mode; + } + } else { + out_info->interlace_mode = GST_VIDEO_INTERLACE_MODE_MIXED; + } + + if (!nvdec->decoder || !gst_video_info_is_equal (out_info, &prev_out_info)) { + updata = TRUE; + if (!gst_cuda_context_push (ctx)) { GST_ERROR_OBJECT (nvdec, "failed to lock CUDA context"); goto error; @@ -229,66 +381,8 @@ parser_sequence_callback (GstNvDec * nvdec, CUVIDEOFORMAT * format) create_info.display_area.top = format->display_area.top; create_info.display_area.right = format->display_area.right; create_info.display_area.bottom = format->display_area.bottom; + create_info.OutputFormat = get_cuda_surface_format_from_gst (out_format); create_info.bitDepthMinus8 = format->bit_depth_luma_minus8; - create_info.OutputFormat = cudaVideoSurfaceFormat_NV12; - switch (format->chroma_format) { - case cudaVideoChromaFormat_444: - if (format->bit_depth_luma_minus8 == 0) { - out_format = GST_VIDEO_FORMAT_Y444; - create_info.OutputFormat = cudaVideoSurfaceFormat_YUV444; - } else if (format->bit_depth_luma_minus8 == 2 || - format->bit_depth_luma_minus8 == 4) { -#if G_BYTE_ORDER == G_LITTLE_ENDIAN - out_format = GST_VIDEO_FORMAT_Y444_16LE; -#else - out_format = GST_VIDEO_FORMAT_Y444_16BE; -#endif - create_info.OutputFormat = cudaVideoSurfaceFormat_YUV444_16Bit; - } else { - GST_ERROR_OBJECT (nvdec, "Unknown 4:4:4 format bitdepth %d", - format->bit_depth_luma_minus8 + 8); - - nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED; - return FALSE; - } - break; - case cudaVideoChromaFormat_420: - if (format->bit_depth_luma_minus8 == 0) { - out_format = GST_VIDEO_FORMAT_NV12; - create_info.OutputFormat = cudaVideoSurfaceFormat_NV12; - } else if (format->bit_depth_luma_minus8 == 2) { -#if G_BYTE_ORDER == G_LITTLE_ENDIAN - out_format = GST_VIDEO_FORMAT_P010_10LE; -#else - out_format = GST_VIDEO_FORMAT_P010_10BE; -#endif - create_info.OutputFormat = cudaVideoSurfaceFormat_P016; - } else if (format->bit_depth_luma_minus8 == 4) { -#if G_BYTE_ORDER == G_LITTLE_ENDIAN - out_format = GST_VIDEO_FORMAT_P016_LE; -#else - out_format = GST_VIDEO_FORMAT_P016_BE; -#endif - create_info.OutputFormat = cudaVideoSurfaceFormat_P016; - } else { - GST_ERROR_OBJECT (nvdec, "Unknown 4:2:0 format bitdepth %d", - format->bit_depth_luma_minus8 + 8); - - nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED; - return FALSE; - } - break; - default: - GST_ERROR_OBJECT (nvdec, "unhandled chroma format %d, bitdepth %d", - format->chroma_format, format->bit_depth_luma_minus8 + 8); - - nvdec->last_ret = GST_FLOW_NOT_NEGOTIATED; - return FALSE; - } - - GST_DEBUG_OBJECT (nvdec, - "out format: %s", gst_video_format_to_string (out_format)); - create_info.DeinterlaceMode = cudaVideoDeinterlaceMode_Weave; create_info.ulTargetWidth = width; create_info.ulTargetHeight = height; @@ -311,86 +405,19 @@ parser_sequence_callback (GstNvDec * nvdec, CUVIDEOFORMAT * format) } } - fps_n = GST_VIDEO_INFO_FPS_N (info); - fps_d = GST_VIDEO_INFO_FPS_D (info); - - if (fps_n < 1 || fps_d < 1) { - fps_n = format->frame_rate.numerator; - fps_d = MAX (1, format->frame_rate.denominator); - } - - if (!gst_pad_has_current_caps (GST_VIDEO_DECODER_SRC_PAD (nvdec)) - || width != nvdec->width || height != nvdec->height - || fps_n != nvdec->fps_n || fps_d != nvdec->fps_d) { + if (!gst_pad_has_current_caps (GST_VIDEO_DECODER_SRC_PAD (nvdec)) || updata) { GstVideoCodecState *state; GstVideoInfo *vinfo; - GstStructure *in_s = NULL; - - nvdec->width = width; - nvdec->height = height; - nvdec->fps_n = fps_n; - nvdec->fps_d = fps_d; state = gst_video_decoder_set_output_state (GST_VIDEO_DECODER (nvdec), - out_format, nvdec->width, nvdec->height, nvdec->input_state); + GST_VIDEO_INFO_FORMAT (out_info), GST_VIDEO_INFO_WIDTH (out_info), + GST_VIDEO_INFO_HEIGHT (out_info), nvdec->input_state); vinfo = &state->info; - vinfo->fps_n = fps_n; - vinfo->fps_d = fps_d; - if (format->progressive_sequence) { - vinfo->interlace_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE; - - /* nvdec doesn't seem to deal with interlacing with hevc so rely - * on upstream's value */ - if (format->codec == cudaVideoCodec_HEVC) { - vinfo->interlace_mode = nvdec->input_state->info.interlace_mode; - } - } else { - vinfo->interlace_mode = GST_VIDEO_INTERLACE_MODE_MIXED; - } - GST_LOG_OBJECT (nvdec, - "Reading colorimetry information full-range %d matrix %d transfer %d primaries %d", - format->video_signal_description.video_full_range_flag, - format->video_signal_description.matrix_coefficients, - format->video_signal_description.transfer_characteristics, - format->video_signal_description.color_primaries); - - if (nvdec->input_state->caps) - in_s = gst_caps_get_structure (nvdec->input_state->caps, 0); - - /* Set colorimetry when upstream did not provide it */ - if (in_s && !gst_structure_has_field (in_s, "colorimetry")) { - GstVideoColorimetry colorimetry = { 0, }; - - if (format->video_signal_description.video_full_range_flag) - colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255; - else - colorimetry.range = GST_VIDEO_COLOR_RANGE_16_235; - - colorimetry.primaries = - gst_video_color_primaries_from_iso - (format->video_signal_description.color_primaries); - - colorimetry.transfer = - gst_video_color_transfer_from_iso - (format->video_signal_description.transfer_characteristics); - - colorimetry.matrix = - gst_video_color_matrix_from_iso - (format->video_signal_description.matrix_coefficients); - - /* Use a colorimetry having at least one valid colorimetry entry, - * because we don't know whether the returned - * colorimetry (by nvdec) was actually parsed information or not. - * Otherwise let GstVideoInfo handle it with default colorimetry */ - if (colorimetry.primaries != GST_VIDEO_COLOR_PRIMARIES_UNKNOWN || - colorimetry.transfer != GST_VIDEO_TRANSFER_UNKNOWN || - colorimetry.matrix != GST_VIDEO_COLOR_MATRIX_UNKNOWN) { - GST_DEBUG_OBJECT (nvdec, - "Found valid colorimetry, update output colorimetry"); - vinfo->colorimetry = colorimetry; - } - } + /* update output info with CUvidparser provided one */ + vinfo->interlace_mode = out_info->interlace_mode; + vinfo->fps_n = out_info->fps_n; + vinfo->fps_d = out_info->fps_d; state->caps = gst_video_info_to_caps (&state->info); nvdec->mem_type = GST_NVDEC_MEM_TYPE_SYSTEM; @@ -557,7 +584,9 @@ parser_display_callback (GstNvDec * nvdec, CUVIDPARSERDISPINFO * dispinfo) GST_BUFFER_DTS (output_buffer) = GST_CLOCK_TIME_NONE; /* assume buffer duration from framerate */ GST_BUFFER_DURATION (output_buffer) = - gst_util_uint64_scale (GST_SECOND, nvdec->fps_d, nvdec->fps_n); + gst_util_uint64_scale (GST_SECOND, + GST_VIDEO_INFO_FPS_D (&nvdec->out_info), + GST_VIDEO_INFO_FPS_N (&nvdec->out_info)); } else { ret = gst_video_decoder_allocate_output_frame (GST_VIDEO_DECODER (nvdec), frame); @@ -670,6 +699,7 @@ gst_nvdec_start (GstVideoDecoder * decoder) nvdec->state = GST_NVDEC_STATE_INIT; nvdec->last_ret = GST_FLOW_OK; + gst_video_info_init (&nvdec->out_info); return TRUE; } @@ -902,7 +932,7 @@ copy_video_frame_to_gl_textures (GstGLContext * context, mcpy2d.WidthInBytes = GST_VIDEO_INFO_COMP_WIDTH (info, i) * GST_VIDEO_INFO_COMP_PSTRIDE (info, i); - mcpy2d.srcDevice = dptr + (i * pitch * nvdec->height); + mcpy2d.srcDevice = dptr + (i * pitch * GST_VIDEO_INFO_HEIGHT (info)); mcpy2d.dstDevice = cuda_ptr; mcpy2d.Height = GST_VIDEO_INFO_COMP_HEIGHT (info, i); @@ -986,7 +1016,7 @@ gst_nvdec_copy_device_to_system (GstNvDec * nvdec, * GST_VIDEO_INFO_COMP_PSTRIDE (info, 0); for (i = 0; i < GST_VIDEO_FRAME_N_PLANES (&video_frame); i++) { - copy_params.srcDevice = dptr + (i * pitch * nvdec->height); + copy_params.srcDevice = dptr + (i * pitch * GST_VIDEO_INFO_HEIGHT (info)); copy_params.dstHost = GST_VIDEO_FRAME_PLANE_DATA (&video_frame, i); copy_params.dstPitch = GST_VIDEO_FRAME_PLANE_STRIDE (&video_frame, i); copy_params.Height = GST_VIDEO_FRAME_COMP_HEIGHT (&video_frame, i); diff --git a/sys/nvcodec/gstnvdec.h b/sys/nvcodec/gstnvdec.h index 86f8872..c7da4ee 100644 --- a/sys/nvcodec/gstnvdec.h +++ b/sys/nvcodec/gstnvdec.h @@ -79,10 +79,7 @@ struct _GstNvDec GstCudaContext *cuda_ctx; CUstream cuda_stream; - guint width; - guint height; - guint fps_n; - guint fps_d; + GstVideoInfo out_info; GstClockTime min_latency; GstVideoCodecState *input_state; GstVideoCodecState *output_state; -- 2.7.4