From 39eee2a83696ec0500957bfd5e444afec6a91787 Mon Sep 17 00:00:00 2001 From: Jaeyun Date: Tue, 2 Oct 2018 21:12:28 +0900 Subject: [PATCH] [Converter/Aggregator] update timestamp set timestamp with distance (bytes from pts) if frames-in is larger than frames-out, it will set same timestamp. in this case, elements update timestamp with distance. Signed-off-by: Jaeyun Jung --- gst/tensor_aggregator/tensor_aggregator.c | 59 +++++++++++++++------ gst/tensor_converter/tensor_converter.c | 85 +++++++++++++++++++++++-------- 2 files changed, 107 insertions(+), 37 deletions(-) diff --git a/gst/tensor_aggregator/tensor_aggregator.c b/gst/tensor_aggregator/tensor_aggregator.c index d2b2e12..1978a34 100644 --- a/gst/tensor_aggregator/tensor_aggregator.c +++ b/gst/tensor_aggregator/tensor_aggregator.c @@ -46,7 +46,7 @@ #define silent_debug(...) \ debug_print (DBG, __VA_ARGS__) -#define silent_debug_caps(caps,msg) do {\ +#define silent_debug_caps(caps,msg) do { \ if (DBG) { \ if (caps) { \ GstStructure *caps_s; \ @@ -100,7 +100,7 @@ enum #define DEFAULT_FRAMES_FLUSH 0 /** - * @brief The index of frames in tensor dimension. + * @brief The dimension index of frames in configured tensor. */ #define DEFAULT_FRAMES_DIMENSION (-1) @@ -164,7 +164,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass) object_class->finalize = gst_tensor_aggregator_finalize; /** - * GstTensorAggregator:frames-in: + * GstTensorAggregator::frames-in: * * The number of frames in incoming buffer. * GstTensorAggregator itself cannot get the frames in buffer. (buffer is a sinle tensor instance) @@ -176,7 +176,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass) DEFAULT_FRAMES_IN, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /** - * GstTensorAggregator:frames-out: + * GstTensorAggregator::frames-out: * * The number of frames in outgoing buffer. (buffer is a sinle tensor instance) * GstTensorAggregator calculates the size of outgoing frames and pushes a buffer to source pad. @@ -187,7 +187,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass) DEFAULT_FRAMES_OUT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /** - * GstTensorAggregator:frames-flush: + * GstTensorAggregator::frames-flush: * * The number of frames to flush. * GstTensorAggregator flushes the bytes (N frames) in GstAdapter after pushing a buffer. @@ -199,7 +199,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass) DEFAULT_FRAMES_FLUSH, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /** - * GstTensorAggregator:frames-dim: + * GstTensorAggregator::frames-dim: * * The dimension index of frames in tensor. * If frames-in and frames-out are different, GstTensorAggregator has to change the dimension of tensor. @@ -214,7 +214,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /** - * GstTensorAggregator:silent: + * GstTensorAggregator::silent: * * The flag to enable/disable debugging messages. */ @@ -496,7 +496,7 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) GstFlowReturn ret = GST_FLOW_OK; GstAdapter *adapter; gsize avail, buf_size, frame_size, out_size; - guint frames_in, frames_out; + guint frames_in, frames_out, frames_flush; self = GST_TENSOR_AGGREGATOR (parent); g_assert (self->tensor_configured); @@ -506,6 +506,8 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) frames_in = self->frames_in; frames_out = self->frames_out; + frames_flush = self->frames_flush; + frame_size = buf_size / frames_in; if (frames_in == frames_out) { /** do nothing, push the incoming buffer */ @@ -517,7 +519,6 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) gst_adapter_push (adapter, buf); - frame_size = buf_size / frames_in; out_size = frame_size * frames_out; g_assert (out_size > 0); @@ -525,24 +526,52 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) ret == GST_FLOW_OK) { GstBuffer *outbuf; GstClockTime pts, dts; + guint64 pts_dist, dts_dist; gsize flush, offset; /** offset for last frame */ - offset = frame_size * (frames_out - 1); - - pts = gst_adapter_prev_pts_at_offset (adapter, offset, NULL); - dts = gst_adapter_prev_dts_at_offset (adapter, offset, NULL); + offset = frame_size * (frames_out - 1) + 1; /** +1 byte */ + + pts = gst_adapter_prev_pts_at_offset (adapter, offset, &pts_dist); + dts = gst_adapter_prev_dts_at_offset (adapter, offset, &dts_dist); + + /** + * Update timestamp of last frame. + * If frames-in is larger then frames-out, the same timestamp (pts and dts) would be returned. + */ + if (frames_in > 1) { + gint fn, fd; + + fn = self->in_config.rate_n; + fd = self->in_config.rate_d; + + if (fn > 0 && fd > 0) { + if (GST_CLOCK_TIME_IS_VALID (pts)) { + pts = + gst_util_uint64_scale_int (pts_dist * fd, GST_SECOND, + fn * frame_size); + } + + if (GST_CLOCK_TIME_IS_VALID (dts)) { + dts = + gst_util_uint64_scale_int (dts_dist * fd, GST_SECOND, + fn * frame_size); + } + } + } outbuf = gst_adapter_get_buffer (adapter, out_size); + outbuf = gst_buffer_make_writable (outbuf); + /** set timestamp */ GST_BUFFER_PTS (outbuf) = pts; GST_BUFFER_DTS (outbuf) = dts; ret = gst_pad_push (self->srcpad, outbuf); /** flush data */ - if (self->frames_flush > 0) { - flush = frame_size * self->frames_flush; + if (frames_flush > 0) { + flush = frame_size * frames_flush; } else { flush = out_size; } diff --git a/gst/tensor_converter/tensor_converter.c b/gst/tensor_converter/tensor_converter.c index 62eb2de..5e9d926 100644 --- a/gst/tensor_converter/tensor_converter.c +++ b/gst/tensor_converter/tensor_converter.c @@ -78,7 +78,7 @@ #define silent_debug(...) \ debug_print (DBG, __VA_ARGS__) -#define silent_debug_caps(caps,msg) do {\ +#define silent_debug_caps(caps,msg) do { \ if (DBG) { \ if (caps) { \ GstStructure *caps_s; \ @@ -178,7 +178,7 @@ gst_tensor_converter_class_init (GstTensorConverterClass * klass) object_class->finalize = gst_tensor_converter_finalize; /** - * GstTensorConverter:frames-per-tensor: + * GstTensorConverter::frames-per-tensor: * * The number of frames in outgoing buffer. (buffer is a sinle tensor instance) * GstTensorConverter can push a buffer with multiple media frames. @@ -190,7 +190,7 @@ gst_tensor_converter_class_init (GstTensorConverterClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /** - * GstTensorConverter:silent: + * GstTensorConverter::silent: * * The flag to enable/disable debugging messages. */ @@ -471,10 +471,20 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) switch (self->in_media_type) { case _NNS_VIDEO: - frame_size = - config->info.dimension[0] * config->info.dimension[1] * - config->info.dimension[2] * tensor_element_size[config->info.type]; - frames_in = 1; /** supposed 1 frame in buffer */ + { + guint color, width, height, type; + + color = config->info.dimension[0]; + width = config->info.dimension[1]; + height = config->info.dimension[2]; + type = tensor_element_size[config->info.type]; + + /** colorspace * width * height * type */ + frame_size = color * width * height * type; + + /** supposed 1 frame in buffer */ + g_assert ((buf_size / GST_VIDEO_INFO_SIZE (&self->in_info.video)) == 1); + frames_in = 1; if (self->remove_padding) { GstMapInfo src_info, dest_info; @@ -495,7 +505,7 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) /** * Refer: https://gstreamer.freedesktop.org/documentation/design/mediatype-video-raw.html */ - size = offset = config->info.dimension[0] * config->info.dimension[1]; + size = offset = color * width * type; g_assert (offset % 4); if (offset % 4) { @@ -503,9 +513,7 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) } for (d0 = 0; d0 < frames_in; d0++) { - /** Supposed to be 0 only, 1 frame in buffer. */ - g_assert (d0 == 0); - for (d1 = 0; d1 < config->info.dimension[2]; d1++) { /** Height */ + for (d1 = 0; d1 < height; d1++) { memcpy (destptr + dest_idx, srcptr + src_idx, size); dest_idx += size; src_idx += offset; @@ -519,7 +527,7 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) gst_buffer_copy_into (inbuf, buf, GST_BUFFER_COPY_TIMESTAMPS, 0, -1); } break; - + } case _NNS_AUDIO: frame_size = GST_AUDIO_INFO_BPF (&self->in_info.audio); frames_in = buf_size / frame_size; @@ -555,6 +563,8 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) return GST_FLOW_ERROR; } + silent_debug ("frames in incoming buffer = %d", frames_in); + if (frames_in == frames_out) { /** do nothing, push the incoming buffer */ return gst_pad_push (self->srcpad, inbuf); @@ -570,16 +580,44 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) ret == GST_FLOW_OK) { GstBuffer *outbuf; GstClockTime pts, dts; + guint64 pts_dist, dts_dist; gsize offset; /** offset for last frame */ - offset = frame_size * (frames_out - 1); + offset = frame_size * (frames_out - 1) + 1; /** +1 byte */ + + pts = gst_adapter_prev_pts_at_offset (adapter, offset, &pts_dist); + dts = gst_adapter_prev_dts_at_offset (adapter, offset, &dts_dist); + + /** + * Update timestamp of last frame. + * If frames-in is larger then frames-out, the same timestamp (pts and dts) would be returned. + */ + if (frames_in > 1) { + gint fn, fd; + + fn = config->rate_n; + fd = config->rate_d; + + if (fn > 0 && fd > 0) { + if (GST_CLOCK_TIME_IS_VALID (pts)) { + pts = + gst_util_uint64_scale_int (pts_dist * fd, GST_SECOND, + fn * frame_size); + } - pts = gst_adapter_prev_pts_at_offset (adapter, offset, NULL); - dts = gst_adapter_prev_dts_at_offset (adapter, offset, NULL); + if (GST_CLOCK_TIME_IS_VALID (dts)) { + dts = + gst_util_uint64_scale_int (dts_dist * fd, GST_SECOND, + fn * frame_size); + } + } + } outbuf = gst_adapter_take_buffer (adapter, out_size); + outbuf = gst_buffer_make_writable (outbuf); + /** set timestamp */ GST_BUFFER_PTS (outbuf) = pts; GST_BUFFER_DTS (outbuf) = dts; @@ -677,7 +715,7 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self, GstStructure *structure; GstTensorConfig config; media_type in_type; - guint frames_dim; /** index of frames in tensor dimension */ + gint frames_dim = -1; /** dimension index of frames in configured tensor */ g_return_val_if_fail (caps != NULL, FALSE); g_return_val_if_fail (gst_caps_is_fixed (caps), FALSE); @@ -690,11 +728,6 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self, return FALSE; } - if (!gst_tensor_config_validate (&config)) { - /** not fully configured */ - return FALSE; - } - switch (in_type) { case _NNS_VIDEO: { @@ -747,7 +780,15 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self, } /** set the number of frames in dimension */ - config.info.dimension[frames_dim] = self->frames_per_tensor; + if (frames_dim >= 0) { + config.info.dimension[frames_dim] = self->frames_per_tensor; + } + + if (!gst_tensor_config_validate (&config)) { + /** not fully configured */ + err_print ("Failed to configure tensor info.\n"); + return FALSE; + } self->in_media_type = in_type; self->tensor_configured = TRUE; -- 2.7.4