[Converter/Aggregator] update timestamp

set timestamp with distance (bytes from pts)

if frames-in is larger than frames-out, it will set same timestamp.
in this case, elements update timestamp with distance.

Signed-off-by: Jaeyun Jung <jy1210.jung@samsung.com>

diff --git a/gst/tensor_aggregator/tensor_aggregator.c b/gst/tensor_aggregator/tensor_aggregator.c
index d2b2e12..1978a34 100644 (file)
--- a/gst/tensor_aggregator/tensor_aggregator.c
+++ b/gst/tensor_aggregator/tensor_aggregator.c
@@ -46,7 +46,7 @@
 #define silent_debug(...) \
     debug_print (DBG, __VA_ARGS__)
 
-#define silent_debug_caps(caps,msg) do {\
+#define silent_debug_caps(caps,msg) do { \
   if (DBG) { \
     if (caps) { \
       GstStructure *caps_s; \
@@ -100,7 +100,7 @@ enum
 #define DEFAULT_FRAMES_FLUSH 0
 
 /**
- * @brief The index of frames in tensor dimension.
+ * @brief The dimension index of frames in configured tensor.
  */
 #define DEFAULT_FRAMES_DIMENSION (-1)
 
@@ -164,7 +164,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass)
   object_class->finalize = gst_tensor_aggregator_finalize;
 
   /**
-   * GstTensorAggregator:frames-in:
+   * GstTensorAggregator::frames-in:
    *
    * The number of frames in incoming buffer.
    * GstTensorAggregator itself cannot get the frames in buffer. (buffer is a sinle tensor instance)
@@ -176,7 +176,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass)
           DEFAULT_FRAMES_IN, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
   /**
-   * GstTensorAggregator:frames-out:
+   * GstTensorAggregator::frames-out:
    *
    * The number of frames in outgoing buffer. (buffer is a sinle tensor instance)
    * GstTensorAggregator calculates the size of outgoing frames and pushes a buffer to source pad.
@@ -187,7 +187,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass)
           DEFAULT_FRAMES_OUT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
   /**
-   * GstTensorAggregator:frames-flush:
+   * GstTensorAggregator::frames-flush:
    *
    * The number of frames to flush.
    * GstTensorAggregator flushes the bytes (N frames) in GstAdapter after pushing a buffer.
@@ -199,7 +199,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass)
           DEFAULT_FRAMES_FLUSH, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
   /**
-   * GstTensorAggregator:frames-dim:
+   * GstTensorAggregator::frames-dim:
    *
    * The dimension index of frames in tensor.
    * If frames-in and frames-out are different, GstTensorAggregator has to change the dimension of tensor.
@@ -214,7 +214,7 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass)
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
   /**
-   * GstTensorAggregator:silent:
+   * GstTensorAggregator::silent:
    *
    * The flag to enable/disable debugging messages.
    */
@@ -496,7 +496,7 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
   GstFlowReturn ret = GST_FLOW_OK;
   GstAdapter *adapter;
   gsize avail, buf_size, frame_size, out_size;
-  guint frames_in, frames_out;
+  guint frames_in, frames_out, frames_flush;
 
   self = GST_TENSOR_AGGREGATOR (parent);
   g_assert (self->tensor_configured);
@@ -506,6 +506,8 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
 
   frames_in = self->frames_in;
   frames_out = self->frames_out;
+  frames_flush = self->frames_flush;
+  frame_size = buf_size / frames_in;
 
   if (frames_in == frames_out) {
     /** do nothing, push the incoming buffer  */
@@ -517,7 +519,6 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
 
   gst_adapter_push (adapter, buf);
 
-  frame_size = buf_size / frames_in;
   out_size = frame_size * frames_out;
   g_assert (out_size > 0);
 
@@ -525,24 +526,52 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
       ret == GST_FLOW_OK) {
     GstBuffer *outbuf;
     GstClockTime pts, dts;
+    guint64 pts_dist, dts_dist;
     gsize flush, offset;
 
     /** offset for last frame */
-    offset = frame_size * (frames_out - 1);
-
-    pts = gst_adapter_prev_pts_at_offset (adapter, offset, NULL);
-    dts = gst_adapter_prev_dts_at_offset (adapter, offset, NULL);
+    offset = frame_size * (frames_out - 1) + 1; /** +1 byte */
+
+    pts = gst_adapter_prev_pts_at_offset (adapter, offset, &pts_dist);
+    dts = gst_adapter_prev_dts_at_offset (adapter, offset, &dts_dist);
+
+    /**
+     * Update timestamp of last frame.
+     * If frames-in is larger then frames-out, the same timestamp (pts and dts) would be returned.
+     */
+    if (frames_in > 1) {
+      gint fn, fd;
+
+      fn = self->in_config.rate_n;
+      fd = self->in_config.rate_d;
+
+      if (fn > 0 && fd > 0) {
+        if (GST_CLOCK_TIME_IS_VALID (pts)) {
+          pts =
+              gst_util_uint64_scale_int (pts_dist * fd, GST_SECOND,
+              fn * frame_size);
+        }
+
+        if (GST_CLOCK_TIME_IS_VALID (dts)) {
+          dts =
+              gst_util_uint64_scale_int (dts_dist * fd, GST_SECOND,
+              fn * frame_size);
+        }
+      }
+    }
 
     outbuf = gst_adapter_get_buffer (adapter, out_size);
+    outbuf = gst_buffer_make_writable (outbuf);
 
+    /** set timestamp */
     GST_BUFFER_PTS (outbuf) = pts;
     GST_BUFFER_DTS (outbuf) = dts;
 
     ret = gst_pad_push (self->srcpad, outbuf);
 
     /** flush data */
-    if (self->frames_flush > 0) {
-      flush = frame_size * self->frames_flush;
+    if (frames_flush > 0) {
+      flush = frame_size * frames_flush;
     } else {
       flush = out_size;
     }

diff --git a/gst/tensor_converter/tensor_converter.c b/gst/tensor_converter/tensor_converter.c
index 62eb2de..5e9d926 100644 (file)
--- a/gst/tensor_converter/tensor_converter.c
+++ b/gst/tensor_converter/tensor_converter.c
@@ -78,7 +78,7 @@
 #define silent_debug(...) \
     debug_print (DBG, __VA_ARGS__)
 
-#define silent_debug_caps(caps,msg) do {\
+#define silent_debug_caps(caps,msg) do { \
   if (DBG) { \
     if (caps) { \
       GstStructure *caps_s; \
@@ -178,7 +178,7 @@ gst_tensor_converter_class_init (GstTensorConverterClass * klass)
   object_class->finalize = gst_tensor_converter_finalize;
 
   /**
-   * GstTensorConverter:frames-per-tensor:
+   * GstTensorConverter::frames-per-tensor:
    *
    * The number of frames in outgoing buffer. (buffer is a sinle tensor instance)
    * GstTensorConverter can push a buffer with multiple media frames.
@@ -190,7 +190,7 @@ gst_tensor_converter_class_init (GstTensorConverterClass * klass)
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
   /**
-   * GstTensorConverter:silent:
+   * GstTensorConverter::silent:
    *
    * The flag to enable/disable debugging messages.
    */
@@ -471,10 +471,20 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
 
   switch (self->in_media_type) {
     case _NNS_VIDEO:
-      frame_size =
-          config->info.dimension[0] * config->info.dimension[1] *
-          config->info.dimension[2] * tensor_element_size[config->info.type];
-      frames_in = 1; /** supposed 1 frame in buffer */
+    {
+      guint color, width, height, type;
+
+      color = config->info.dimension[0];
+      width = config->info.dimension[1];
+      height = config->info.dimension[2];
+      type = tensor_element_size[config->info.type];
+
+      /** colorspace * width * height * type */
+      frame_size = color * width * height * type;
+
+      /** supposed 1 frame in buffer */
+      g_assert ((buf_size / GST_VIDEO_INFO_SIZE (&self->in_info.video)) == 1);
+      frames_in = 1;
 
       if (self->remove_padding) {
         GstMapInfo src_info, dest_info;
@@ -495,7 +505,7 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
         /**
          * Refer: https://gstreamer.freedesktop.org/documentation/design/mediatype-video-raw.html
          */
-        size = offset = config->info.dimension[0] * config->info.dimension[1];
+        size = offset = color * width * type;
 
         g_assert (offset % 4);
         if (offset % 4) {
@@ -503,9 +513,7 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
         }
 
         for (d0 = 0; d0 < frames_in; d0++) {
-          /** Supposed to be 0 only, 1 frame in buffer. */
-          g_assert (d0 == 0);
-          for (d1 = 0; d1 < config->info.dimension[2]; d1++) { /** Height */
+          for (d1 = 0; d1 < height; d1++) {
             memcpy (destptr + dest_idx, srcptr + src_idx, size);
             dest_idx += size;
             src_idx += offset;
@@ -519,7 +527,7 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
         gst_buffer_copy_into (inbuf, buf, GST_BUFFER_COPY_TIMESTAMPS, 0, -1);
       }
       break;
-
+    }
     case _NNS_AUDIO:
       frame_size = GST_AUDIO_INFO_BPF (&self->in_info.audio);
       frames_in = buf_size / frame_size;
@@ -555,6 +563,8 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
       return GST_FLOW_ERROR;
   }
 
+  silent_debug ("frames in incoming buffer = %d", frames_in);
+
   if (frames_in == frames_out) {
     /** do nothing, push the incoming buffer  */
     return gst_pad_push (self->srcpad, inbuf);
@@ -570,16 +580,44 @@ gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
       ret == GST_FLOW_OK) {
     GstBuffer *outbuf;
     GstClockTime pts, dts;
+    guint64 pts_dist, dts_dist;
     gsize offset;
 
     /** offset for last frame */
-    offset = frame_size * (frames_out - 1);
+    offset = frame_size * (frames_out - 1) + 1; /** +1 byte */
+
+    pts = gst_adapter_prev_pts_at_offset (adapter, offset, &pts_dist);
+    dts = gst_adapter_prev_dts_at_offset (adapter, offset, &dts_dist);
+
+    /**
+     * Update timestamp of last frame.
+     * If frames-in is larger then frames-out, the same timestamp (pts and dts) would be returned.
+     */
+    if (frames_in > 1) {
+      gint fn, fd;
+
+      fn = config->rate_n;
+      fd = config->rate_d;
+
+      if (fn > 0 && fd > 0) {
+        if (GST_CLOCK_TIME_IS_VALID (pts)) {
+          pts =
+              gst_util_uint64_scale_int (pts_dist * fd, GST_SECOND,
+              fn * frame_size);
+        }
 
-    pts = gst_adapter_prev_pts_at_offset (adapter, offset, NULL);
-    dts = gst_adapter_prev_dts_at_offset (adapter, offset, NULL);
+        if (GST_CLOCK_TIME_IS_VALID (dts)) {
+          dts =
+              gst_util_uint64_scale_int (dts_dist * fd, GST_SECOND,
+              fn * frame_size);
+        }
+      }
+    }
 
     outbuf = gst_adapter_take_buffer (adapter, out_size);
+    outbuf = gst_buffer_make_writable (outbuf);
 
+    /** set timestamp */
     GST_BUFFER_PTS (outbuf) = pts;
     GST_BUFFER_DTS (outbuf) = dts;
 
@@ -677,7 +715,7 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self,
   GstStructure *structure;
   GstTensorConfig config;
   media_type in_type;
-  guint frames_dim; /** index of frames in tensor dimension */
+  gint frames_dim = -1; /** dimension index of frames in configured tensor */
 
   g_return_val_if_fail (caps != NULL, FALSE);
   g_return_val_if_fail (gst_caps_is_fixed (caps), FALSE);
@@ -690,11 +728,6 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self,
     return FALSE;
   }
 
-  if (!gst_tensor_config_validate (&config)) {
-    /** not fully configured */
-    return FALSE;
-  }
-
   switch (in_type) {
     case _NNS_VIDEO:
     {
@@ -747,7 +780,15 @@ gst_tensor_converter_parse_caps (GstTensorConverter * self,
   }
 
   /** set the number of frames in dimension */
-  config.info.dimension[frames_dim] = self->frames_per_tensor;
+  if (frames_dim >= 0) {
+    config.info.dimension[frames_dim] = self->frames_per_tensor;
+  }
+
+  if (!gst_tensor_config_validate (&config)) {
+    /** not fully configured */
+    err_print ("Failed to configure tensor info.\n");
+    return FALSE;
+  }
 
   self->in_media_type = in_type;
   self->tensor_configured = TRUE;