[Convert] Apply x-raw/RGB format data structure.

Data of x-raw/RGB is:

[height][width][RGB] and width has padding with stride = 4.

Thus, tensor output of convert2tensor generates width = 4 x N with zero-padding.

Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>

diff --git a/convert2tensor/README.md b/convert2tensor/README.md
index 380f156..e5ee801 100644 (file)
--- a/convert2tensor/README.md
+++ b/convert2tensor/README.md
@@ -1,19 +1,16 @@
 # NNStreamer::convert2tensor
 
 TODO Version 0.0.1
-Direct conversion of bitmap stream to [RGB][height][width] tensor.
+Direct conversion of video/x-raw 1view / non-interace(progressive) stream to [height][width-stride-4][RGB] tensor.
 The main objective of this version is to draft standard tensor format for Gstreamer.
 
 TODO Version 0.0.2
-Direct conversion of general video stream to [RGB][height][width] tensor.
-
-TODO Version 0.0.3
 Support basic dimension reform (order of dimsisions, color space changes)
 
-TODO Version 0.0.4
+TODO Version 0.0.3
 Support dimension reshape (width/height)
 
-TODO Version 0.0.5
+TODO Version 0.0.4
 Support color space conversions
 
 TODO Version 0.1.0
@@ -25,8 +22,11 @@ Direct conversion of general audio stream to FORMAT-TO-BE-DETERMINED
 - Proposed Name: other/tensor
 - Properties
   - rank: int (0: scalar, 1: vector, 2: matrix, 3: 3-tensor, ...)
-  - dimension: int[] (1 .. rank)
-  - type: int32, uint32, float32, float64, int16, uint16, int8, uint8, ... (C types only?)
-  - framerate; fraction
-
-  - data: (binary data, can be treated as an C array of [dimension[0]][dimension[1]]...)
+  - dim1: int (depth / color-RGB)
+  - dim2: int (width)
+  - dim3: int (height) / With version 0.0.1, this is with rstride-4.
+  - dim4: int (batch. 1 for image stream)
+  - type: string: int32, uint32, float32, float64, int16, uint16, int8, uint8
+  - framerate; fraction (TODO: to redefine the range)
+
+  - data: (binary data, can be treated as an C array of [dim4][dim3][dim2][dim1])

diff --git a/convert2tensor/convert2tensor.c b/convert2tensor/convert2tensor.c
index 2b14d34..73af346 100644 (file)
--- a/convert2tensor/convert2tensor.c
+++ b/convert2tensor/convert2tensor.c
@@ -108,11 +108,11 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
     GST_PAD_ALWAYS,
     GST_STATIC_CAPS ("other/tensor, "
-                       "rank = (uint) [ 1, 4 ], "
-                      "dim1 = (uint) [ 1, 65535 ], "
-                      "dim2 = (uint) [ 1, 65535 ], "
-                      "dim3 = (uint) [ 1, 65535 ], "
-                      "dim4 = (uint) [ 1, 65535 ], "
+                       "rank = (int) [ 1, 4 ], "
+                       "dim1 = (int) [ 1, 65535 ], "
+                       "dim2 = (int) [ 1, 65535 ], "
+                       "dim3 = (int) [ 1, 65535 ], "
+                       "dim4 = (int) [ 1, 65535 ], "
                       "type = (string) { float32, float64, int32, uint32, int16, uint16, int8, uint8 }, "
                       "framerate = (fraction) [ 0, 1024 ], ")
     );
@@ -269,11 +269,16 @@ gst_convert2tensor_configure_tensor(const GstCaps *caps, GstConvert2Tensor *filt
   /* This caps is coming from video/x-raw */
   structure = gst_caps_get_structure(caps, 0);
   rank = 3; /* [color-space][height][width] */
-  return_false_if_fail(gst_structure_get_int(structure, "width", &dimension[0]));
-  return_false_if_fail(gst_structure_get_int(structure, "height", &dimension[1]));
+  return_false_if_fail(gst_structure_get_int(structure, "width", &dimension[1]));
+  return_false_if_fail(gst_structure_get_int(structure, "height", &dimension[2]));
   return_false_if_fail(gst_structure_get_fraction(structure, "framerate", &framerate_numerator, &framerate_denominator));
   type = _C2T_UINT8; /* Assume color depth per component is 8 bit */
-  dimension[2] = 3; /* R G B */
+  if (dimension[1] % 4) {
+    g_print("  Width(dim2) is not divisible with 4. Width is adjusted %d -> %d\n",
+        dimension[1], (dimension[1] + 3) / 4 * 4);
+    dimension[1] = (dimension[1] + 3) / 4 * 4;
+  }
+  dimension[0] = 3; /* R G B */
   dimension[3] = 1; /* This is 3-D Tensor */
   tensorFrameSize = GstConvert2TensorDataSize[type] * dimension[0] * dimension[1] * dimension[2] * dimension[3];
   /* Refer: https://gstreamer.freedesktop.org/documentation/design/mediatype-video-raw.html */
@@ -353,9 +358,6 @@ GST_PLUGIN_DEFINE (
 static GstFlowReturn gst_c2t_transformer_videoframe(GstConvert2Tensor *filter,
                                                GstVideoFrame *inframe, GstBuffer *outbuf)
 {
-  /* @TODO RGB are interlaced! FIXME!
-     @TODO Width has stride-4. FIXME!
-   */
   return gst_buffer_copy_into(outbuf, inframe->buffer,
       GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0,
       GST_VIDEO_FRAME_SIZE(inframe));

diff --git a/convert2tensor/convert2tensor.h b/convert2tensor/convert2tensor.h
index fa43b02..042b6ef 100644 (file)
--- a/convert2tensor/convert2tensor.h
+++ b/convert2tensor/convert2tensor.h
@@ -49,7 +49,7 @@
  *                Be careful: this filter assumes that the user has attached
  *               rawvideoparser as a preprocessor for this filter so that
  *               the incoming buffer is nicely aligned in the array of
- *               uint8[RGB][height][width].
+ *               uint8[height][width w/ rstride=4][RGB].
  *
  * @see                http://github.com/TO-BE-DETERMINED-SOON
  * @author     MyungJoo Ham <myungjoo.ham@samsung.com>
@@ -119,10 +119,7 @@ struct _GstConvert2Tensor
   gboolean silent;     /**< True if logging is minimized */
   gboolean tensorConfigured;   /**< True if already successfully configured tensor metadata */
   gint rank;           /**< Tensor Rank (# dimensions) */
-  gint dimension[GST_CONVERT2TENSOR_TENSOR_RANK_LIMIT];
-      /**< Dimensions. We support up to 4th ranks.
-       *  @caution The first dimension is always 4 x N.
-       **/
+  gint dimension[GST_CONVERT2TENSOR_TENSOR_RANK_LIMIT]; /**< Dimensions. We support up to 4th ranks.  **/
   tensor_type type;            /**< Type of each element in the tensor. User must designate this. Otherwise, this is UINT8 for video/x-raw byte stream */
   gint framerate_numerator;    /**< framerate is in fraction, which is numerator/denominator */
   gint framerate_denominator;  /**< framerate is in fraction, which is numerator/denominator */