From 3d7df5fb8cb97ebd2cb42d3c1867c96cd22f1215 Mon Sep 17 00:00:00 2001 From: Jaeyun Date: Wed, 10 Oct 2018 13:23:10 +0900 Subject: [PATCH] [Aggregator] update code to concatenate output 1. add property concat (default true, to do the concatenation) 2. add functions to check dimension before pushing the buffer into source pad TODO : update concat code with the axis Signed-off-by: Jaeyun Jung --- gst/tensor_aggregator/README.md | 10 +- gst/tensor_aggregator/tensor_aggregator.c | 313 +++++++++++++++++++++++++++++- gst/tensor_aggregator/tensor_aggregator.h | 1 + 3 files changed, 318 insertions(+), 6 deletions(-) diff --git a/gst/tensor_aggregator/README.md b/gst/tensor_aggregator/README.md index 61ceef9..dac569d 100644 --- a/gst/tensor_aggregator/README.md +++ b/gst/tensor_aggregator/README.md @@ -62,10 +62,14 @@ It does not support ```other/tensors``` because each frame (or a set of frames c - frames-dim: The dimension index of frames in tensor. (Default value is (NNS_TENSOR_RANK_LIMIT - 1)) - If frames-in and frames-out are different, GstTensorAggregator has to change the dimension of tensor. + If ```frames-in``` and ```frames-out``` are different, GstTensorAggregator has to change the dimension of tensor. With this property, GstTensorAggregator changes the out-caps. - - If set this value in 0 ~ (NNS_TENSOR_RANK_LIMIT - 2), GstTensorAggregator will concatenate the output buffer. + + If set this value in 0 ~ (NNS_TENSOR_RANK_LIMIT - 2) and ```concat``` is true, GstTensorAggregator will concatenate the output buffer. + +- concat: The flag to concatenate output buffer. (Default true) + + If ```concat``` is true and ```frames-out``` is larger than 1, GstTensorAggregator will concatenate the output buffer with the axis ```frames-dim```. ### Properties for debugging diff --git a/gst/tensor_aggregator/tensor_aggregator.c b/gst/tensor_aggregator/tensor_aggregator.c index 6f32ddf..ca93016 100644 --- a/gst/tensor_aggregator/tensor_aggregator.c +++ b/gst/tensor_aggregator/tensor_aggregator.c @@ -31,6 +31,7 @@ #include #endif +#include #include "tensor_aggregator.h" /** @@ -76,6 +77,7 @@ enum PROP_FRAMES_OUT, PROP_FRAMES_FLUSH, PROP_FRAMES_DIMENSION, + PROP_CONCAT, PROP_SILENT }; @@ -105,6 +107,11 @@ enum #define DEFAULT_FRAMES_DIMENSION (NNS_TENSOR_RANK_LIMIT - 1) /** + * @brief Flag to concatenate output buffer. + */ +#define DEFAULT_CONCAT TRUE + +/** * @brief Template for sink pad. */ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", @@ -212,6 +219,16 @@ gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /** + * GstTensorAggregator::concat: + * + * The flag to concatenate output buffer. + * If concat is true and frames-out is larger than 1, GstTensorAggregator will concatenate the output buffer with the axis frames-dim. + */ + g_object_class_install_property (object_class, PROP_CONCAT, + g_param_spec_boolean ("concat", "Concat", "Concatenate output buffer", + DEFAULT_CONCAT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + /** * GstTensorAggregator::silent: * * The flag to enable/disable debugging messages. @@ -263,6 +280,7 @@ gst_tensor_aggregator_init (GstTensorAggregator * self) self->frames_out = DEFAULT_FRAMES_OUT; self->frames_flush = DEFAULT_FRAMES_FLUSH; self->frames_dim = DEFAULT_FRAMES_DIMENSION; + self->concat = DEFAULT_CONCAT; self->adapter = gst_adapter_new (); gst_tensor_aggregator_reset (self); @@ -312,6 +330,9 @@ gst_tensor_aggregator_set_property (GObject * object, guint prop_id, case PROP_FRAMES_DIMENSION: self->frames_dim = g_value_get_uint (value); break; + case PROP_CONCAT: + self->concat = g_value_get_boolean (value); + break; case PROP_SILENT: self->silent = g_value_get_boolean (value); break; @@ -345,6 +366,9 @@ gst_tensor_aggregator_get_property (GObject * object, guint prop_id, case PROP_FRAMES_DIMENSION: g_value_set_uint (value, self->frames_dim); break; + case PROP_CONCAT: + g_value_set_boolean (value, self->concat); + break; case PROP_SILENT: g_value_set_boolean (value, self->silent); break; @@ -485,6 +509,288 @@ gst_tensor_aggregator_src_query (GstPad * pad, GstObject * parent, } /** + * @brief Check tensor dimension and axis to concatenate data. + * @param self this pointer to GstTensorAggregator + * @param info tensor info for one frame + * @return True if needed to concatenate + */ +static gboolean +gst_tensor_aggregator_check_concat_axis (GstTensorAggregator * self, + const GstTensorInfo * info) +{ + guint i; + + g_assert (info != NULL); + g_assert (self->frames_dim < NNS_TENSOR_RANK_LIMIT); + + /** + * Check condition to concatenate data. + */ + if (self->concat && self->frames_out > 1) { + for (i = self->frames_dim + 1; i < NNS_TENSOR_RANK_LIMIT; i++) { + if (info->dimension[i] > 1) { + /** concatenate data */ + return TRUE; + } + } + } + + return FALSE; +} + +/** + * @brief Change the data in buffer with given axis. + * @param self this pointer to GstTensorAggregator + * @param outbuf buffer to be concatenated + * @param info tensor info for one frame + */ +static void +gst_tensor_aggregator_concat (GstTensorAggregator * self, GstBuffer * outbuf, + const GstTensorInfo * info) +{ + GstBuffer *srcbuf; + GstMapInfo src_info, dest_info; + guint f, d0, d1; + gsize block_size; + gsize src_idx, dest_idx; + gsize frame_size; + + frame_size = gst_tensor_info_get_size (info); + g_assert (frame_size > 0); + + srcbuf = gst_buffer_copy (outbuf); + g_assert (gst_buffer_map (srcbuf, &src_info, GST_MAP_READ)); + g_assert (gst_buffer_map (outbuf, &dest_info, GST_MAP_WRITE)); + + /** + * Concatenate output buffer with given axis (frames-dim) + * If frames-dim is equal to (NNS_TENSOR_RANK_LIMIT - 1), nothing to do. + * (In this case, this function will not be called. See gst_tensor_aggregator_check_concat_axis ()) + ******************************************************************** + * Ex1) concatenate 2 frames with dimension 3:4:2:1 + ******************************************************************** + * frame 1 + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ] + * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ] + * ] + * + * frame 2 + * [ + * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ] + * ] + ******************************************************************** + * 1-1. result with frames-dim 3 (3:4:2:2) + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ] + * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ] + * ] + * [ + * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ] + * ] + ******************************************************************** + * 1-2. result with frames-dim 2 (3:4:4:1) + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ] + * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ] + * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ] + * ] + ******************************************************************** + * 1-3. result with frames-dim 1 (3:8:2:1) + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] + * [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] + * [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ] + * ] + ******************************************************************** + * 1-4. result with frames-dim 0 (6:4:2:1) + * [ + * [ [1101 1102 1103 2101 2102 2103] [1104 1105 1106 2104 2105 2106] + * [1107 1108 1109 2107 2108 2109] [1110 1111 1112 2110 2111 2112] ] + * [ [1113 1114 1115 2113 2114 2115] [1116 1117 1118 2116 2117 2118] + * [1119 1120 1121 2119 2120 2121] [1122 1123 1124 2122 2123 2124] ] + * ] + ******************************************************************** + * Ex2) concatenate 2 frames with dimension 3:4:2:2 + ******************************************************************** + * frame 1 + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ] + * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ] + * ] + * [ + * [ [1201 1202 1203] [1204 1205 1206] [1207 1208 1209] [1210 1211 1212] ] + * [ [1213 1214 1215] [1216 1217 1218] [1219 1220 1221] [1222 1223 1224] ] + * ] + * + * frame 2 + * [ + * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ] + * ] + * [ + * [ [2201 2202 2203] [2204 2205 2206] [2207 2208 2209] [2210 2211 2212] ] + * [ [2213 2214 2215] [2216 2217 2218] [2219 2220 2221] [2222 2223 2224] ] + * ] + ******************************************************************** + * 2-1. result with frames-dim 3 (3:4:2:4) + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ] + * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ] + * ] + * [ + * [ [1201 1202 1203] [1204 1205 1206] [1207 1208 1209] [1210 1211 1212] ] + * [ [1213 1214 1215] [1216 1217 1218] [1219 1220 1221] [1222 1223 1224] ] + * ] + * [ + * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ] + * ] + * [ + * [ [2201 2202 2203] [2204 2205 2206] [2207 2208 2209] [2210 2211 2212] ] + * [ [2213 2214 2215] [2216 2217 2218] [2219 2220 2221] [2222 2223 2224] ] + * ] + ******************************************************************** + * 2-2. result with frames-dim 2 (3:4:4:2) + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ] + * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ] + * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ] + * ] + * [ + * [ [1201 1202 1203] [1204 1205 1206] [1207 1208 1209] [1210 1211 1212] ] + * [ [1213 1214 1215] [1216 1217 1218] [1219 1220 1221] [1222 1223 1224] ] + * [ [2201 2202 2203] [2204 2205 2206] [2207 2208 2209] [2210 2211 2212] ] + * [ [2213 2214 2215] [2216 2217 2218] [2219 2220 2221] [2222 2223 2224] ] + * ] + ******************************************************************** + * 2-3. result with frames-dim 1 (3:8:2:2) + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] + * [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] + * [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ] + * ] + * [ + * [ [1201 1202 1203] [1204 1205 1206] [1207 1208 1209] [1210 1211 1212] + * [2201 2202 2203] [2204 2205 2206] [2207 2208 2209] [2210 2211 2212] ] + * [ [1213 1214 1215] [1216 1217 1218] [1219 1220 1221] [1222 1223 1224] + * [2213 2214 2215] [2216 2217 2218] [2219 2220 2221] [2222 2223 2224] ] + * ] + ******************************************************************** + * 2-4. result with frames-dim 0 (6:4:2:2) + * [ + * [ [1101 1102 1103 2101 2102 2103] [1104 1105 1106 2104 2105 2106] + * [1107 1108 1109 2107 2108 2109] [1110 1111 1112 2110 2111 2112] ] + * [ [1113 1114 1115 2113 2114 2115] [1116 1117 1118 2116 2117 2118] + * [1119 1120 1121 2119 2120 2121] [1122 1123 1124 2122 2123 2124] ] + * ] + * [ + * [ [1201 1202 1203 2201 2202 2203] [1204 1205 1206 2204 2205 2206] + * [1207 1208 1209 2207 2208 2209] [1210 1211 1212 2210 2211 2212] ] + * [ [1213 1214 1215 2213 2214 2215] [1216 1217 1218 2216 2217 2218] + * [1219 1220 1221 2219 2220 2221] [1222 1223 1224 2222 2223 2224] ] + * ] + ******************************************************************** + * Ex3) concatenate 2 frames with dimension 3:4:1:1 + ******************************************************************** + * frame 1 + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ] + * ] + * + * frame 2 + * [ + * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * ] + ******************************************************************** + * 3-1. result with frames-dim 3 (3:4:1:2) + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ] + * ] + * [ + * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * ] + ******************************************************************** + * 3-2. result with frames-dim 2 (3:4:2:1) + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ] + * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * ] + ******************************************************************** + * 3-3. result with frames-dim 1 (3:8:1:1) + * [ + * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] + * [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ] + * ] + ******************************************************************** + * 3-4. result with frames-dim 0 (6:4:1:1) + * [ + * [ [1101 1102 1103 2101 2102 2103] [1104 1105 1106 2104 2105 2106] + * [1107 1108 1109 2107 2108 2109] [1110 1111 1112 2110 2111 2112] ] + * ] + ******************************************************************** + */ + + /* get block size */ + block_size = tensor_element_size[info->type]; + for (d0 = 0; d0 <= self->frames_dim; d0++) { + block_size *= info->dimension[d0]; + } + + /** + * @todo add code to concatenate data with given axis (self->frames_dim) + * now added one case (frames_dim 1) + */ + dest_idx = 0; + for (d0 = 0; d0 < info->dimension[3]; d0++) { + for (d1 = 0; d1 < info->dimension[2]; d1++) { + for (f = 0; f < self->frames_out; f++) { + src_idx = frame_size * f; + src_idx += ((d0 * info->dimension[2]) + d1) * block_size; + + memcpy (dest_info.data + dest_idx, src_info.data + src_idx, block_size); + dest_idx += block_size; + } + } + } + + gst_buffer_unmap (srcbuf, &src_info); + gst_buffer_unmap (outbuf, &dest_info); + + gst_buffer_unref (srcbuf); +} + +/** + * @brief Push the buffer to source pad. (Concatenate the buffer if needed) + */ +static GstFlowReturn +gst_tensor_aggregator_push (GstTensorAggregator * self, GstBuffer * outbuf, + gsize frame_size) +{ + GstTensorInfo info; + + /** tensor info for one frame */ + info = self->out_config.info; + g_assert (self->frames_dim < NNS_TENSOR_RANK_LIMIT); + info.dimension[self->frames_dim] /= self->frames_out; + + g_assert (frame_size == gst_tensor_info_get_size (&info)); + + if (gst_tensor_aggregator_check_concat_axis (self, &info)) { + /** change data in buffer with given axis */ + gst_tensor_aggregator_concat (self, outbuf, &info); + } + + return gst_pad_push (self->srcpad, outbuf); +} + +/** * @brief Chain function, this function does the actual processing. */ static GstFlowReturn @@ -508,8 +814,9 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) frame_size = buf_size / frames_in; if (frames_in == frames_out) { - /** do nothing, push the incoming buffer */ - return gst_pad_push (self->srcpad, buf); + /** push the incoming buffer (do concat if needed) */ + buf = gst_buffer_make_writable (buf); + return gst_tensor_aggregator_push (self, buf, frame_size); } adapter = self->adapter; @@ -565,7 +872,7 @@ gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) GST_BUFFER_PTS (outbuf) = pts; GST_BUFFER_DTS (outbuf) = dts; - ret = gst_pad_push (self->srcpad, outbuf); + ret = gst_tensor_aggregator_push (self, outbuf, frame_size); /** flush data */ if (frames_flush > 0) { diff --git a/gst/tensor_aggregator/tensor_aggregator.h b/gst/tensor_aggregator/tensor_aggregator.h index 7affbdc..d6c1866 100644 --- a/gst/tensor_aggregator/tensor_aggregator.h +++ b/gst/tensor_aggregator/tensor_aggregator.h @@ -58,6 +58,7 @@ struct _GstTensorAggregator GstPad *srcpad; /**< src pad */ gboolean silent; /**< true to print minimized log */ + gboolean concat; /**< true to concatenate output buffer */ guint frames_in; /**< number of frames in input buffer */ guint frames_out; /**< number of frames in output buffer */ guint frames_flush; /**< number of frames to flush */ -- 2.7.4