From 5d0f8f2bb60958e0818d007c1e7fd1ca4ed9bc7b Mon Sep 17 00:00:00 2001 From: "jijoong.moon" Date: Mon, 2 Jul 2018 10:59:56 +0900 Subject: [PATCH] [Tensors] Add tensor dimensions string for negotiation Add tensors dimensions string for negotiation. The default cap form of "other/tenosrs" is "other/tensors, " \ "rank = (int) [ 1, 4 ], " \ "num_tensors = (int) [1, 65535], "\ "type = (string) { float32, float64, int32, uint32, int16, uint16, int8, uint8 }, " \ "framerate = (fraction) [ 0/1, 2147483647/1 ]" \ /* Dimensions of Tensors for negotiation */ /* "dimensions = (string) dim1:dim2:dim3:dim4, dim1:dim2:dim3:dim4" */ Signed-off-by: jijoong.moon --- common/tensor_meta.c | 65 +++++++++++++++++++++- include/tensor_common.h | 17 ++++++ include/tensor_meta.h | 2 + tests/nnstreamer_tensors/runTest.sh | 2 +- .../tensors_test/gsttensorscheck.c | 56 +++++++++++-------- .../tensors_test/gsttensorscheck.h | 4 +- .../tensors_test/gsttesttensors.c | 59 +++++++++----------- .../tensors_test/gsttesttensors.h | 5 +- 8 files changed, 150 insertions(+), 60 deletions(-) diff --git a/common/tensor_meta.c b/common/tensor_meta.c index 84c686c..8c042a5 100644 --- a/common/tensor_meta.c +++ b/common/tensor_meta.c @@ -51,6 +51,9 @@ #include #include +#include +#include +#include GType gst_meta_tensor_api_get_type (void) @@ -176,13 +179,17 @@ gst_append_tensor (GstBuffer * buffer, GstMemory * mem, tensor_dim * dim) g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL); gst_buffer_append_memory (buffer, mem); + GstMetaTensor *meta = GST_META_TENSOR_GET (buffer); if (!meta) { meta = gst_make_tensors (buffer); } meta->num_tensors = meta->num_tensors + 1; - + if (gst_buffer_n_memory (buffer) != meta->num_tensors) + err_print + ("Number of memory block in buffer(%d) is not compatible with Meta (%d)\n", + gst_buffer_n_memory (buffer), meta->num_tensors); d = g_slice_new (tensor_dim); memcpy (d, dim, sizeof (tensor_dim)); meta->dimensions = g_list_append (meta->dimensions, d); @@ -197,9 +204,15 @@ gst_get_tensor (GstBuffer * buffer, gint nth) GstMemory *mem; meta = (GstMetaTensor *) gst_buffer_get_meta_tensor (buffer); if (!meta) { + err_print ("Cannot get meta!\n"); return (GstMemory *) buffer; } else { + if (gst_buffer_n_memory (buffer) != gst_get_num_tensors (buffer)) + err_print + ("Number of memory block in buffer(%d) is not compatible with Meta (%d)\n", + gst_buffer_n_memory (buffer), gst_get_num_tensors (buffer)); mem = gst_buffer_get_memory (buffer, nth); + gst_memory_unref (mem); return mem; } } @@ -248,3 +261,53 @@ gst_get_num_tensors (GstBuffer * buffer) GstMetaTensor *meta = (GstMetaTensor *) gst_buffer_get_meta_tensor (buffer); return meta->num_tensors; } + +GArray * +parse_dimensions (const gchar * dim_string) +{ + GArray *dimensions; + gint num_tensors; + gchar *tempbuf; + char **arr; + + char *pch; + num_tensors = 0; + tempbuf = strdup (dim_string); + pch = strtok (tempbuf, " ,"); + while (pch != NULL) { + pch = strtok (NULL, " ,"); + num_tensors++; + } + + dimensions = + g_array_sized_new (FALSE, FALSE, sizeof (tensor_dim *), num_tensors); + arr = (char **) malloc (sizeof (char *) * num_tensors); + + num_tensors = 0; + pch = NULL; + tempbuf = strdup (dim_string); + + pch = strtok (tempbuf, " ,"); + while (pch != NULL) { + arr[num_tensors] = strdup (pch); + pch = strtok (NULL, " ,"); + num_tensors++; + } + + for (int i = 0; i < num_tensors; i++) { + char *p = strtok (arr[i], " :"); + int c = 0; + tensor_dim *d = g_new0 (tensor_dim, 1); + while (p != NULL) { + (*d)[c] = atoi (p); + p = strtok (NULL, " :"); + c++; + } + g_array_append_val (dimensions, d); + } + + free (arr); + + return dimensions; + +} diff --git a/include/tensor_common.h b/include/tensor_common.h index 597a166..d1be346 100644 --- a/include/tensor_common.h +++ b/include/tensor_common.h @@ -71,6 +71,23 @@ G_BEGIN_DECLS "framerate = (fraction) [ 0/1, 2147483647/1 ]" /** + * @brief Default static capibility for other/tensors + * + * This type uses GstMetaTensor to describe tensor. So there is no need to ask information + * to identify each tensor. + * + */ +#define GST_TENSORS_CAP_DEFAULT \ + "other/tensors, " \ + "rank = (int) [ 1, 4 ], " \ + "num_tensors = (int) [1, 65535], "\ + "type = (string) { float32, float64, int32, uint32, int16, uint16, int8, uint8 }, " \ + "framerate = (fraction) [ 0/1, 2147483647/1 ]" + /* Dimensions of Tensors for negotiation. It's comment out here, + but when we call gst_structure_get_string, it actually is working well.*/ + /* "dimensions = (string) dim1:dim2:dim3:dim4, dim1:dim2:dim3:dim4" \ */ + +/** * @brief Possible input stream types for other/tensor. * * This is realted with media input stream to other/tensor. diff --git a/include/tensor_meta.h b/include/tensor_meta.h index ea80d6b..d56fc93 100644 --- a/include/tensor_meta.h +++ b/include/tensor_meta.h @@ -150,6 +150,8 @@ GstFlowReturn gst_remove_tensor (GstBuffer *buffer, gint nth); */ gint gst_get_num_tensors (GstBuffer *buffer); +GArray * parse_dimensions (const gchar* dim_string); + G_END_DECLS #endif /* __GST_TENSOR_META_H__ */ diff --git a/tests/nnstreamer_tensors/runTest.sh b/tests/nnstreamer_tensors/runTest.sh index 66e735c..371d7de 100755 --- a/tests/nnstreamer_tensors/runTest.sh +++ b/tests/nnstreamer_tensors/runTest.sh @@ -11,7 +11,7 @@ else sopath=$1 fi -gstTest "--gst-plugin-path=../../build/tests/nnstreamer_tensors/tensors_test filesrc location=testcase01_RGB_640x480.png ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,width=640,height=480,framerate=0/1 ! testtensors silent=TRUE ! tensorscheck ! filesink location=\"testcase01_RGB_640x480.nonip.log\" sync=true" 1 +gstTest "--gst-plugin-path=../../build/tests/nnstreamer_tensors/tensors_test filesrc location=testcase01_RGB_640x480.png ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,width=640,height=480,framerate=0/1 ! testtensors silent=TRUE ! tensorscheck silent=TRUE ! filesink location=\"testcase01_RGB_640x480.nonip.log\" sync=true" 1 compareAllSizeLimit testcase01_RGB_640x480.golden testcase01_RGB_640x480.nonip.log 1 diff --git a/tests/nnstreamer_tensors/tensors_test/gsttensorscheck.c b/tests/nnstreamer_tensors/tensors_test/gsttensorscheck.c index a164563..3b8ede5 100644 --- a/tests/nnstreamer_tensors/tensors_test/gsttensorscheck.c +++ b/tests/nnstreamer_tensors/tensors_test/gsttensorscheck.c @@ -96,7 +96,7 @@ enum static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT) + GST_STATIC_CAPS (GST_TENSORS_CAP_DEFAULT) ); static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", @@ -252,27 +252,36 @@ GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, gboolean ret; gint dim; GstCaps *othercaps; + const gchar *dim_string; GstStructure *s = gst_caps_get_structure (caps, 0); gst_structure_get_int (s, "rank", &filter->rank); - gst_structure_get_int (s, "dim1", &dim); - filter->dimension[0] = dim; - gst_structure_get_int (s, "dim2", &dim); - filter->dimension[1] = dim; - gst_structure_get_int (s, "dim3", &dim); - filter->dimension[2] = dim; - gst_structure_get_int (s, "dim4", &dim); - filter->dimension[3] = dim; + gst_structure_get_int (s, "num_tensors", &dim); + filter->num_tensors = dim; + + dim_string = gst_structure_get_string (s, "dimensions"); + if (dim_string) { + debug_print (!filter->silent, "dimension sting : %s\n", dim_string); + filter->dimensions = parse_dimensions (dim_string); + for (int i = 0; i < filter->num_tensors; i++) { + tensor_dim *d = g_array_index (filter->dimensions, tensor_dim *, i); + debug_print (!filter->silent, "dimensions[%d] %d %d %d %d\n", i, (*d)[0], + (*d)[1], (*d)[2], (*d)[3]); + } + } else { + err_print ("Cannot get dimensions for negotiation!\n"); + } gst_structure_get_fraction (s, "framerate", &filter->framerate_numerator, &filter->framerate_denominator); filter->type = _NNS_UINT8; + tensor_dim *d = g_array_index (filter->dimensions, tensor_dim *, 0); othercaps = gst_caps_new_simple ("other/tensor", "rank", G_TYPE_INT, filter->rank, - "dim1", G_TYPE_INT, filter->dimension[0], - "dim2", G_TYPE_INT, filter->dimension[1], - "dim3", G_TYPE_INT, filter->dimension[2], - "dim4", G_TYPE_INT, filter->dimension[3], + "dim1", G_TYPE_INT, (*d)[0], + "dim2", G_TYPE_INT, (*d)[1], + "dim3", G_TYPE_INT, (*d)[2], + "dim4", G_TYPE_INT, (*d)[3], "type", G_TYPE_STRING, tensor_element_typename[filter->type], "framerate", GST_TYPE_FRACTION, filter->framerate_numerator, filter->framerate_denominator, NULL); @@ -305,25 +314,27 @@ gst_tensors_check (Gsttensorscheck * filter, GstBuffer * inbuf) outbuf = gst_buffer_new (); /* Making output buffer (one big buffer for check tensors) */ - buffer_mem = - gst_allocator_alloc (NULL, - filter->dimension[0] * filter->dimension[1] * filter->dimension[2] * - filter->dimension[3], NULL); + buffer_mem = gst_allocator_alloc (NULL, + /* filter->dimension[0] * filter->dimension[1] * filter->dimension[2] * */ + /* filter->dimension[3], NULL); */ + 3 * 640 * 480 * 1, NULL); gst_buffer_append_memory (outbuf, buffer_mem); gst_buffer_map (outbuf, &dest_info, GST_MAP_WRITE); /* Get number of tensors */ num_tensors = gst_get_num_tensors (inbuf); - + debug_print (!filter->silent, "Number of Tensors : %d\n", num_tensors); for (int i = 0; i < num_tensors; i++) { GstMemory *mem = gst_get_tensor (inbuf, i); - gst_memory_unref (mem); + if (!mem) + debug_print (!filter->silent, "Cannot get memory\n"); dim = gst_get_tensordim (inbuf, i); ret = gst_memory_map (mem, &info, GST_MAP_WRITE); - - if (!ret) + if (!ret) { + debug_print (!filter->silent, "Cannot map memory\n"); return NULL; + } size_t span = 0; size_t span1 = 0; for (d0 = 0; d0 < (*dim)[3]; d0++) { @@ -384,9 +395,6 @@ gst_tensorscheck_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) filter = GST_TENSORSCHECK (parent); - if (filter->silent == FALSE) - g_print ("I'm plugged, therefore I'm in.\n"); - if (filter->passthrough) return gst_pad_push (filter->srcpad, buf); diff --git a/tests/nnstreamer_tensors/tensors_test/gsttensorscheck.h b/tests/nnstreamer_tensors/tensors_test/gsttensorscheck.h index 8418719..af07ed9 100644 --- a/tests/nnstreamer_tensors/tensors_test/gsttensorscheck.h +++ b/tests/nnstreamer_tensors/tensors_test/gsttensorscheck.h @@ -88,7 +88,9 @@ struct _Gsttensorscheck gboolean silent; /* For Tensor */ - tensor_dim dimension; + /* tensor_dim dimension; */ + GArray *dimensions; + gint num_tensors; gint rank; tensor_type type; gint framerate_numerator; diff --git a/tests/nnstreamer_tensors/tensors_test/gsttesttensors.c b/tests/nnstreamer_tensors/tensors_test/gsttesttensors.c index 4405f70..f13fbdf 100644 --- a/tests/nnstreamer_tensors/tensors_test/gsttesttensors.c +++ b/tests/nnstreamer_tensors/tensors_test/gsttesttensors.c @@ -102,7 +102,7 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT) + GST_STATIC_CAPS (GST_TENSORS_CAP_DEFAULT) ); #define gst_testtensors_parent_class parent_class @@ -257,28 +257,25 @@ GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, format = gst_structure_get_string (s, "format"); if (!g_strcmp0 (format, "RGB")) - filter->dimension[0] = 3; + filter->num_tensors = 3; else - filter->dimension[0] = 4; + filter->num_tensors = 4; gst_structure_get_int (s, "width", &dim); - filter->dimension[1] = dim; + filter->width = dim; gst_structure_get_int (s, "height", &dim); - filter->dimension[2] = dim; - filter->dimension[3] = 1; + filter->height = dim; gst_structure_get_fraction (s, "framerate", &filter->framerate_numerator, &filter->framerate_denominator); filter->type = _NNS_UINT8; filter->rank = 3; - othercaps = gst_caps_new_simple ("other/tensor", + othercaps = gst_caps_new_simple ("other/tensors", "rank", G_TYPE_INT, filter->rank, - "dim1", G_TYPE_INT, filter->dimension[0], - "dim2", G_TYPE_INT, filter->dimension[1], - "dim3", G_TYPE_INT, filter->dimension[2], - "dim4", G_TYPE_INT, filter->dimension[3], + "num_tensors", G_TYPE_INT, filter->num_tensors, "type", G_TYPE_STRING, tensor_element_typename[filter->type], "framerate", GST_TYPE_FRACTION, filter->framerate_numerator, - filter->framerate_denominator, NULL); + filter->framerate_denominator, "dimensions", G_TYPE_STRING, + "1:640:480:1 ,1:640:480:3 ,1:640:480:1", NULL); ret = gst_pad_set_caps (filter->srcpad, othercaps); gst_caps_unref (othercaps); return ret; @@ -295,37 +292,36 @@ gst_test_tensors (Gsttesttensors * filter, GstBuffer * inbuf) { GstBuffer *outbuf; gint num_tensor; - GstMapInfo info, src_info; + GstMapInfo src_info; - int d0, d1, d2; + int d1, d2; outbuf = gst_buffer_new (); gst_buffer_map (inbuf, &src_info, GST_MAP_READ); gst_make_tensors (outbuf); - for (num_tensor = 0; num_tensor < filter->dimension[0]; num_tensor++) { - - GstMemory *mem = - gst_allocator_alloc (NULL, filter->dimension[1] * filter->dimension[2], + for (num_tensor = 0; num_tensor < filter->num_tensors; num_tensor++) { + GstMapInfo info; + GstMemory *mem = gst_allocator_alloc (NULL, filter->width * filter->height, NULL); + gst_memory_map (mem, &info, GST_MAP_WRITE); size_t span = 0; size_t span1 = 0; - for (d0 = 0; d0 < filter->dimension[3]; d0++) { - g_assert (d0 == 0); - for (d1 = 0; d1 < filter->dimension[2]; d1++) { - span = d1 * filter->dimension[1] * filter->dimension[0]; - span1 = d1 * filter->dimension[1]; - for (d2 = 0; d2 < filter->dimension[1]; d2++) { - info.data[span1 + d2] = - src_info.data[span + (d2 * filter->dimension[0]) + num_tensor]; - } + + for (d1 = 0; d1 < filter->height; d1++) { + span = d1 * filter->width * filter->num_tensors; + span1 = d1 * filter->width; + for (d2 = 0; d2 < filter->width; d2++) { + info.data[span1 + d2] = + src_info.data[span + (d2 * filter->num_tensors) + num_tensor]; } } + tensor_dim dim; dim[0] = 1; - dim[1] = filter->dimension[1]; - dim[2] = filter->dimension[2]; + dim[1] = filter->width; + dim[2] = filter->height; dim[3] = 1; gst_memory_unmap (mem, &info); @@ -379,9 +375,6 @@ gst_testtensors_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) filter = GST_TESTTENSORS (parent); - if (filter->silent == FALSE) - g_print ("I'm plugged, therefore I'm in.\n"); - /* just push out the incoming buffer without touching it */ if (filter->passthrough) return gst_pad_push (filter->srcpad, buf); @@ -389,6 +382,8 @@ gst_testtensors_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) out = gst_test_tensors (filter, buf); gst_buffer_unref (buf); + + /* In order to keep the data in outbuffer, we have to increase buffer ref count */ gst_buffer_ref (out); return gst_pad_push (filter->srcpad, out); diff --git a/tests/nnstreamer_tensors/tensors_test/gsttesttensors.h b/tests/nnstreamer_tensors/tensors_test/gsttesttensors.h index 133a1cf..47fc79b 100644 --- a/tests/nnstreamer_tensors/tensors_test/gsttesttensors.h +++ b/tests/nnstreamer_tensors/tensors_test/gsttesttensors.h @@ -86,9 +86,12 @@ struct _Gsttesttensors GstPad *sinkpad, *srcpad; gboolean silent; + gint width; + gint height; /* For Tensor */ - tensor_dim dimension; + /* tensor_dim dimension; */ + gint num_tensors; gint rank; tensor_type type; gint framerate_numerator; -- 2.7.4