From 96f48525bd49b130af7bb51de7861b69f2fa1bf3 Mon Sep 17 00:00:00 2001 From: Jaeyun Date: Wed, 19 Sep 2018 21:21:59 +0900 Subject: [PATCH] [Filter] change filter-framework callbacks Change framework callbacks to support multi-tensors. To support multi tensors, 1. GstTensorFilter allocates the memory block for each output tensor. 2. Fill input and output tensor info (GstTensorMemory) 3. GstTensorFilter calls 'invoke' with the structure GstTensorMemory. Signed-off-by: Jaeyun Jung --- gst/tensor_filter/tensor_filter.c | 114 ++++++++++++--------- gst/tensor_filter/tensor_filter.h | 10 +- gst/tensor_filter/tensor_filter_custom.c | 32 +++--- gst/tensor_filter/tensor_filter_tensorflow.c | 15 ++- gst/tensor_filter/tensor_filter_tensorflow_core.cc | 15 +-- gst/tensor_filter/tensor_filter_tensorflow_lite.c | 16 ++- .../tensor_filter_tensorflow_lite_core.cc | 51 ++++----- include/tensor_filter_custom.h | 27 ++--- include/tensor_filter_tensorflow_core.h | 4 +- include/tensor_filter_tensorflow_lite_core.h | 21 ++-- include/tensor_typedef.h | 5 +- .../nnstreamer_customfilter_example_average.c | 14 +-- .../nnstreamer_customfilter_example_passthrough.c | 10 +- ...mer_customfilter_example_passthrough_variable.c | 10 +- .../nnstreamer_customfilter_example_scaler.c | 18 ++-- ...treamer_customfilter_example_scaler_allocator.c | 40 ++++---- 16 files changed, 202 insertions(+), 200 deletions(-) diff --git a/gst/tensor_filter/tensor_filter.c b/gst/tensor_filter/tensor_filter.c index 404ac9b..890f722 100644 --- a/gst/tensor_filter/tensor_filter.c +++ b/gst/tensor_filter/tensor_filter.c @@ -636,67 +636,85 @@ gst_tensor_filter_transform (GstBaseTransform * trans, GstBuffer * inbuf, GstBuffer * outbuf) { GstTensorFilter *self; - gsize outBufSize; - uint8_t *inptr, *outptr; - uint8_t *retoutptr; - GstMapInfo inInfo, outInfo; + GstTensorFilterProperties *prop; + GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT]; + GstMapInfo in_info[NNS_TENSOR_SIZE_LIMIT]; + GstMemory *out_mem[NNS_TENSOR_SIZE_LIMIT]; + GstMapInfo out_info[NNS_TENSOR_SIZE_LIMIT]; + GstTensorMemory in_tensors[NNS_TENSOR_SIZE_LIMIT]; + GstTensorMemory out_tensors[NNS_TENSOR_SIZE_LIMIT]; + gint i, ret; self = GST_TENSOR_FILTER_CAST (trans); + prop = &self->prop; if (G_UNLIKELY (!self->configured)) goto unknown_format; - if (G_UNLIKELY (!self->prop.fw)) + if (G_UNLIKELY (!prop->fw)) goto unknown_framework; - if (G_UNLIKELY (!self->prop.model_file)) + if (G_UNLIKELY (!prop->model_file)) goto unknown_model; - if (G_UNLIKELY (!self->prop.fw->invoke_NN)) + if (G_UNLIKELY (!prop->fw->invoke_NN)) goto unknown_invoke; - /* 0. Check all properties and inbuf size. */ - silent_debug ("Invoking %s with %s model\n", self->prop.fw->name, - self->prop.model_file); + /* 0. Check all properties. */ + silent_debug ("Invoking %s with %s model\n", prop->fw->name, + prop->model_file); + + /* 1. Set input tensors from inbuf. */ + g_assert (gst_buffer_n_memory (inbuf) == prop->input_meta.num_tensors); + + for (i = 0; i < prop->input_meta.num_tensors; i++) { + in_mem[i] = gst_buffer_peek_memory (inbuf, i); + g_assert (gst_memory_map (in_mem[i], &in_info[i], GST_MAP_READ)); - /* 1. Allocate outbuf if allocate_in_invoke is FALSE */ + in_tensors[i].data = in_info[i].data; + in_tensors[i].size = in_info[i].size; + in_tensors[i].type = prop->input_meta.info[i].type; + } + + /* 2. Prepare output tensors. */ g_assert (outbuf); + g_assert (gst_buffer_get_size (outbuf) == 0); - outBufSize = gst_tensor_filter_out_size (self, -1); + for (i = 0; i < prop->output_meta.num_tensors; i++) { + out_tensors[i].data = NULL; + out_tensors[i].size = gst_tensor_filter_out_size (self, i); + out_tensors[i].type = prop->output_meta.info[i].type; - if (self->prop.fw->allocate_in_invoke == FALSE) { - if (gst_buffer_get_size (outbuf) < outBufSize) { - /** @todo: write a routine to say aloud when this happens */ - gst_buffer_set_size (outbuf, outBufSize); + /* allocate memory if allocate_in_invoke is FALSE */ + if (prop->fw->allocate_in_invoke == FALSE) { + out_mem[i] = gst_allocator_alloc (NULL, out_tensors[i].size, NULL); + g_assert (gst_memory_map (out_mem[i], &out_info[i], GST_MAP_WRITE)); + + out_tensors[i].data = out_info[i].data; } - silent_debug ("outbuf = %lu / expected = %lu\n", - gst_buffer_get_size (outbuf), outBufSize); - g_assert (gst_buffer_get_size (outbuf) >= outBufSize); + } - /* 2. Call the filter-subplugin callback, "invoke" */ - gst_buffer_map (inbuf, &inInfo, GST_MAP_READ); - gst_buffer_map (outbuf, &outInfo, GST_MAP_WRITE); - inptr = inInfo.data; - outptr = outInfo.data; + /* 3. Call the filter-subplugin callback, "invoke" */ + gst_tensor_filter_call (self, ret, invoke_NN, in_tensors, out_tensors); + g_assert (ret == 0); + + /* 4. Update result and free map info. */ + for (i = 0; i < prop->output_meta.num_tensors; i++) { + if (prop->fw->allocate_in_invoke) { + /* filter-subplugin allocated new memory, update this */ + out_mem[i] = + gst_memory_new_wrapped (0, out_tensors[i].data, out_tensors[i].size, + 0, out_tensors[i].size, NULL, NULL); + } else { + gst_memory_unmap (out_mem[i], &out_info[i]); + } - gst_tensor_filter_call (self, retoutptr, invoke_NN, inptr, outptr); - g_assert (outptr == retoutptr); + /* append the memory block to outbuf */ + gst_buffer_append_memory (outbuf, out_mem[i]); + } - gst_buffer_unmap (inbuf, &inInfo); - gst_buffer_unmap (outbuf, &outInfo); - } else { - GstMemory *mem; - gst_buffer_map (inbuf, &inInfo, GST_MAP_READ); - g_assert (gst_buffer_get_size (outbuf) == 0); - - inptr = inInfo.data; - gst_tensor_filter_call (self, retoutptr, invoke_NN, inptr, NULL); - gst_buffer_unmap (inbuf, &inInfo); - - mem = - gst_memory_new_wrapped (0, retoutptr, outBufSize, 0, outBufSize, NULL, - NULL); - gst_buffer_insert_memory (outbuf, -1, mem); + for (i = 0; i < prop->input_meta.num_tensors; i++) { + gst_memory_unmap (in_mem[i], &in_info[i]); } - /* 3. Return result! */ + /* 5. Return result! */ return GST_FLOW_OK; unknown_format: GST_ELEMENT_ERROR (self, CORE, NOT_IMPLEMENTED, (NULL), ("unknown format")); @@ -1124,13 +1142,11 @@ gst_tensor_filter_transform_size (GstBaseTransform * trans, g_assert (self->configured); - if (self->prop.fw->allocate_in_invoke == TRUE) { - /* Do not allocate outbuf. invoke_NN will allocate! */ - *othersize = 0; - return TRUE; - } - - *othersize = gst_tensor_filter_out_size (self, -1); + /** + * Consider multi-tensors. + * Set each memory block in transform() + */ + *othersize = 0; return TRUE; } diff --git a/gst/tensor_filter/tensor_filter.h b/gst/tensor_filter/tensor_filter.h index 358b6a5..6f59c67 100644 --- a/gst/tensor_filter/tensor_filter.h +++ b/gst/tensor_filter/tensor_filter.h @@ -110,15 +110,15 @@ struct _GstTensorFilterFramework gboolean allow_in_place; /**< TRUE if InPlace transfer of input-to-output is allowed. Not supported in main, yet */ gboolean allocate_in_invoke; /**< TRUE if invoke_NN is going to allocate outputptr by itself and return the address via outputptr. Do not change this value after cap negotiation is complete (or the stream has been started). */ - uint8_t *(*invoke_NN) (const GstTensorFilter * filter, void **private_data, - const uint8_t * inputptr, uint8_t * outputptr); + int (*invoke_NN) (const GstTensorFilter * filter, void **private_data, + const GstTensorMemory * input, GstTensorMemory * output); /**< Mandatory callback. Invoke the given network model. * * @param[in] filter "this" pointer. Use this to read property values * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. - * @param[in] inputptr Input tensor. Allocated and filled by tensor_filter/main - * @param[out] outputptr Output tensor. Allocated by tensor_filter/main and to be filled by invoke_NN. N/C if allocate_in_invoke is TRUE. - * @return outputptr if allocate_in_invoke = 00 if OK. non-zero if error. + * @param[in] input The array of input tensors. Allocated and filled by tensor_filter/main + * @param[out] output The array of output tensors. Allocated by tensor_filter/main and to be filled by invoke_NN. If allocate_in_invoke is TRUE, sub-plugin should allocate the memory block for output tensor. (data in GstTensorMemory) + * @return 0 if OK. non-zero if error. */ int (*getInputDimension) (const GstTensorFilter * filter, diff --git a/gst/tensor_filter/tensor_filter_custom.c b/gst/tensor_filter/tensor_filter_custom.c index 0afff28..45f700f 100644 --- a/gst/tensor_filter/tensor_filter_custom.c +++ b/gst/tensor_filter/tensor_filter_custom.c @@ -118,12 +118,13 @@ custom_open (const GstTensorFilter * filter, void **private_data) /** * @brief The mandatory callback for GstTensorFilterFramework * @param filter The parent object - * @param[in] inptr The input tensor - * @param[out] outptr The output tensor + * @param[in] input The array of input tensors + * @param[out] output The array of output tensors + * @return 0 if OK. non-zero if error. */ -static uint8_t * +static int custom_invoke (const GstTensorFilter * filter, void **private_data, - const uint8_t * inptr, uint8_t * outptr) + const GstTensorMemory * input, GstTensorMemory * output) { int retval = custom_loadlib (filter, private_data); internal_data *ptr; @@ -134,22 +135,13 @@ custom_invoke (const GstTensorFilter * filter, void **private_data, ptr = *private_data; if (ptr->methods->invoke) { - retval = ptr->methods->invoke (ptr->customFW_private_data, &(filter->prop), - inptr, outptr); - if (retval == 0) - return outptr; - else - return NULL; + return ptr->methods->invoke (ptr->customFW_private_data, &(filter->prop), + input, output); } else if (ptr->methods->allocate_invoke) { - size_t size; - uint8_t *retptr = ptr->methods->allocate_invoke (ptr->customFW_private_data, - &(filter->prop), inptr, &size); - g_assert (size == - (get_tensor_element_count (filter->prop.output_meta.info[0].dimension) * - tensor_element_size[filter->prop.output_meta.info[0].type])); - return retptr; + return ptr->methods->allocate_invoke (ptr->customFW_private_data, + &(filter->prop), input, output); } else { - return NULL; + return -1; } } @@ -234,8 +226,8 @@ custom_close (const GstTensorFilter * filter, void **private_data) GstTensorFilterFramework NNS_support_custom = { .name = "custom", - .allow_in_place = FALSE, /* custom cannot support in-place (outptr == inptr). */ - .allocate_in_invoke = FALSE, /* Let tensor_flow allocate output buffers */ + .allow_in_place = FALSE, /* custom cannot support in-place (output == input). */ + .allocate_in_invoke = FALSE, /* GstTensorFilter allocates output buffers */ .invoke_NN = custom_invoke, /* We need to disable getI/O-dim or setI-dim with the first call */ diff --git a/gst/tensor_filter/tensor_filter_tensorflow.c b/gst/tensor_filter/tensor_filter_tensorflow.c index 77dbd08..4020916 100644 --- a/gst/tensor_filter/tensor_filter_tensorflow.c +++ b/gst/tensor_filter/tensor_filter_tensorflow.c @@ -78,23 +78,20 @@ tf_open (const GstTensorFilter * filter, void **private_data) /** * @brief The mandatory callback for GstTensorFilterFramework - * @param[in] inptr The input tensor - * @param[out] outptr The output tensor + * @param[in] input The array of input tensors + * @param[out] output The array of output tensors */ -static uint8_t * +static int tf_invoke (const GstTensorFilter * filter, void **private_data, - const uint8_t * inptr, uint8_t * outptr) + const GstTensorMemory * input, GstTensorMemory * output) { int retval; - uint8_t *allocated_outptr; tf_data *tf; tf = *private_data; g_assert (filter->privateData && *private_data == filter->privateData); - retval = - tf_core_invoke (tf->tf_private_data, (uint8_t *) inptr, - &allocated_outptr); + retval = tf_core_invoke (tf->tf_private_data, input, output); g_assert (retval == 0); - return allocated_outptr; + return retval; } /** diff --git a/gst/tensor_filter/tensor_filter_tensorflow_core.cc b/gst/tensor_filter/tensor_filter_tensorflow_core.cc index 603ca76..ac21154 100644 --- a/gst/tensor_filter/tensor_filter_tensorflow_core.cc +++ b/gst/tensor_filter/tensor_filter_tensorflow_core.cc @@ -164,12 +164,12 @@ TFCore::getOutputTensorSize () /** * @brief run the model with the input. - * @param[in] inptr : The input tensor - * @param[out] outptr : The output tensor + * @param[in] input : The array of input tensors + * @param[out] output : The array of output tensors * @return 0 if OK. non-zero if error. */ int -TFCore::invoke (uint8_t * inptr, uint8_t ** outptr) +TFCore::invoke (const GstTensorMemory * input, GstTensorMemory * output) { return 0; } @@ -257,13 +257,14 @@ tf_core_getOutputSize (void *tf) /** * @brief invoke the model * @param tf : the class object - * @param[in] inptr : The input tensor - * @param[out] outptr : The output tensor + * @param[in] input : The array of input tensors + * @param[out] output : The array of output tensors * @return 0 if OK. non-zero if error. */ int -tf_core_invoke (void *tf, uint8_t * inptr, uint8_t ** outptr) +tf_core_invoke (void *tf, const GstTensorMemory * input, + GstTensorMemory * output) { TFCore *c = (TFCore *) tf; - return c->invoke (inptr, outptr); + return c->invoke (input, output); } diff --git a/gst/tensor_filter/tensor_filter_tensorflow_lite.c b/gst/tensor_filter/tensor_filter_tensorflow_lite.c index 90adb0a..7af3132 100644 --- a/gst/tensor_filter/tensor_filter_tensorflow_lite.c +++ b/gst/tensor_filter/tensor_filter_tensorflow_lite.c @@ -78,23 +78,21 @@ tflite_open (const GstTensorFilter * filter, void **private_data) /** * @brief The mandatory callback for GstTensorFilterFramework - * @param[in] inptr The input tensor - * @param[out] outptr The output tensor + * @param[in] input The array of input tensors + * @param[out] output The array of output tensors + * @return 0 if OK. non-zero if error. */ -static uint8_t * +static int tflite_invoke (const GstTensorFilter * filter, void **private_data, - const uint8_t * inptr, uint8_t * outptr) + const GstTensorMemory * input, GstTensorMemory * output) { int retval; - uint8_t *allocated_outptr; tflite_data *tf; tf = *private_data; g_assert (filter->privateData && *private_data == filter->privateData); - retval = - tflite_core_invoke (tf->tflite_private_data, (uint8_t *) inptr, - &allocated_outptr); + retval = tflite_core_invoke (tf->tflite_private_data, input, output); g_assert (retval == 0); - return allocated_outptr; + return retval; } /** diff --git a/gst/tensor_filter/tensor_filter_tensorflow_lite_core.cc b/gst/tensor_filter/tensor_filter_tensorflow_lite_core.cc index 272e520..b9eeb2a 100644 --- a/gst/tensor_filter/tensor_filter_tensorflow_lite_core.cc +++ b/gst/tensor_filter/tensor_filter_tensorflow_lite_core.cc @@ -117,7 +117,7 @@ TFLiteCore::loadModel () * @param tfType : the defined type of Tensorflow Lite * @return the enum of defined _NNS_TYPE */ -_nns_tensor_type +tensor_type TFLiteCore::getTensorType (TfLiteType tfType) { switch (tfType) { @@ -272,41 +272,44 @@ TFLiteCore::getOutputTensorDim (GstTensorsInfo * info) /** * @brief run the model with the input. - * @param[in] inptr : The input tensor - * @param[out] outptr : The output tensor + * @param[in] input : The array of input tensors + * @param[out] output : The array of output tensors * @return 0 if OK. non-zero if error. */ int -TFLiteCore::invoke (uint8_t * inptr, uint8_t ** outptr) +TFLiteCore::invoke (const GstTensorMemory * input, GstTensorMemory * output) { #if (DBG) struct timeval start_time, stop_time; gettimeofday (&start_time, nullptr); #endif - int output_number_of_pixels = 1; + int num_of_input[NNS_TENSOR_SIZE_LIMIT]; + for (int i = 0; i < NNS_TENSOR_SIZE_LIMIT; i++) { + num_of_input[i] = 1; + } int sizeOfArray = NNS_TENSOR_RANK_LIMIT; - for (int i = 0; i < sizeOfArray; i++) { - output_number_of_pixels *= inputTensorMeta.info[0].dimension[i]; + if (interpreter->AllocateTensors () != kTfLiteOk) { + printf ("Failed to allocate tensors\n"); + return -2; } for (int i = 0; i < getInputTensorSize (); i++) { - int input = interpreter->inputs ()[i]; + int in_tensor = interpreter->inputs ()[i]; - if (interpreter->AllocateTensors () != kTfLiteOk) { - _print_log ("Failed to allocate tensors"); - return -2; + for (int j = 0; j < sizeOfArray; j++) { + num_of_input[i] *= inputTensorMeta.info[i].dimension[j]; } - inputTensors[0] = inptr; - for (int j = 0; j < output_number_of_pixels; j++) { + for (int j = 0; j < num_of_input[i]; j++) { if (inputTensorMeta.info[i].type == _NNS_FLOAT32) { - (interpreter->typed_tensor < float >(input))[j] = - ((float) inputTensors[i][j] - 127.5f) / 127.5f; + (interpreter->typed_tensor < float >(in_tensor))[j] = + (((float *) input[i].data)[j] - 127.5f) / 127.5f; } else if (inputTensorMeta.info[i].type == _NNS_UINT8) { - (interpreter->typed_tensor < uint8_t > (input))[j] = inputTensors[i][j]; + (interpreter->typed_tensor < uint8_t > (in_tensor))[j] = + ((uint8_t *) input[i].data)[j]; } } } @@ -317,15 +320,12 @@ TFLiteCore::invoke (uint8_t * inptr, uint8_t ** outptr) } for (int i = 0; i < outputTensorMeta.num_tensors; i++) { - if (outputTensorMeta.info[i].type == _NNS_FLOAT32) { - outputTensors[i] = - (uint8_t *) interpreter->typed_output_tensor < float >(i); + output[i].data = interpreter->typed_output_tensor < float >(i); } else if (outputTensorMeta.info[i].type == _NNS_UINT8) { - outputTensors[i] = interpreter->typed_output_tensor < uint8_t > (i); + output[i].data = interpreter->typed_output_tensor < uint8_t > (i); } } - *outptr = outputTensors[0]; #if (DBG) gettimeofday (&stop_time, nullptr); @@ -414,13 +414,14 @@ tflite_core_getOutputSize (void *tflite) /** * @brief invoke the model * @param tflite : the class object - * @param[in] inptr : The input tensor - * @param[out] outptr : The output tensor + * @param[in] input : The array of input tensors + * @param[out] output : The array of output tensors * @return 0 if OK. non-zero if error. */ int -tflite_core_invoke (void *tflite, uint8_t * inptr, uint8_t ** outptr) +tflite_core_invoke (void *tflite, const GstTensorMemory * input, + GstTensorMemory * output) { TFLiteCore *c = (TFLiteCore *) tflite; - return c->invoke (inptr, outptr); + return c->invoke (input, output); } diff --git a/include/tensor_filter_custom.h b/include/tensor_filter_custom.h index eadfe67..7a833f3 100644 --- a/include/tensor_filter_custom.h +++ b/include/tensor_filter_custom.h @@ -40,7 +40,7 @@ /** * @brief A function that is called before calling other functions. - * @param[in] prop Tensor_Filter's property values. Do not change its values. + * @param[in] prop GstTensorFilter's property values. Do not change its values. * @return The returned pointer will be passed to other functions as "private_data". */ typedef void *(*NNS_custom_init_func) (const GstTensorFilterProperties * prop); @@ -48,7 +48,7 @@ typedef void *(*NNS_custom_init_func) (const GstTensorFilterProperties * prop); /** * @brief A function that is called after calling other functions, when it's ready to close. * @param[in] private_data If you have allocated *private_data at init, free it here. - * @param[in] prop Tensor_Filter's property values. Do not change its values. + * @param[in] prop GstTensorFilter's property values. Do not change its values. */ typedef void (*NNS_custom_exit_func) (void *private_data, const GstTensorFilterProperties * prop); @@ -56,7 +56,7 @@ typedef void (*NNS_custom_exit_func) (void *private_data, /** * @brief Get input tensor type. * @param[in] private_data The pointer returned by NNStreamer_custom_init. - * @param[in] prop Tensor_Filter's property values. Do not change its values. + * @param[in] prop GstTensorFilter's property values. Do not change its values. * @param[out] info Structure for tensor info. */ typedef int (*NNS_custom_get_input_dimension) (void *private_data, @@ -65,7 +65,7 @@ typedef int (*NNS_custom_get_input_dimension) (void *private_data, /** * @brief Get output tensor type. * @param[in] private_data The pointer returned by NNStreamer_custom_init. - * @param[in] prop Tensor_Filter's property values. Do not change its values. + * @param[in] prop GstTensorFilter's property values. Do not change its values. * @param[out] info Structure for tensor info. */ typedef int (*NNS_custom_get_output_dimension) (void *private_data, @@ -74,7 +74,7 @@ typedef int (*NNS_custom_get_output_dimension) (void *private_data, /** * @brief Set input dim by framework. Let custom plutin set output dim accordingly. * @param[in] private_data The pointer returned by NNStreamer_custom_init - * @param[in] prop Tensor_Filter's property values. Do not change its values. + * @param[in] prop GstTensorFilter's property values. Do not change its values. * @param[in] in_info Input tensor info designated by the gstreamer framework. Note that this is not a fixed value and gstreamer may try different values during pad-cap negotiations. * @param[out] out_info Output tensor info according to the input tensor info. * @@ -88,24 +88,25 @@ typedef int (*NNS_custom_set_input_dimension) (void *private_data, /** * @brief Invoke the "main function". Without allocating output buffer. (fill in the given output buffer) * @param[in] private_data The pointer returned by NNStreamer_custom_init. - * @param[in] prop Tensor_Filter's property values. Do not change its values. - * @param[in] inptr pointer to input tensor, size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller - * @param[out] outptr pointer to output tensor, size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller + * @param[in] prop GstTensorFilter's property values. Do not change its values. + * @param[in] input The array of input tensors, each tensor size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller + * @param[out] output The array of output tensors, each tensor size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller * @return 0 if success */ typedef int (*NNS_custom_invoke) (void *private_data, - const GstTensorFilterProperties * prop, const uint8_t * inptr, uint8_t * outptr); + const GstTensorFilterProperties * prop, const GstTensorMemory * input, GstTensorMemory * output); /** * @brief Invoke the "main function". Without allocating output buffer. (fill in the given output buffer) * @param[in] private_data The pointer returned by NNStreamer_custom_init. - * @param[in] prop Tensor_Filter's property values. Do not change its values. - * @param[in] inptr pointer to input tensor, size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller + * @param[in] prop GstTensorFilter's property values. Do not change its values. + * @param[in] input pointer to input tensor, size = dim1 x dim2 x dim3 x dim4 x typesize, allocated by caller + * @param[out] output The array of output tensors, each tensor size = dim1 x dim2 x dim3 x dim4 x typesize, the memory block for output tensor should be allocated. (data in GstTensorMemory) * @param[out] size The allocated size. * @return The output buffer allocated in the invoke function */ -typedef uint8_t *(*NNS_custom_allocate_invoke) (void *private_data, - const GstTensorFilterProperties * prop, const uint8_t * inptr, size_t * size); +typedef int (*NNS_custom_allocate_invoke) (void *private_data, + const GstTensorFilterProperties * prop, const GstTensorMemory * input, GstTensorMemory * output); /** * @brief Custom Filter Class diff --git a/include/tensor_filter_tensorflow_core.h b/include/tensor_filter_tensorflow_core.h index a975dc9..4f84fc4 100644 --- a/include/tensor_filter_tensorflow_core.h +++ b/include/tensor_filter_tensorflow_core.h @@ -68,7 +68,7 @@ public: int getOutputTensorDim (GstTensorsInfo * info); int getInputTensorDimSize (); int getOutputTensorDimSize (); - int invoke (uint8_t * inptr, uint8_t ** outptr); + int invoke (const GstTensorMemory * input, GstTensorMemory * output); private: /** @@ -97,7 +97,7 @@ extern "C" extern int tf_core_getOutputDim (void *tf, GstTensorsInfo * info); extern int tf_core_getInputSize (void *tf); extern int tf_core_getOutputSize (void *tf); - extern int tf_core_invoke (void *tf, uint8_t * inptr, uint8_t ** outptr); + extern int tf_core_invoke (void *tf, const GstTensorMemory * input, GstTensorMemory * output); #ifdef __cplusplus } diff --git a/include/tensor_filter_tensorflow_lite_core.h b/include/tensor_filter_tensorflow_lite_core.h index 6415a2b..ed3fa7a 100644 --- a/include/tensor_filter_tensorflow_lite_core.h +++ b/include/tensor_filter_tensorflow_lite_core.h @@ -51,23 +51,20 @@ public: int getOutputTensorSize (); int getInputTensorDim (GstTensorsInfo * info); int getOutputTensorDim (GstTensorsInfo * info); - int invoke (uint8_t * inptr, uint8_t ** outptr); + int invoke (const GstTensorMemory * input, GstTensorMemory * output); private: const char *model_path; - tensors inputTensors; /**< The list of input tensors */ - tensors outputTensors; /**< The list of output tensors */ - - GstTensorsInfo inputTensorMeta; /**< The meta of input tensors */ - GstTensorsInfo outputTensorMeta; /**< The meta of input tensors */ + GstTensorsInfo inputTensorMeta; /**< The tensor info of input tensors */ + GstTensorsInfo outputTensorMeta; /**< The tensor info of output tensors */ std::unique_ptr < tflite::Interpreter > interpreter; std::unique_ptr < tflite::FlatBufferModel > model; double get_ms (struct timeval t); - _nns_tensor_type getTensorType (TfLiteType tfType); + tensor_type getTensorType (TfLiteType tfType); int getTensorDim (int tensor_idx, tensor_dim dim); }; @@ -81,14 +78,12 @@ extern "C" extern void *tflite_core_new (const char *_model_path); extern void tflite_core_delete (void *tflite); extern const char *tflite_core_getModelPath (void *tflite); - extern int tflite_core_getInputDim (void *tflite, - GstTensorsInfo * info); - extern int tflite_core_getOutputDim (void *tflite, - GstTensorsInfo * info); + extern int tflite_core_getInputDim (void *tflite, GstTensorsInfo * info); + extern int tflite_core_getOutputDim (void *tflite, GstTensorsInfo * info); extern int tflite_core_getOutputSize (void *tflite); extern int tflite_core_getInputSize (void *tflite); - extern int tflite_core_invoke (void *tflite, uint8_t * inptr, - uint8_t ** outptr); + extern int tflite_core_invoke (void *tflite, const GstTensorMemory * input, + GstTensorMemory * output); #ifdef __cplusplus } diff --git a/include/tensor_typedef.h b/include/tensor_typedef.h index bfca16b..18a5ba6 100644 --- a/include/tensor_typedef.h +++ b/include/tensor_typedef.h @@ -102,7 +102,6 @@ typedef enum } GstTensor_Filter_CheckStatus; typedef uint32_t tensor_dim[NNS_TENSOR_RANK_LIMIT]; -typedef uint8_t *tensors[NNS_TENSOR_SIZE_LIMIT]; /**< Array of tensors */ /** * @brief The unit of each data tensors. It will be used as an input/output tensor of other/tensors. @@ -133,10 +132,10 @@ typedef struct } GstTensorsInfo; /** - * @brief Tensor_Filter's properties for NN framework (internal data structure) + * @brief GstTensorFilter's properties for NN framework (internal data structure) * * Because custom filters of tensor_filter may need to access internal data - * of Tensor_Filter, we define this data structure here. + * of GstTensorFilter, we define this data structure here. */ typedef struct _GstTensorFilterProperties { diff --git a/nnstreamer_example/custom_example_average/nnstreamer_customfilter_example_average.c b/nnstreamer_example/custom_example_average/nnstreamer_customfilter_example_average.c index 3d085c4..9f7294c 100644 --- a/nnstreamer_example/custom_example_average/nnstreamer_customfilter_example_average.c +++ b/nnstreamer_example/custom_example_average/nnstreamer_customfilter_example_average.c @@ -83,10 +83,10 @@ set_inputDim (void *private_data, const GstTensorFilterProperties * prop, /** * @brief do_avg */ -#define do_avg(type, sumtype) do {\ +#define do_avg(type,sumtype) do {\ sumtype *avg = (sumtype *) malloc(sizeof(sumtype) * prop->input_meta.info[0].dimension[0]); \ - type *iptr = (type *) inptr; \ - type *optr = (type *) outptr; \ + type *iptr = (type *) input[0].data; \ + type *optr = (type *) output[0].data; \ for (z = 0; z < prop->input_meta.info[0].dimension[3]; z++) { \ for (y = 0; y < prop->input_meta.info[0].dimension[0]; y++) \ avg[y] = 0; \ @@ -109,7 +109,7 @@ set_inputDim (void *private_data, const GstTensorFilterProperties * prop, */ static int pt_invoke (void *private_data, const GstTensorFilterProperties * prop, - const uint8_t * inptr, uint8_t * outptr) + const GstTensorMemory * input, GstTensorMemory * output) { pt_data *data = private_data; uint32_t c, x, y, z; @@ -127,8 +127,8 @@ pt_invoke (void *private_data, const GstTensorFilterProperties * prop, prop->input_meta.info[0].dimension[2]; assert (data); - assert (inptr); - assert (outptr); + assert (input); + assert (output); /* This assumes the limit is 4 */ assert (NNS_TENSOR_RANK_LIMIT == 4); @@ -173,7 +173,7 @@ pt_invoke (void *private_data, const GstTensorFilterProperties * prop, default: assert (0); /* Type Mismatch */ } - assert (inptr != outptr); + assert (input[0].data != output[0].data); return 0; } diff --git a/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough.c b/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough.c index 46a0002..60d9091 100644 --- a/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough.c +++ b/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough.c @@ -115,19 +115,19 @@ get_outputDim (void *private_data, const GstTensorFilterProperties * prop, */ static int pt_invoke (void *private_data, const GstTensorFilterProperties * prop, - const uint8_t * inptr, uint8_t * outptr) + const GstTensorMemory * input, GstTensorMemory * output) { pt_data *data = private_data; size_t size; g_assert (data); - g_assert (inptr); - g_assert (outptr); + g_assert (input); + g_assert (output); size = get_tensor_element_count (data->dim) * tensor_element_size[data->type]; - g_assert (inptr != outptr); - memcpy (outptr, inptr, size); + g_assert (input[0].data != output[0].data); + memcpy (output[0].data, input[0].data, size); return 0; } diff --git a/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough_variable.c b/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough_variable.c index a1047db..c4afd25 100644 --- a/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough_variable.c +++ b/nnstreamer_example/custom_example_passthrough/nnstreamer_customfilter_example_passthrough_variable.c @@ -75,20 +75,20 @@ set_inputDim (void *private_data, const GstTensorFilterProperties * prop, */ static int pt_invoke (void *private_data, const GstTensorFilterProperties * prop, - const uint8_t * inptr, uint8_t * outptr) + const GstTensorMemory * input, GstTensorMemory * output) { pt_data *data = private_data; size_t size; g_assert (data); - g_assert (inptr); - g_assert (outptr); + g_assert (input); + g_assert (output); size = get_tensor_element_count (prop->output_meta.info[0].dimension) * tensor_element_size[prop->output_meta.info[0].type]; - g_assert (inptr != outptr); - memcpy (outptr, inptr, size); + g_assert (input[0].data != output[0].data); + memcpy (output[0].data, input[0].data, size); return 0; } diff --git a/nnstreamer_example/custom_example_scaler/nnstreamer_customfilter_example_scaler.c b/nnstreamer_example/custom_example_scaler/nnstreamer_customfilter_example_scaler.c index 21d6f6c..908d679 100644 --- a/nnstreamer_example/custom_example_scaler/nnstreamer_customfilter_example_scaler.c +++ b/nnstreamer_example/custom_example_scaler/nnstreamer_customfilter_example_scaler.c @@ -136,7 +136,7 @@ set_inputDim (void *private_data, const GstTensorFilterProperties * prop, */ static int pt_invoke (void *private_data, const GstTensorFilterProperties * prop, - const uint8_t * inptr, uint8_t * outptr) + const GstTensorMemory * input, GstTensorMemory * output) { pt_data *data = private_data; uint32_t ox, oy, x, y, z, elementsize; @@ -144,8 +144,8 @@ pt_invoke (void *private_data, const GstTensorFilterProperties * prop, uint32_t iidx0, iidx1, iidx2; assert (data); - assert (inptr); - assert (outptr); + assert (input); + assert (output); /* This assumes the limit is 4 */ assert (NNS_TENSOR_RANK_LIMIT == 4); @@ -185,18 +185,18 @@ pt_invoke (void *private_data, const GstTensorFilterProperties * prop, && ix < prop->input_meta.info[0].dimension[1] && iy < prop->input_meta.info[0].dimension[2]); - /* outptr[z][y][x][c] = inptr[z][iy][ix][c]; */ + /* output[z][y][x][c] = input[z][iy][ix][c]; */ for (sz = 0; sz < elementsize; sz++) - *(outptr + elementsize * (c + x * oidx0 + y * oidx1 + z * oidx2) + - sz) = - *(inptr + elementsize * (c + ix * iidx0 + iy * iidx1 + - z * iidx2) + sz); + *((uint8_t *) output[0].data + elementsize * (c + x * oidx0 + + y * oidx1 + z * oidx2) + sz) = + *((uint8_t *) input[0].data + elementsize * (c + ix * iidx0 + + iy * iidx1 + z * iidx2) + sz); } } } } - assert (inptr != outptr); + assert (input[0].data != output[0].data); return 0; } diff --git a/nnstreamer_example/custom_example_scaler/nnstreamer_customfilter_example_scaler_allocator.c b/nnstreamer_example/custom_example_scaler/nnstreamer_customfilter_example_scaler_allocator.c index c453e51..8c3d44a 100644 --- a/nnstreamer_example/custom_example_scaler/nnstreamer_customfilter_example_scaler_allocator.c +++ b/nnstreamer_example/custom_example_scaler/nnstreamer_customfilter_example_scaler_allocator.c @@ -135,24 +135,28 @@ set_inputDim (void *private_data, const GstTensorFilterProperties * prop, /** * @brief invoke-alloc callback of tensor_filter custom */ -static uint8_t * +static int pt_allocate_invoke (void *private_data, - const GstTensorFilterProperties * prop, const uint8_t * inptr, - size_t * size) + const GstTensorFilterProperties * prop, const GstTensorMemory * input, + GstTensorMemory * output) { pt_data *data = private_data; uint32_t ox, oy, x, y, z, elementsize; uint32_t oidx0, oidx1, oidx2; uint32_t iidx0, iidx1, iidx2; - - *size = - get_tensor_element_count (prop->output_meta.info[0].dimension) * - tensor_element_size[prop->output_meta.info[0].type]; - uint8_t *outptr = (uint8_t *) malloc (sizeof (uint8_t) * *size); + size_t size; assert (data); - assert (inptr); - assert (outptr); + assert (input); + assert (output); + + /* allocate output data */ + elementsize = tensor_element_size[prop->output_meta.info[0].type]; + + size = + elementsize * + get_tensor_element_count (prop->output_meta.info[0].dimension); + output[0].data = malloc (size); /* This assumes the limit is 4 */ assert (NNS_TENSOR_RANK_LIMIT == 4); @@ -163,8 +167,6 @@ pt_allocate_invoke (void *private_data, prop->output_meta.info[0].dimension[3]); assert (prop->input_meta.info[0].type == prop->output_meta.info[0].type); - elementsize = tensor_element_size[prop->input_meta.info[0].type]; - ox = (data->new_x > 0) ? data->new_x : prop->output_meta.info[0].dimension[1]; oy = (data->new_y > 0) ? data->new_y : prop->output_meta.info[0].dimension[2]; @@ -191,20 +193,20 @@ pt_allocate_invoke (void *private_data, && ix < prop->input_meta.info[0].dimension[1] && iy < prop->input_meta.info[0].dimension[2]); - /* outptr[z][y][x][c] = inptr[z][iy][ix][c]; */ + /* output[z][y][x][c] = input[z][iy][ix][c]; */ for (sz = 0; sz < elementsize; sz++) - *(outptr + elementsize * (c + x * oidx0 + y * oidx1 + z * oidx2) + - sz) = - *(inptr + elementsize * (c + ix * iidx0 + iy * iidx1 + - z * iidx2) + sz); + *((uint8_t *) output[0].data + elementsize * (c + x * oidx0 + + y * oidx1 + z * oidx2) + sz) = + *((uint8_t *) input[0].data + elementsize * (c + ix * iidx0 + + iy * iidx1 + z * iidx2) + sz); } } } } - assert (inptr != outptr); + assert (input[0].data != output[0].data); - return outptr; + return 0; } /** -- 2.7.4