From ba3172e86d7b1b47f3b0d3e4ca4eec30ca51e4eb Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Tue, 24 Dec 2019 17:31:26 +0900 Subject: [PATCH] [subplugin/ext] Extensions not update their functions NNStreamer extensions for tensor filter update their functions based on the model file However, this behavior is not thread-safe. Updated for extensions to not update their functions for allocate_in_invoke For allocate_in_invoke, tensor_filter needs to know based on the model if the allocate_in_invoke is supported or not So added another interface in GstTensorFilterFramework - allocateInInvoke V2: Added allocate_in_invoke to be set properly in single API implementation of tensor_filter Also added some minor updates in comment Added minor bugfix in python extension Related Issue: #2034 Signed-off-by: Parichay Kapoor --- api/capi/src/tensor_filter_single.c | 6 +++-- .../tensor_filter/tensor_filter_caffe2.cc | 1 + .../tensor_filter/tensor_filter_python.cc | 12 +++++----- .../tensor_filter/tensor_filter_tensorflow.cc | 3 ++- gst/nnstreamer/nnstreamer_plugin_api_filter.h | 10 ++++----- gst/nnstreamer/tensor_filter/tensor_filter.c | 7 ++++-- .../tensor_filter/tensor_filter_common.c | 21 +++++++++++++++++ .../tensor_filter/tensor_filter_common.h | 8 +++++++ .../tensor_filter/tensor_filter_custom.c | 26 +++++++++++++++------- 9 files changed, 71 insertions(+), 23 deletions(-) diff --git a/api/capi/src/tensor_filter_single.c b/api/capi/src/tensor_filter_single.c index 23919dd..a26238e 100644 --- a/api/capi/src/tensor_filter_single.c +++ b/api/capi/src/tensor_filter_single.c @@ -346,6 +346,7 @@ g_tensor_filter_single_invoke (GTensorFilterSingle * self, { GstTensorFilterPrivate *priv; guint i; + gboolean allocate_in_invoke; priv = &self->priv; @@ -365,9 +366,10 @@ g_tensor_filter_single_invoke (GTensorFilterSingle * self, } /** Setup output buffer */ + allocate_in_invoke = gst_tensor_filter_allocate_in_invoke (priv); for (i = 0; i < priv->prop.output_meta.num_tensors; i++) { /* allocate memory if allocate_in_invoke is FALSE */ - if (priv->fw->allocate_in_invoke == FALSE) { + if (allocate_in_invoke == FALSE) { output[i].data = g_malloc (output[i].size); if (!output[i].data) { g_critical ("Failed to allocate the output tensor."); @@ -380,7 +382,7 @@ g_tensor_filter_single_invoke (GTensorFilterSingle * self, return TRUE; error: - if (priv->fw->allocate_in_invoke == FALSE) + if (allocate_in_invoke == FALSE) for (i = 0; i < priv->prop.output_meta.num_tensors; i++) g_free (output[i].data); return FALSE; diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_caffe2.cc b/ext/nnstreamer/tensor_filter/tensor_filter_caffe2.cc index 9a7c327..ba231bf 100644 --- a/ext/nnstreamer/tensor_filter/tensor_filter_caffe2.cc +++ b/ext/nnstreamer/tensor_filter/tensor_filter_caffe2.cc @@ -556,6 +556,7 @@ caffe2_getOutputDim (const GstTensorFilterProperties * prop, /** * @brief The optional callback for GstTensorFilterFramework + * @param[in] private_data caffe2 plugin's private data * @param[in] data The data element. */ static void diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_python.cc b/ext/nnstreamer/tensor_filter/tensor_filter_python.cc index 87465ad..dee706e 100644 --- a/ext/nnstreamer/tensor_filter/tensor_filter_python.cc +++ b/ext/nnstreamer/tensor_filter/tensor_filter_python.cc @@ -590,7 +590,7 @@ PYCore::run (const GstTensorMemory * input, GstTensorMemory * output) PyObject *param = PyList_New(0); for (unsigned int i = 0; i < inputTensorMeta.num_tensors; i++) { /** create a Numpy array wrapper (1-D) for NNS tensor data */ - npy_intp input_dims[] = {(npy_intp) input[i].size}; + npy_intp input_dims[] = {(npy_intp) (input[i].size / gst_tensor_get_element_size (input[i].type))}; PyObject *input_array = PyArray_SimpleNewFromData( 1, input_dims, getNumpyType(input[i].type), input[i].data); PyList_Append(param, input_array); @@ -717,8 +717,8 @@ PYCore::getNumpyType (tensor_type tType) * @brief The mandatory callback for GstTensorFilterFramework * @param prop: property of tensor_filter instance * @param private_data : python plugin's private data - * @param[in] input The array of input tensors - * @param[out] output The array of output tensors + * @param input : The array of input tensors + * @param output : The array of output tensors */ static int py_run (const GstTensorFilterProperties * prop, void **private_data, @@ -735,7 +735,8 @@ py_run (const GstTensorFilterProperties * prop, void **private_data, /** * @brief The optional callback for GstTensorFilterFramework - * @param[in] data The data element. + * @param data : The data element. + * @param private_data : python plugin's private data */ static void py_destroyNotify (void **private_data, void *data) @@ -745,8 +746,9 @@ py_destroyNotify (void **private_data, void *data) if (it != PYCore::outputArrayMap.end()){ Py_XDECREF(it->second); PYCore::outputArrayMap.erase (it); - } else + } else { g_critical("Cannot find output data: 0x%lx", (unsigned long) data); + } } /** diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc index 5332015..cd5e9cf 100644 --- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc +++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc @@ -714,7 +714,8 @@ tf_getOutputDim (const GstTensorFilterProperties * prop, void **private_data, /** * @brief The optional callback for GstTensorFilterFramework - * @param[in] data The data element. + * @param data : The data element. + * @param private_data : tensorflow plugin's private data */ static void tf_destroyNotify (void **private_data, void *data) diff --git a/gst/nnstreamer/nnstreamer_plugin_api_filter.h b/gst/nnstreamer/nnstreamer_plugin_api_filter.h index 49af6d4..e632c6d 100644 --- a/gst/nnstreamer/nnstreamer_plugin_api_filter.h +++ b/gst/nnstreamer/nnstreamer_plugin_api_filter.h @@ -360,11 +360,11 @@ typedef struct _GstTensorFilterFramework */ int (*allocateInInvoke) (void **private_data); - /**< Optional. tensor_filter.c will call it when allocate_in_invoke is set to TRUE. This check if the provided model for the framework supports allocation at invoke or not. If this is not defined, then the value of allocate_in_invoke is assumed to be final for all models. - * - * @param[in] private_data A subplugin may save its internal private data here. - * @return 0 if supported. -errno if not supported. - */ + /**< Optional. tensor_filter.c will call it when allocate_in_invoke is set to TRUE. This check if the provided model for the framework supports allocation at invoke or not. If this is not defined, then the value of allocate_in_invoke is assumed to be final for all models. + * + * @param[in] private_data A subplugin may save its internal private data here. + * @return 0 if supported. -errno if not supported. + */ }; /** diff --git a/gst/nnstreamer/tensor_filter/tensor_filter.c b/gst/nnstreamer/tensor_filter/tensor_filter.c index 73784a4..8c56b5c 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter.c @@ -357,6 +357,7 @@ gst_tensor_filter_transform (GstBaseTransform * trans, GstTensorMemory out_tensors[NNS_TENSOR_SIZE_LIMIT]; guint i; gint ret; + gboolean allocate_in_invoke; self = GST_TENSOR_FILTER_CAST (trans); priv = &self->priv; @@ -376,6 +377,7 @@ gst_tensor_filter_transform (GstBaseTransform * trans, /* 0. Check all properties. */ silent_debug ("Invoking %s with %s model\n", priv->fw->name, GST_STR_NULL (prop->model_files[0])); + allocate_in_invoke = gst_tensor_filter_allocate_in_invoke (priv); /* 1. Set input tensors from inbuf. */ g_assert (gst_buffer_n_memory (inbuf) == prop->input_meta.num_tensors); @@ -399,7 +401,7 @@ gst_tensor_filter_transform (GstBaseTransform * trans, out_tensors[i].type = prop->output_meta.info[i].type; /* allocate memory if allocate_in_invoke is FALSE */ - if (priv->fw->allocate_in_invoke == FALSE) { + if (allocate_in_invoke == FALSE) { out_mem[i] = gst_allocator_alloc (NULL, out_tensors[i].size, NULL); g_assert (gst_memory_map (out_mem[i], &out_info[i], GST_MAP_WRITE)); @@ -414,10 +416,11 @@ gst_tensor_filter_transform (GstBaseTransform * trans, /* 4. Update result and free map info. */ for (i = 0; i < prop->output_meta.num_tensors; i++) { - if (priv->fw->allocate_in_invoke) { + if (allocate_in_invoke) { GPtrArray *data_array = g_ptr_array_new (); g_ptr_array_add (data_array, (gpointer) self); g_ptr_array_add (data_array, (gpointer) out_tensors[i].data); + /* filter-subplugin allocated new memory, update this */ out_mem[i] = gst_memory_new_wrapped (0, out_tensors[i].data, out_tensors[i].size, diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_common.c b/gst/nnstreamer/tensor_filter/tensor_filter_common.c index f521215..305a75d 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter_common.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter_common.c @@ -416,6 +416,27 @@ gst_tensor_filter_parse_modelpaths_string (GstTensorFilterProperties * prop, } } +/** + * @brief check if the allocate_in_invoke is valid for the framework + * @param[in] priv Struct containing the properties of the object + * @return TRUE if valid, FALSE on error + */ +gboolean +gst_tensor_filter_allocate_in_invoke (GstTensorFilterPrivate * priv) +{ + int allocate_in_invoke; + + allocate_in_invoke = priv->fw->allocate_in_invoke; + if (allocate_in_invoke == TRUE && priv->fw->allocateInInvoke) { + if (priv->fw->allocateInInvoke (&priv->privateData) == 0) { + allocate_in_invoke = TRUE; + } else { + allocate_in_invoke = FALSE; + } + } + + return allocate_in_invoke; +} /** * @brief Printout the comparison results of two tensors. diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_common.h b/gst/nnstreamer/tensor_filter/tensor_filter_common.h index 66560d8..d22a83f 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter_common.h +++ b/gst/nnstreamer/tensor_filter/tensor_filter_common.h @@ -57,6 +57,14 @@ gst_tensor_filter_compare_tensors (GstTensorsInfo * info1, GstTensorsInfo * info2); /** + * @brief check if the allocate_in_invoke is valid for the framework + * @param[in] priv Struct containing the properties of the object + * @return TRUE if valid, FALSE on error + */ +extern gboolean +gst_tensor_filter_allocate_in_invoke (GstTensorFilterPrivate * priv); + +/** * @brief Installs all the properties for tensor_filter * @param[in] gobject_class Glib object class whose properties will be set */ diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_custom.c b/gst/nnstreamer/tensor_filter/tensor_filter_custom.c index 2874264..b48279f 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter_custom.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter_custom.c @@ -37,8 +37,6 @@ void init_filter_custom (void) __attribute__ ((constructor)); void fini_filter_custom (void) __attribute__ ((destructor)); -static GstTensorFilterFramework NNS_support_custom; - /** * @brief internal_data */ @@ -131,9 +129,6 @@ custom_open (const GstTensorFilterProperties * prop, void **private_data) ptr = *private_data; g_assert (!ptr->methods->invoke != !ptr->methods->allocate_invoke); /* XOR! */ - if (ptr->methods->allocate_invoke) { - NNS_support_custom.allocate_in_invoke = TRUE; - } return 0; } @@ -256,23 +251,38 @@ custom_destroyNotify (void **private_data, void *data) } } +/** + * @brief The optional callback for GstTensorFilterFramework + */ +static int +custom_allocateInInvoke (void **private_data) +{ + internal_data *ptr = *private_data; + + if (ptr && ptr->methods->allocate_invoke) { + return 0; + } + + return -EINVAL; +} + static gchar filter_subplugin_custom[] = "custom"; static GstTensorFilterFramework NNS_support_custom = { .version = GST_TENSOR_FILTER_FRAMEWORK_V0, .name = filter_subplugin_custom, .allow_in_place = FALSE, /* custom cannot support in-place (output == input). */ - .allocate_in_invoke = FALSE, /* GstTensorFilter allocates output buffers */ + .allocate_in_invoke = TRUE, /* GstTensorFilter allocates output buffers */ .run_without_model = FALSE, /* custom needs a so file */ .invoke_NN = custom_invoke, - /* We need to disable getI/O-dim or setI-dim with the first call */ .getInputDimension = custom_getInputDim, .getOutputDimension = custom_getOutputDim, .setInputDimension = custom_setInputDim, .open = custom_open, .close = custom_close, - .destroyNotify = custom_destroyNotify, /* default null. if allocate_in_invoke is true, this will be set from custom filter. */ + .destroyNotify = custom_destroyNotify, /* if custom filter model supports allocate_in_invoke, this will be set from custom filter. */ + .allocateInInvoke = custom_allocateInInvoke, }; /** @brief Initialize this object for tensor_filter subplugin runtime register */ -- 2.7.4