From 00c27fcef20584c93878d4df913dfb8da1e19769 Mon Sep 17 00:00:00 2001 From: MyungJoo Ham Date: Tue, 29 Jan 2019 18:15:05 +0900 Subject: [PATCH] [Filter:*] Cut build dependency of filter-subplugin to filter Enable tensor-filter subplugins to be built without dependencies on tensor_filter. After this commit, we can build filter subplugins with the exported "tensor_typedef.h". This enables #786 and #761 Signed-off-by: MyungJoo Ham --- gst/nnstreamer/tensor_filter/tensor_filter.c | 6 +- gst/nnstreamer/tensor_filter/tensor_filter.h | 94 --------------------- .../tensor_filter/tensor_filter_custom.c | 60 +++++++------- .../tensor_filter/tensor_filter_tensorflow.c | 51 ++++++------ .../tensor_filter/tensor_filter_tensorflow_core.h | 3 +- .../tensor_filter/tensor_filter_tensorflow_lite.c | 50 +++++++----- gst/nnstreamer/tensor_typedef.h | 95 ++++++++++++++++++++++ 7 files changed, 185 insertions(+), 174 deletions(-) diff --git a/gst/nnstreamer/tensor_filter/tensor_filter.c b/gst/nnstreamer/tensor_filter/tensor_filter.c index 2051eda..4bfd717 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter.c @@ -298,7 +298,7 @@ static gboolean gst_tensor_filter_stop (GstBaseTransform * trans); #define gst_tensor_filter_open_fw(filter) do { \ if (filter->prop.fw_opened == FALSE && filter->fw) { \ if (filter->fw->open != NULL) {\ - if (filter->fw->open (filter, &filter->privateData) == 0) \ + if (filter->fw->open (&filter->prop, &filter->privateData) == 0) \ filter->prop.fw_opened = TRUE; \ } else {\ filter->prop.fw_opened = TRUE; \ @@ -312,7 +312,7 @@ static gboolean gst_tensor_filter_stop (GstBaseTransform * trans); #define gst_tensor_filter_close_fw(filter) do { \ if (filter->prop.fw_opened) { \ if (filter->fw && filter->fw->close) \ - filter->fw->close (filter, &filter->privateData); \ + filter->fw->close (&filter->prop, &filter->privateData); \ filter->prop.fw_opened = FALSE; \ g_free_const (filter->prop.fwname); \ filter->prop.fwname = NULL; \ @@ -327,7 +327,7 @@ static gboolean gst_tensor_filter_stop (GstBaseTransform * trans); gst_tensor_filter_open_fw (filter); \ ret = -1; \ if (filter->prop.fw_opened && filter->fw && filter->fw->funcname) { \ - ret = filter->fw->funcname (filter, &filter->privateData, __VA_ARGS__); \ + ret = filter->fw->funcname (&filter->prop, &filter->privateData, __VA_ARGS__); \ } \ } while (0) diff --git a/gst/nnstreamer/tensor_filter/tensor_filter.h b/gst/nnstreamer/tensor_filter/tensor_filter.h index acb06df..7519a42 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter.h +++ b/gst/nnstreamer/tensor_filter/tensor_filter.h @@ -51,7 +51,6 @@ G_BEGIN_DECLS typedef struct _GstTensorFilter GstTensorFilter; typedef struct _GstTensorFilterClass GstTensorFilterClass; -typedef struct _GstTensorFilterFramework GstTensorFilterFramework; /** * @brief Internal data structure for tensor_filter instances. @@ -97,99 +96,6 @@ struct _GstTensorFilterClass */ GType gst_tensor_filter_get_type (void); -/** - * @brief Subplugin definition - * - * Common callback parameters: - * filter Filter properties. Read Only - * private_data Subplugin's private data. Set this (*private_data = XXX) if you want to change filter->private_data - */ -struct _GstTensorFilterFramework -{ - gchar *name; /**< Name of the neural network framework, searchable by FRAMEWORK property */ - gboolean allow_in_place; /**< TRUE if InPlace transfer of input-to-output is allowed. Not supported in main, yet */ - gboolean allocate_in_invoke; /**< TRUE if invoke_NN is going to allocate outputptr by itself and return the address via outputptr. Do not change this value after cap negotiation is complete (or the stream has been started). */ - - int (*invoke_NN) (const GstTensorFilter * filter, void **private_data, - const GstTensorMemory * input, GstTensorMemory * output); - /**< Mandatory callback. Invoke the given network model. - * - * @param[in] filter "this" pointer. Use this to read property values - * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. - * @param[in] input The array of input tensors. Allocated and filled by tensor_filter/main - * @param[out] output The array of output tensors. Allocated by tensor_filter/main and to be filled by invoke_NN. If allocate_in_invoke is TRUE, sub-plugin should allocate the memory block for output tensor. (data in GstTensorMemory) - * @return 0 if OK. non-zero if error. - */ - - int (*getInputDimension) (const GstTensorFilter * filter, - void **private_data, GstTensorsInfo * info); - /**< Optional. Set NULL if not supported. Get dimension of input tensor - * If getInputDimension is NULL, setInputDimension must be defined. - * If getInputDimension is defined, it is recommended to define getOutputDimension - * - * @param[in] filter "this" pointer. Use this to read property values - * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. - * @param[out] info structure of tensor info (return value) - * @return the size of input tensors - */ - - int (*getOutputDimension) (const GstTensorFilter * filter, - void **private_data, GstTensorsInfo * info); - /**< Optional. Set NULL if not supported. Get dimension of output tensor - * If getInputDimension is NULL, setInputDimension must be defined. - * If getInputDimension is defined, it is recommended to define getOutputDimension - * - * @param[in] filter "this" pointer. Use this to read property values - * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. - * @param[out] info structure of tensor info (return value) - * @return the size of output tensors - */ - - int (*setInputDimension) (const GstTensorFilter * filter, - void **private_data, const GstTensorsInfo * in_info, - GstTensorsInfo * out_info); - /**< Optional. Set Null if not supported. Tensor_filter::main will - * configure input dimension from pad-cap in run-time for the sub-plugin. - * Then, the sub-plugin is required to return corresponding output dimension - * If this is NULL, both getInput/OutputDimension must be non-NULL. - * - * When you use this, do NOT allocate or fix internal data structure based on it - * until invoke is called. Gstreamer may try different dimensions before - * settling down. - * - * @param[in] filter "this" pointer. Use this to read property values - * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. - * @param[in] in_info structure of input tensor info - * @param[out] out_info structure of output tensor info (return value) - * @return 0 if OK. non-zero if error. - */ - - int (*open) (const GstTensorFilter * filter, void **private_data); - /**< Optional. tensor_filter.c will call this before any of other callbacks and will call once before calling close - * - * @param[in] filter "this" pointer. Use this to read property values - * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, open() allocates memory for private_data. - * @return 0 if ok. < 0 if error. - */ - - void (*close) (const GstTensorFilter * filter, void **private_data); - /**< Optional. tensor_filter.c will not call other callbacks after calling close. Free-ing private_data is this function's responsibility. Set NULL after that. - * - * @param[in] filter "this" pointer. Use this to read property values - * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, close() frees private_data and set NULL. - */ - - void (*destroyNotify) (void * data); - /**< Optional. tensor_filter.c will call it when 'allocate_in_invoke' flag of the framework is TRUE. Basically, it is called when the data element is destroyed. If it's set as NULL, g_free() will be used as a default. It will be helpful when the data pointer is included as an object of a nnfw. For instance, if the data pointer is removed when the object is gone, it occurs error. In this case, the objects should be maintained for a while first and destroyed when the data pointer is destroyed. Those kinds of logic could be defined at this method. - * - * @param[in] data the data element. - */ -}; - -/* extern functions for subplugin management */ -extern gboolean tensor_filter_probe (GstTensorFilterFramework *tfsp); -extern void tensor_filter_exit (const gchar *name); - G_END_DECLS #endif /* __GST_TENSOR_FILTER_H__ */ diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_custom.c b/gst/nnstreamer/tensor_filter/tensor_filter_custom.c index 5b58bdd..5a1be82 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter_custom.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter_custom.c @@ -52,28 +52,27 @@ typedef struct _internal_data internal_data; * @return 0 if successfully loaded. 1 if skipped (already loaded). -1 if error */ static int -custom_loadlib (const GstTensorFilter * filter, void **private_data) +custom_loadlib (const GstTensorFilterProperties * prop, void **private_data) { internal_data *ptr; char *dlsym_error; - if (filter->privateData != NULL) { + if (*private_data != NULL) { /** @todo : Check the integrity of filter->data and filter->model_file, nnfw */ return 1; } - if (!filter->prop.model_file || filter->prop.model_file[0] == '\0') { + if (!prop->model_file || prop->model_file[0] == '\0') { /* The .so file path is not given */ return -1; } ptr = g_new0 (internal_data, 1); /* Fill Zero! */ *private_data = ptr; - g_assert (*private_data == filter->privateData); ptr->parent = GstTensorFilter_of_privateData (private_data); /* Load .so if this is the first time for this instance. */ - ptr->handle = dlopen (filter->prop.model_file, RTLD_NOW); + ptr->handle = dlopen (prop->model_file, RTLD_NOW); if (!ptr->handle) { g_free (ptr); *private_data = NULL; @@ -93,7 +92,7 @@ custom_loadlib (const GstTensorFilter * filter, void **private_data) } g_assert (ptr->methods->initfunc); - ptr->customFW_private_data = ptr->methods->initfunc (&(filter->prop)); + ptr->customFW_private_data = ptr->methods->initfunc (prop); /* After init func, (getInput XOR setInput) && (getOutput XOR setInput) must hold! */ /** @todo Double check if this check is really required and safe */ @@ -107,9 +106,9 @@ custom_loadlib (const GstTensorFilter * filter, void **private_data) * @brief The open callback for GstTensorFilterFramework. Called before anything else */ static int -custom_open (const GstTensorFilter * filter, void **private_data) +custom_open (const GstTensorFilterProperties * prop, void **private_data) { - int retval = custom_loadlib (filter, private_data); + int retval = custom_loadlib (prop, private_data); internal_data *ptr; /* This must be called only once */ @@ -128,29 +127,29 @@ custom_open (const GstTensorFilter * filter, void **private_data) /** * @brief The mandatory callback for GstTensorFilterFramework - * @param filter The parent object + * @param prop The properties of parent object * @param[in] input The array of input tensors * @param[out] output The array of output tensors * @return 0 if OK. non-zero if error. */ static int -custom_invoke (const GstTensorFilter * filter, void **private_data, +custom_invoke (const GstTensorFilterProperties * prop, void **private_data, const GstTensorMemory * input, GstTensorMemory * output) { - int retval = custom_loadlib (filter, private_data); + int retval = custom_loadlib (prop, private_data); internal_data *ptr; /* Actually, tensor_filter must have called getInput/OotputDim first. */ g_assert (retval == 1); - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); ptr = *private_data; if (ptr->methods->invoke) { - return ptr->methods->invoke (ptr->customFW_private_data, &(filter->prop), + return ptr->methods->invoke (ptr->customFW_private_data, prop, input, output); } else if (ptr->methods->allocate_invoke) { return ptr->methods->allocate_invoke (ptr->customFW_private_data, - &(filter->prop), input, output); + prop, input, output); } else { return -1; } @@ -160,79 +159,76 @@ custom_invoke (const GstTensorFilter * filter, void **private_data, * @brief The optional callback for GstTensorFilterFramework */ static int -custom_getInputDim (const GstTensorFilter * filter, void **private_data, +custom_getInputDim (const GstTensorFilterProperties * prop, void **private_data, GstTensorsInfo * info) { - int retval = custom_loadlib (filter, private_data); + int retval = custom_loadlib (prop, private_data); internal_data *ptr; g_assert (retval == 1); /* open must be called before */ - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); ptr = *private_data; if (ptr->methods->getInputDim == NULL) { return -1; } - return ptr->methods->getInputDim (ptr->customFW_private_data, &(filter->prop), - info); + return ptr->methods->getInputDim (ptr->customFW_private_data, prop, info); } /** * @brief The optional callback for GstTensorFilterFramework */ static int -custom_getOutputDim (const GstTensorFilter * filter, void **private_data, - GstTensorsInfo * info) +custom_getOutputDim (const GstTensorFilterProperties * prop, + void **private_data, GstTensorsInfo * info) { - int retval = custom_loadlib (filter, private_data); + int retval = custom_loadlib (prop, private_data); internal_data *ptr; g_assert (retval == 1); /* open must be called before */ - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); ptr = *private_data; if (ptr->methods->getOutputDim == NULL) { return -1; } - return ptr->methods->getOutputDim (ptr->customFW_private_data, - &(filter->prop), info); + return ptr->methods->getOutputDim (ptr->customFW_private_data, prop, info); } /** * @brief The set-input-dim callback for GstTensorFilterFramework */ static int -custom_setInputDim (const GstTensorFilter * filter, void **private_data, +custom_setInputDim (const GstTensorFilterProperties * prop, void **private_data, const GstTensorsInfo * in_info, GstTensorsInfo * out_info) { - int retval = custom_loadlib (filter, private_data); + int retval = custom_loadlib (prop, private_data); internal_data *ptr; g_assert (retval == 1); /* open must be called before */ - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); ptr = *private_data; if (ptr->methods->setInputDim == NULL) return -1; return ptr->methods->setInputDim (ptr->customFW_private_data, - &(filter->prop), in_info, out_info); + prop, in_info, out_info); } /** * @brief Free privateData and move on. */ static void -custom_close (const GstTensorFilter * filter, void **private_data) +custom_close (const GstTensorFilterProperties * prop, void **private_data) { internal_data *ptr = *private_data; - ptr->methods->exitfunc (ptr->customFW_private_data, &(filter->prop)); + ptr->methods->exitfunc (ptr->customFW_private_data, prop); g_free (ptr); *private_data = NULL; - g_assert (filter->privateData == NULL); } static GstTensorFilterFramework NNS_support_custom = { diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow.c b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow.c index 25fe4b9..296a306 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow.c @@ -26,7 +26,7 @@ * */ -#include "tensor_filter.h" +#include #include "tensor_filter_tensorflow_core.h" #include #include @@ -45,79 +45,78 @@ typedef struct _Tf_data tf_data; * @brief Free privateData and move on. */ static void -tf_close (const GstTensorFilter * filter, void **private_data) +tf_close (const GstTensorFilterProperties * prop, void **private_data) { tf_data *tf; tf = *private_data; tf_core_delete (tf->tf_private_data); g_free (tf); *private_data = NULL; - g_assert (filter->privateData == NULL); } /** * @brief Load tensorflow modelfile - * @param filter : tensor_filter instance + * @param prop: property of tensor_filter instance * @param private_data : tensorflow plugin's private data * @return 0 if successfully loaded. 1 if skipped (already loaded). * -1 if the object construction is failed. * -2 if the object initialization if failed */ static int -tf_loadModelFile (const GstTensorFilter * filter, void **private_data) +tf_loadModelFile (const GstTensorFilterProperties * prop, void **private_data) { tf_data *tf; - if (filter->privateData != NULL) { - /** @todo : Check the integrity of filter->data and filter->model_file, nnfw */ + if (*private_data != NULL) { tf = *private_data; - if (strcmp (filter->prop.model_file, - tf_core_getModelPath (tf->tf_private_data))) { - tf_close (filter, private_data); + if (strcmp (prop->model_file, tf_core_getModelPath (tf->tf_private_data))) { + tf_close (prop, private_data); } else { return 1; } } tf = g_new0 (tf_data, 1); /** initialize tf Fill Zero! */ *private_data = tf; - tf->tf_private_data = tf_core_new (filter->prop.model_file); + tf->tf_private_data = tf_core_new (prop->model_file); if (tf->tf_private_data) { - if (tf_core_init (tf->tf_private_data, &filter->prop)) { - GST_ERROR ("failed to initailize the object: tensorflow"); + if (tf_core_init (tf->tf_private_data, prop)) { + g_printerr ("failed to initailize the object: tensorflow"); return -2; } return 0; } else { - GST_ERROR ("failed to create the object: tensorflow"); + g_printerr ("failed to create the object: tensorflow"); return -1; } } /** * @brief The open callback for GstTensorFilterFramework. Called before anything else - * @param filter : tensor_filter instance + * @param prop: property of tensor_filter instance * @param private_data : tensorflow plugin's private data */ static int -tf_open (const GstTensorFilter * filter, void **private_data) +tf_open (const GstTensorFilterProperties * prop, void **private_data) { - int retval = tf_loadModelFile (filter, private_data); + int retval = tf_loadModelFile (prop, private_data); g_assert (retval == 0); /** This must be called only once */ return 0; } /** * @brief The mandatory callback for GstTensorFilterFramework + * @param prop: property of tensor_filter instance + * @param private_data : tensorflow plugin's private data * @param[in] input The array of input tensors * @param[out] output The array of output tensors */ static int -tf_run (const GstTensorFilter * filter, void **private_data, +tf_run (const GstTensorFilterProperties * prop, void **private_data, const GstTensorMemory * input, GstTensorMemory * output) { int retval; tf_data *tf; tf = *private_data; - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); retval = tf_core_run (tf->tf_private_data, input, output); g_assert (retval == 0); return retval; @@ -125,28 +124,34 @@ tf_run (const GstTensorFilter * filter, void **private_data, /** * @brief The optional callback for GstTensorFilterFramework + * @param prop: property of tensor_filter instance + * @param private_data : tensorflow plugin's private data + * @param[out] info The dimesions and types of input tensors */ static int -tf_getInputDim (const GstTensorFilter * filter, void **private_data, +tf_getInputDim (const GstTensorFilterProperties * prop, void **private_data, GstTensorsInfo * info) { tf_data *tf; tf = *private_data; - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); int ret = tf_core_getInputDim (tf->tf_private_data, info); return ret; } /** * @brief The optional callback for GstTensorFilterFramework + * @param prop: property of tensor_filter instance + * @param private_data : tensorflow plugin's private data + * @param[out] info The dimesions and types of output tensors */ static int -tf_getOutputDim (const GstTensorFilter * filter, void **private_data, +tf_getOutputDim (const GstTensorFilterProperties * prop, void **private_data, GstTensorsInfo * info) { tf_data *tf; tf = *private_data; - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); int ret = tf_core_getOutputDim (tf->tf_private_data, info); return ret; } diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.h b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.h index 835f27c..26bf7db 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.h +++ b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.h @@ -26,6 +26,8 @@ #ifndef TENSOR_FILTER_TENSORFLOW_H #define TENSOR_FILTER_TENSORFLOW_H +#include + #ifdef __cplusplus #include #include @@ -48,7 +50,6 @@ #include #include -#include using namespace tensorflow; diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c index 9dffa89..7b73a84 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c @@ -26,7 +26,7 @@ * */ -#include "tensor_filter.h" +#include #include "tensor_filter_tensorflow_lite_core.h" #include #include @@ -45,78 +45,80 @@ typedef struct _Tflite_data tflite_data; * @brief Free privateData and move on. */ static void -tflite_close (const GstTensorFilter * filter, void **private_data) +tflite_close (const GstTensorFilterProperties * prop, void **private_data) { tflite_data *tf; tf = *private_data; tflite_core_delete (tf->tflite_private_data); g_free (tf); *private_data = NULL; - g_assert (filter->privateData == NULL); } /** * @brief Load tensorflow lite modelfile - * @param filter : tensor_filter instance + * @param prop property of tensor_filter instance * @param private_data : tensorflow lite plugin's private data * @return 0 if successfully loaded. 1 if skipped (already loaded). * -1 if the object construction is failed. * -2 if the object initialization if failed */ static int -tflite_loadModelFile (const GstTensorFilter * filter, void **private_data) +tflite_loadModelFile (const GstTensorFilterProperties * prop, + void **private_data) { tflite_data *tf; - if (filter->privateData != NULL) { + if (*private_data != NULL) { /** @todo : Check the integrity of filter->data and filter->model_file, nnfw */ tf = *private_data; - if (strcmp (filter->prop.model_file, + if (strcmp (prop->model_file, tflite_core_getModelPath (tf->tflite_private_data))) { - tflite_close (filter, private_data); + tflite_close (prop, private_data); } else { return 1; } } tf = g_new0 (tflite_data, 1); /** initialize tf Fill Zero! */ *private_data = tf; - tf->tflite_private_data = tflite_core_new (filter->prop.model_file); + tf->tflite_private_data = tflite_core_new (prop->model_file); if (tf->tflite_private_data) { if (tflite_core_init (tf->tflite_private_data)) { - GST_ERROR ("failed to initialize the object: Tensorflow-lite"); + g_printerr ("failed to initialize the object: Tensorflow-lite"); return -2; } return 0; } else { - GST_ERROR ("failed to create the object: Tensorflow-lite"); + g_printerr ("failed to create the object: Tensorflow-lite"); return -1; } } /** * @brief The open callback for GstTensorFilterFramework. Called before anything else - * @param filter : tensor_filter instance + * @param prop property of tensor_filter instance * @param private_data : tensorflow lite plugin's private data */ static int -tflite_open (const GstTensorFilter * filter, void **private_data) +tflite_open (const GstTensorFilterProperties * prop, void **private_data) { - return tflite_loadModelFile (filter, private_data); + return tflite_loadModelFile (prop, private_data); } /** * @brief The mandatory callback for GstTensorFilterFramework + * @param prop property of tensor_filter instance + * @param private_data : tensorflow lite plugin's private data * @param[in] input The array of input tensors * @param[out] output The array of output tensors * @return 0 if OK. non-zero if error. */ static int -tflite_invoke (const GstTensorFilter * filter, void **private_data, +tflite_invoke (const GstTensorFilterProperties * prop, void **private_data, const GstTensorMemory * input, GstTensorMemory * output) { int retval; tflite_data *tf; tf = *private_data; - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); retval = tflite_core_invoke (tf->tflite_private_data, input, output); g_assert (retval == 0); return retval; @@ -124,28 +126,34 @@ tflite_invoke (const GstTensorFilter * filter, void **private_data, /** * @brief The optional callback for GstTensorFilterFramework + * @param prop property of tensor_filter instance + * @param private_data : tensorflow lite plugin's private data + * @param[out] info The dimesions and types of input tensors */ static int -tflite_getInputDim (const GstTensorFilter * filter, void **private_data, +tflite_getInputDim (const GstTensorFilterProperties * prop, void **private_data, GstTensorsInfo * info) { tflite_data *tf; tf = *private_data; - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); int ret = tflite_core_getInputDim (tf->tflite_private_data, info); return ret; } /** * @brief The optional callback for GstTensorFilterFramework + * @param prop property of tensor_filter instance + * @param private_data : tensorflow lite plugin's private data + * @param[out] info The dimesions and types of output tensors */ static int -tflite_getOutputDim (const GstTensorFilter * filter, void **private_data, - GstTensorsInfo * info) +tflite_getOutputDim (const GstTensorFilterProperties * prop, + void **private_data, GstTensorsInfo * info) { tflite_data *tf; tf = *private_data; - g_assert (filter->privateData && *private_data == filter->privateData); + g_assert (*private_data); int ret = tflite_core_getOutputDim (tf->tflite_private_data, info); return ret; } diff --git a/gst/nnstreamer/tensor_typedef.h b/gst/nnstreamer/tensor_typedef.h index d2e5cbe..1d9b25a 100644 --- a/gst/nnstreamer/tensor_typedef.h +++ b/gst/nnstreamer/tensor_typedef.h @@ -198,6 +198,8 @@ typedef struct int32_t rate_d; /**< framerate is in fraction, which is numerator/denominator */ } GstTensorsConfig; + +/** @todo Separate headers per subplugin-category */ /** * @brief GstTensorFilter's properties for NN framework (internal data structure) * @@ -219,4 +221,97 @@ typedef struct _GstTensorFilterProperties const char *custom_properties; /**< sub-plugin specific custom property values in string */ } GstTensorFilterProperties; +/** + * @brief Tensor_Filter Subplugin definition + * + * Common callback parameters: + * filter Filter properties. Read Only + * private_data Subplugin's private data. Set this (*private_data = XXX) if you want to change filter->private_data + */ +typedef struct _GstTensorFilterFramework +{ + char *name; /**< Name of the neural network framework, searchable by FRAMEWORK property */ + int allow_in_place; /**< TRUE(nonzero) if InPlace transfer of input-to-output is allowed. Not supported in main, yet */ + int allocate_in_invoke; /**< TRUE(nonzero) if invoke_NN is going to allocate outputptr by itself and return the address via outputptr. Do not change this value after cap negotiation is complete (or the stream has been started). */ + + int (*invoke_NN) (const GstTensorFilterProperties * prop, void **private_data, + const GstTensorMemory * input, GstTensorMemory * output); + /**< Mandatory callback. Invoke the given network model. + * + * @param[in] prop read-only property values + * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. + * @param[in] input The array of input tensors. Allocated and filled by tensor_filter/main + * @param[out] output The array of output tensors. Allocated by tensor_filter/main and to be filled by invoke_NN. If allocate_in_invoke is TRUE, sub-plugin should allocate the memory block for output tensor. (data in GstTensorMemory) + * @return 0 if OK. non-zero if error. + */ + + int (*getInputDimension) (const GstTensorFilterProperties * prop, + void **private_data, GstTensorsInfo * info); + /**< Optional. Set NULL if not supported. Get dimension of input tensor + * If getInputDimension is NULL, setInputDimension must be defined. + * If getInputDimension is defined, it is recommended to define getOutputDimension + * + * @param[in] prop read-only property values + * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. + * @param[out] info structure of tensor info (return value) + * @return the size of input tensors + */ + + int (*getOutputDimension) (const GstTensorFilterProperties * prop, + void **private_data, GstTensorsInfo * info); + /**< Optional. Set NULL if not supported. Get dimension of output tensor + * If getInputDimension is NULL, setInputDimension must be defined. + * If getInputDimension is defined, it is recommended to define getOutputDimension + * + * @param[in] prop read-only property values + * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. + * @param[out] info structure of tensor info (return value) + * @return the size of output tensors + */ + + int (*setInputDimension) (const GstTensorFilterProperties * prop, + void **private_data, const GstTensorsInfo * in_info, + GstTensorsInfo * out_info); + /**< Optional. Set Null if not supported. Tensor_filter::main will + * configure input dimension from pad-cap in run-time for the sub-plugin. + * Then, the sub-plugin is required to return corresponding output dimension + * If this is NULL, both getInput/OutputDimension must be non-NULL. + * + * When you use this, do NOT allocate or fix internal data structure based on it + * until invoke is called. Gstreamer may try different dimensions before + * settling down. + * + * @param[in] prop read-only property values + * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. + * @param[in] in_info structure of input tensor info + * @param[out] out_info structure of output tensor info (return value) + * @return 0 if OK. non-zero if error. + */ + + int (*open) (const GstTensorFilterProperties * prop, void **private_data); + /**< Optional. tensor_filter.c will call this before any of other callbacks and will call once before calling close + * + * @param[in] prop read-only property values + * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, open() allocates memory for private_data. + * @return 0 if ok. < 0 if error. + */ + + void (*close) (const GstTensorFilterProperties * prop, void **private_data); + /**< Optional. tensor_filter.c will not call other callbacks after calling close. Free-ing private_data is this function's responsibility. Set NULL after that. + * + * @param[in] prop read-only property values + * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, close() frees private_data and set NULL. + */ + + void (*destroyNotify) (void * data); + /**< Optional. tensor_filter.c will call it when 'allocate_in_invoke' flag of the framework is TRUE. Basically, it is called when the data element is destroyed. If it's set as NULL, g_free() will be used as a default. It will be helpful when the data pointer is included as an object of a nnfw. For instance, if the data pointer is removed when the object is gone, it occurs error. In this case, the objects should be maintained for a while first and destroyed when the data pointer is destroyed. Those kinds of logic could be defined at this method. + * + * @param[in] data the data element. + */ +} GstTensorFilterFramework; + +/* extern functions for subplugin management, exist in tensor_filter.c */ +extern int tensor_filter_probe (GstTensorFilterFramework *tfsp); +extern void tensor_filter_exit (const char *name); + #endif /*__GST_TENSOR_TYPEDEF_H__*/ -- 2.7.4