static gchar filter_subplugin_armnn[] = "armnn";
static GstTensorFilterFramework NNS_support_armnn = {
- .name = filter_subplugin_armnn,
- .allow_in_place = FALSE, /** @todo: support this to optimize performance later. */
- .allocate_in_invoke = FALSE,
- .run_without_model = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = armnn_invoke,
- .getInputDimension = armnn_getInputDim,
- .getOutputDimension = armnn_getOutputDim,
- .setInputDimension = NULL,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = armnn_open,
.close = armnn_close,
};
void
init_filter_armnn (void)
{
+ NNS_support_armnn.name = filter_subplugin_armnn;
+ NNS_support_armnn.allow_in_place = FALSE; /** @todo: support this to optimize performance later. */
+ NNS_support_armnn.allocate_in_invoke = FALSE;
+ NNS_support_armnn.run_without_model = FALSE;
+ NNS_support_armnn.verify_model_path = FALSE;
+ NNS_support_armnn.invoke_NN = armnn_invoke;
+ NNS_support_armnn.getInputDimension = armnn_getInputDim;
+ NNS_support_armnn.getOutputDimension = armnn_getOutputDim;
+
nnstreamer_filter_probe (&NNS_support_armnn);
}
* @param[in] data The data element.
*/
static void
-caffe2_destroyNotify (void *data)
+caffe2_destroyNotify (void **private_data, void *data)
{
/* do nothing */
}
static gchar filter_subplugin_caffe2[] = "caffe2";
static GstTensorFilterFramework NNS_support_caffe2 = {
- .name = filter_subplugin_caffe2,
- .allow_in_place = FALSE, /** @todo: support this to optimize performance later. */
- .allocate_in_invoke = TRUE,
- .run_without_model = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = caffe2_run,
- .getInputDimension = caffe2_getInputDim,
- .getOutputDimension = caffe2_getOutputDim,
- .setInputDimension = NULL,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = caffe2_open,
.close = caffe2_close,
- .destroyNotify = caffe2_destroyNotify,
};
/** @brief Initialize this object for tensor_filter subplugin runtime register */
void
init_filter_caffe2 (void)
{
+ NNS_support_caffe2.name = filter_subplugin_caffe2;
+ NNS_support_caffe2.allow_in_place = FALSE; /** @todo: support this to optimize performance later. */
+ NNS_support_caffe2.allocate_in_invoke = TRUE;
+ NNS_support_caffe2.run_without_model = FALSE;
+ NNS_support_caffe2.verify_model_path = FALSE;
+ NNS_support_caffe2.invoke_NN = caffe2_run;
+ NNS_support_caffe2.getInputDimension = caffe2_getInputDim;
+ NNS_support_caffe2.getOutputDimension = caffe2_getOutputDim;
+ NNS_support_caffe2.destroyNotify = caffe2_destroyNotify;
+
nnstreamer_filter_probe (&NNS_support_caffe2);
}
bool tensor_filter_cpp::dlclose_all_called = false;
static GstTensorFilterFramework NNS_support_cpp = {
- .name = filter_subplugin_cpp,
- .allow_in_place = FALSE, /** @todo: support this to optimize performance later. */
- .allocate_in_invoke = FALSE,
- .run_without_model = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = tensor_filter_cpp::invoke,
- .getInputDimension = tensor_filter_cpp::getInputDim,
- .getOutputDimension = tensor_filter_cpp::getOutputDim,
- .setInputDimension = tensor_filter_cpp::setInputDim,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = tensor_filter_cpp::open,
.close = tensor_filter_cpp::close,
};
+G_BEGIN_DECLS
+void init_filter_cpp (void) __attribute__ ((constructor));
+void fini_filter_cpp (void) __attribute__ ((destructor));
+
+/** @brief Initialize this object for tensor_filter subplugin runtime register */
+void
+init_filter_cpp (void)
+{
+ NNS_support_cpp.name = filter_subplugin_cpp;
+ NNS_support_cpp.allow_in_place = FALSE; /** @todo: support this to optimize performance later. */
+ NNS_support_cpp.allocate_in_invoke = FALSE;
+ NNS_support_cpp.run_without_model = FALSE;
+ NNS_support_cpp.verify_model_path = FALSE;
+ NNS_support_cpp.invoke_NN = tensor_filter_cpp::invoke;
+ NNS_support_cpp.getInputDimension = tensor_filter_cpp::getInputDim;
+ NNS_support_cpp.getOutputDimension = tensor_filter_cpp::getOutputDim;
+ NNS_support_cpp.setInputDimension = tensor_filter_cpp::setInputDim;
+ nnstreamer_filter_probe (&NNS_support_cpp);
+}
+
+/** @brief Destruct the subplugin */
+void
+fini_filter_cpp (void)
+{
+ nnstreamer_filter_exit (NNS_support_cpp.name);
+ tensor_filter_cpp::dlclose_all ();
+}
+G_END_DECLS
#define loadClass(name, ptr) \
class tensor_filter_cpp *name = (tensor_filter_cpp *) *(ptr); \
cpp->ref_count--;
}
-G_BEGIN_DECLS
-void init_filter_cpp (void) __attribute__ ((constructor));
-void fini_filter_cpp (void) __attribute__ ((destructor));
-
/**
* @brief Call dlclose for all handle
*/
#endif
dlclose_all_called = true;
}
-
-/** @brief Initialize this object for tensor_filter subplugin runtime register */
-void
-init_filter_cpp (void)
-{
- nnstreamer_filter_probe (&NNS_support_cpp);
-}
-
-/** @brief Destruct the subplugin */
-void
-fini_filter_cpp (void)
-{
- nnstreamer_filter_exit (NNS_support_cpp.name);
- tensor_filter_cpp::dlclose_all ();
-}
-G_END_DECLS
static gchar filter_subplugin_edgetpu[] = "edgetpu";
static GstTensorFilterFramework NNS_support_edgetpu = {
- .name = filter_subplugin_edgetpu,
- .allow_in_place = FALSE,
- .allocate_in_invoke = FALSE,
- .run_without_model = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = edgetpu_invoke,
- .getInputDimension = edgetpu_getInputDim,
- .getOutputDimension = edgetpu_getOutputDim,
- .setInputDimension = NULL,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = edgetpu_open,
.close = edgetpu_close,
- .destroyNotify = NULL,
- .reloadModel = NULL,
};
/**@brief Initialize this object for tensor_filter subplugin runtime register */
void
init_filter_edgetpu (void)
{
+ NNS_support_edgetpu.name = filter_subplugin_edgetpu;
+ NNS_support_edgetpu.allow_in_place = FALSE;
+ NNS_support_edgetpu.allocate_in_invoke = FALSE;
+ NNS_support_edgetpu.run_without_model = FALSE;
+ NNS_support_edgetpu.verify_model_path = FALSE;
+ NNS_support_edgetpu.invoke_NN = edgetpu_invoke;
+ NNS_support_edgetpu.getInputDimension = edgetpu_getInputDim;
+ NNS_support_edgetpu.getOutputDimension = edgetpu_getOutputDim;
+
nnstreamer_filter_probe (&NNS_support_edgetpu);
}
static gchar filter_subplugin_movidius_ncsdk2[] = "movidius-ncsdk2";
static GstTensorFilterFramework NNS_support_movidius_ncsdk2 = {
- .name = filter_subplugin_movidius_ncsdk2,
- .allow_in_place = FALSE,
- .allocate_in_invoke = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = _mvncsdk2_invoke,
- .getInputDimension = _mvncsdk2_getInputDim,
- .getOutputDimension = _mvncsdk2_getOutputDim,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = _mvncsdk2_open,
.close = _mvncsdk2_close,
};
void
init_filter_mvncsdk2 (void)
{
+ NNS_support_movidius_ncsdk2.name = filter_subplugin_movidius_ncsdk2;
+ NNS_support_movidius_ncsdk2.allow_in_place = FALSE;
+ NNS_support_movidius_ncsdk2.allocate_in_invoke = FALSE;
+ NNS_support_movidius_ncsdk2.verify_model_path = FALSE;
+ NNS_support_movidius_ncsdk2.invoke_NN = _mvncsdk2_invoke;
+ NNS_support_movidius_ncsdk2.getInputDimension = _mvncsdk2_getInputDim;
+ NNS_support_movidius_ncsdk2.getOutputDimension = _mvncsdk2_getOutputDim;
+
nnstreamer_filter_probe (&NNS_support_movidius_ncsdk2);
}
static gchar filter_subplugin_nnfw[] = "nnfw";
static GstTensorFilterFramework NNS_support_nnfw = {
- .name = filter_subplugin_nnfw,
- .allow_in_place = FALSE,
- .allocate_in_invoke = FALSE,
- .run_without_model = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = nnfw_invoke,
- .getInputDimension = nnfw_getInputDim,
- .getOutputDimension = nnfw_getOutputDim,
- .setInputDimension = nnfw_setInputDim,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = nnfw_open,
.close = nnfw_close,
};
void
init_filter_nnfw (void)
{
+ NNS_support_nnfw.name = filter_subplugin_nnfw;
+ NNS_support_nnfw.allow_in_place = FALSE;
+ NNS_support_nnfw.allocate_in_invoke = FALSE;
+ NNS_support_nnfw.run_without_model = FALSE;
+ NNS_support_nnfw.verify_model_path = FALSE;
+ NNS_support_nnfw.invoke_NN = nnfw_invoke;
+ NNS_support_nnfw.getInputDimension = nnfw_getInputDim;
+ NNS_support_nnfw.getOutputDimension = nnfw_getOutputDim;
+ NNS_support_nnfw.setInputDimension = nnfw_setInputDim;
+
nnstreamer_filter_probe (&NNS_support_nnfw);
}
static gchar filter_subplugin_openvino[] = "openvino";
static GstTensorFilterFramework NNS_support_openvino = {
- .name = filter_subplugin_openvino,
- .allow_in_place = FALSE,
- .allocate_in_invoke = FALSE,
- .run_without_model = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = ov_invoke,
- .getInputDimension = ov_getInputDim,
- .getOutputDimension = ov_getOutputDim,
- .setInputDimension = NULL,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = ov_open,
.close = ov_close,
};
void
init_filter_openvino (void)
{
+ NNS_support_openvino.name = filter_subplugin_openvino;
+ NNS_support_openvino.allow_in_place = FALSE;
+ NNS_support_openvino.allocate_in_invoke = FALSE;
+ NNS_support_openvino.run_without_model = FALSE;
+ NNS_support_openvino.verify_model_path = FALSE;
+ NNS_support_openvino.invoke_NN = ov_invoke;
+ NNS_support_openvino.getInputDimension = ov_getInputDim;
+ NNS_support_openvino.getOutputDimension = ov_getOutputDim;
+
nnstreamer_filter_probe (&NNS_support_openvino);
}
* @param[in] data The data element.
*/
static void
-py_destroyNotify (void *data)
+py_destroyNotify (void **private_data, void *data)
{
std::map <void*, PyArrayObject*>::iterator it;
it = PYCore::outputArrayMap.find(data);
#endif
static GstTensorFilterFramework NNS_support_python = {
- .name = filter_subplugin_python,
- .allow_in_place = FALSE, /** @todo: support this to optimize performance later. */
- .allocate_in_invoke = TRUE,
- .run_without_model = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = py_run,
- /** dimension-related callbacks are dynamically assigned */
- .getInputDimension = py_getInputDim,
- .getOutputDimension = py_getOutputDim,
- .setInputDimension = py_setInputDim,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = py_open,
.close = py_close,
- .destroyNotify = py_destroyNotify,
+ {
+ }
};
/** @brief Initialize this object for tensor_filter subplugin runtime register */
void
init_filter_py (void)
{
+ NNS_support_python.name = filter_subplugin_python;
+ NNS_support_python.allow_in_place = FALSE; /** @todo: support this to optimize performance later. */
+ NNS_support_python.allocate_in_invoke = TRUE;
+ NNS_support_python.run_without_model = FALSE;
+ NNS_support_python.verify_model_path = FALSE;
+ NNS_support_python.invoke_NN = py_run;
+ /** dimension-related callbacks are dynamically updated */
+ NNS_support_python.getInputDimension = py_getInputDim;
+ NNS_support_python.getOutputDimension = py_getOutputDim;
+ NNS_support_python.setInputDimension = py_setInputDim;
+ NNS_support_python.destroyNotify = py_destroyNotify;
+
nnstreamer_filter_probe (&NNS_support_python);
filter_framework = &NNS_support_python;
/** Python should be initialized and finalized only once */
static gchar filter_subplugin_pytorch[] = "pytorch";
static GstTensorFilterFramework NNS_support_pytorch = {
- .name = filter_subplugin_pytorch,
- .allow_in_place = FALSE,
- .allocate_in_invoke = FALSE,
- .run_without_model = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = torch_invoke,
- .getInputDimension = torch_getInputDim,
- .getOutputDimension = torch_getOutputDim,
- .setInputDimension = NULL,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = torch_open,
.close = torch_close,
- .destroyNotify = NULL,
- .reloadModel = NULL,
- .checkAvailability = torch_checkAvailability,
};
/** @brief Initialize this object for tensor_filter subplugin runtime register */
void
init_filter_torch (void)
{
+ NNS_support_pytorch.name = filter_subplugin_pytorch;
+ NNS_support_pytorch.allow_in_place = FALSE;
+ NNS_support_pytorch.allocate_in_invoke = FALSE;
+ NNS_support_pytorch.run_without_model = FALSE;
+ NNS_support_pytorch.verify_model_path = FALSE;
+ NNS_support_pytorch.invoke_NN = torch_invoke;
+ NNS_support_pytorch.getInputDimension = torch_getInputDim;
+ NNS_support_pytorch.getOutputDimension = torch_getOutputDim;
+ NNS_support_pytorch.checkAvailability = torch_checkAvailability;
+
nnstreamer_filter_probe (&NNS_support_pytorch);
}
* @param[in] data The data element.
*/
static void
-tf_destroyNotify (void *data)
+tf_destroyNotify (void **private_data, void *data)
{
TF_DeleteTensor ( (TFCore::outputTensorMap.find (data))->second);
TFCore::outputTensorMap.erase (data);
static gchar filter_subplugin_tensorflow[] = "tensorflow";
static GstTensorFilterFramework NNS_support_tensorflow = {
- .name = filter_subplugin_tensorflow,
- .allow_in_place = FALSE, /** @todo: support this to optimize performance later. */
- .allocate_in_invoke = TRUE,
- .run_without_model = FALSE,
- .verify_model_path = FALSE,
- .invoke_NN = tf_run,
- .getInputDimension = tf_getInputDim,
- .getOutputDimension = tf_getOutputDim,
- .setInputDimension = NULL,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = tf_open,
.close = tf_close,
- .destroyNotify = tf_destroyNotify,
};
/** @brief Initialize this object for tensor_filter subplugin runtime register */
void
init_filter_tf (void)
{
+ NNS_support_tensorflow.name = filter_subplugin_tensorflow;
+ NNS_support_tensorflow.allow_in_place = FALSE; /** @todo: support this to optimize performance later. */
+ NNS_support_tensorflow.allocate_in_invoke = TRUE;
+ NNS_support_tensorflow.run_without_model = FALSE;
+ NNS_support_tensorflow.verify_model_path = FALSE;
+ NNS_support_tensorflow.invoke_NN = tf_run;
+ NNS_support_tensorflow.getInputDimension = tf_getInputDim;
+ NNS_support_tensorflow.getOutputDimension = tf_getOutputDim;
+ NNS_support_tensorflow.destroyNotify = tf_destroyNotify;
+
nnstreamer_filter_probe (&NNS_support_tensorflow);
}
static gchar filter_subplugin_tensorflow_lite[] = "tensorflow-lite";
static GstTensorFilterFramework NNS_support_tensorflow_lite = {
- .name = filter_subplugin_tensorflow_lite,
- .allow_in_place = FALSE, /** @todo: support this to optimize performance later. */
- .allocate_in_invoke = FALSE,
- .run_without_model = FALSE,
- .verify_model_path = TRUE,
- .invoke_NN = tflite_invoke,
- .getInputDimension = tflite_getInputDim,
- .getOutputDimension = tflite_getOutputDim,
- .setInputDimension = tflite_setInputDim,
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.open = tflite_open,
.close = tflite_close,
- .destroyNotify = NULL,
- .reloadModel = tflite_reloadModel,
- .checkAvailability = tflite_checkAvailability,
};
/** @brief Initialize this object for tensor_filter subplugin runtime register */
void
init_filter_tflite (void)
{
+ NNS_support_tensorflow_lite.name = filter_subplugin_tensorflow_lite,
+ NNS_support_tensorflow_lite.allow_in_place = FALSE; /** @todo: support this to optimize performance later. */
+ NNS_support_tensorflow_lite.allocate_in_invoke = FALSE;
+ NNS_support_tensorflow_lite.run_without_model = FALSE;
+ NNS_support_tensorflow_lite.verify_model_path = TRUE;
+ NNS_support_tensorflow_lite.invoke_NN = tflite_invoke;
+ NNS_support_tensorflow_lite.getInputDimension = tflite_getInputDim;
+ NNS_support_tensorflow_lite.getOutputDimension = tflite_getOutputDim;
+ NNS_support_tensorflow_lite.setInputDimension = tflite_setInputDim;
+ NNS_support_tensorflow_lite.reloadModel = tflite_reloadModel;
+ NNS_support_tensorflow_lite.checkAvailability = tflite_checkAvailability;
nnstreamer_filter_probe (&NNS_support_tensorflow_lite);
}
#define ACCL_NPU_SRCN_STR "npu.srcn" /** srcn hardware supported by nnfw */
#define ACCL_NPU_SR_STR "npu.sr"
+#define GST_TENSOR_FILTER_FRAMEWORK_BASE (0xDEAFDEAD00000000ULL)
+#define GST_TENSOR_FILTER_FRAMEWORK_V0 (GST_TENSOR_FILTER_FRAMEWORK_BASE)
+#define GST_TENSOR_FILTER_FRAMEWORK_V1 (GST_TENSOR_FILTER_FRAMEWORK_BASE | 0x10000ULL)
+
+/** TODO: update this to 1 after supporting version 1 GstTensorFilterFramework in tensor_filter.c */
+#define GST_TENSOR_FILTER_API_VERSION_DEFINED (0)
+#define GST_TENSOR_FILTER_API_VERSION_MIN (0) /* The minimum API version supported (could be obsolete) */
+#define GST_TENSOR_FILTER_API_VERSION_MAX (0) /* The maximum API version supported (recommended) */
+
+/**
+ * @brief Check the value of the version field of GstTensorFilterFramework
+ */
+#define checkGstTensorFitlerFrameworkVersion(value, version) \
+ ((GST_TENSOR_FILTER_FRAMEWORK_BASE | ((version) << 16)) == (value & 0xFFFFFFFFFFFF0000ULL))
+
#ifdef __cplusplus
extern "C" {
#endif
tensors_layout output_layout; /**< data layout info provided as a property to tensor_filter for the output, defaults to _NNS_LAYOUT_ANY for all the tensors */
const char *custom_properties; /**< sub-plugin specific custom property values in string */
- const char *accl_str; /**< accelerator configuration passed in as parameter */
+ union {
+ accl_hw *hw_list; /**< accelerators supported by framework intersected with user provided accelerator preference, use in GstTensorFilterFramework V1 only */
+ const char *accl_str; /**< accelerator configuration passed in as parameter, use in GstTensorFilterFramework V0 only */
+ };
} GstTensorFilterProperties;
/**
- * @brief Tensor_Filter Subplugin definition
+ * @brief Tensor_Filter Subplugin framework related information
*
- * Common callback parameters:
- * prop Filter properties. Read Only.
- * private_data Subplugin's private data. Set this (*private_data = XXX) if you want to change filter->private_data.
+ * All the information except the supported acclerator is provided statically.
+ * Accelerators can be provided based on static or dynamic check dependent on framework support.
*/
-typedef struct _GstTensorFilterFramework
+typedef struct _GstTensorFilterFrameworkInfo
{
char *name; /**< Name of the neural network framework, searchable by FRAMEWORK property */
int allow_in_place; /**< TRUE(nonzero) if InPlace transfer of input-to-output is allowed. Not supported in main, yet */
int allocate_in_invoke; /**< TRUE(nonzero) if invoke_NN is going to allocate outputptr by itself and return the address via outputptr. Do not change this value after cap negotiation is complete (or the stream has been started). */
int run_without_model; /**< TRUE(nonzero) when the neural network framework does not need a model file. Tensor-filter will run invoke_NN without model. */
int verify_model_path; /**< TRUE(nonzero) when the NNS framework, not the sub-plugin, should verify the path of model files. */
+ accl_hw *hw_list; /**< List of supported hardwares by the framework. Positive response of this check does not guarantee successful running of model with this accelerator. */
+} GstTensorFilterFrameworkInfo;
+
+/**
+ * @brief Tensor_Filter Subplugin related events
+ *
+ * These are possible set of events that can be supported by the tensor filter subplugin.
+ */
+typedef enum
+{
+ DESTROY_NOTIFY, /**< Free the data element allocated in the invoke callback */
+ RELOAD_MODEL, /**< Reloads the subplugin with newely provided model */
+ SET_INPUT_PROP, /**< Update input tensor info and layout */
+ SET_OUTPUT_PROP, /**< Update output tensor info and layout */
+ SET_ACCELERATOR, /**< Update accelerator of the subplugin to be used as backend */
+} event_ops;
+
+/**
+ * @brief Tensor_Filter Subplugin's model related info gathering operations
+ */
+typedef enum
+{
+ GET_IN_OUT_INFO, /**< Gets the input and output tensor info */
+ SET_INPUT_INFO, /**< Sets the provided input tensor info, and get updated output tensor info */
+} model_info_ops;
+
+/**
+ * @brief User data for the tensor_tilter subplugin related events
+ */
+typedef struct _GstTensorFilterFrameworkEventData
+{
+ /** Union of the user data for each event supported by eventHandler */
+ union {
+ /** for DESTROY_NOTIFY event */
+ struct {
+ void * data; /**< The data element to be destroyed */
+ };
+
+ /** for RELOAD_MODEL event */
+ struct {
+ const char **model_files; /**< Filepath to the new list of model files */
+ int num_models; /**< Updated number of the model files */
+ };
+
+ /** for SET_INPUT_PROP/SET_OUTPUT_PROP event */
+ struct {
+ const GstTensorsInfo * info; /**< The tensor info to be updated to */
+ tensors_layout layout; /**< The layout of the tensor to be updated to */
+ };
+
+ /** for SET_ACCELERATOR event */
+ struct {
+ accl_hw *hw_list; /**< accelerators supported by framework intersected with the new user provided accelerator preference */
+ };
+
+ };
+} GstTensorFilterFrameworkEventData;
+
+/**
+ * @brief Tensor_Filter Subplugin definition
+ *
+ * Common callback parameters:
+ * prop Filter properties. Read Only.
+ * private_data Subplugin's private data. Set this (*private_data = XXX) if you want to change filter->private_data.
+ */
+typedef struct _GstTensorFilterFramework
+{
+ uint64_t version;
+ /**< Version of the struct
+ * | 32bit (validity check) | 16bit (API version) | 16bit (Subplugin's internal version. Tensor_filter does not case) |
+ * API version will be 0x0 (earlier version (_GstTensorFilterFramework_v0)) or 0x1 (newer version (_GstTensorFilterFramework_v1))
+ */
+
+ int (*open) (const GstTensorFilterProperties * prop, void **private_data);
+ /**< Optional. tensor_filter.c will call this before any of other callbacks and will call once before calling close.
+ *
+ * Note: If 'open' callback is not defined, then the private_data passed in other callbacks will be NULL.
+ *
+ * @param[in] prop read-only property values
+ * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, open() allocates memory for private_data.
+ * @return 0 if ok. < 0 if error.
+ */
- int (*invoke_NN) (const GstTensorFilterProperties * prop, void **private_data,
- const GstTensorMemory * input, GstTensorMemory * output);
+ void (*close) (const GstTensorFilterProperties * prop, void **private_data);
+ /**< Optional. tensor_filter.c will not call other callbacks after calling close. Free-ing private_data is this function's responsibility. Set NULL after that.
+ *
+ * @param[in] prop read-only property values
+ * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, close() frees private_data and set NULL.
+ */
+
+ /**
+ * @brief Distinct elements between two versions of the subplugin interfaces
+ */
+ union
+ {
+ /**
+ * @brief Tensor_Filter Subplugin definition Version 0
+ */
+ struct /** _GstTensorFilterFramework_v0 */
+ {
+ char *name; /**< Name of the neural network framework, searchable by FRAMEWORK property */
+ int allow_in_place; /**< TRUE(nonzero) if InPlace transfer of input-to-output is allowed. Not supported in main, yet */
+ int allocate_in_invoke; /**< TRUE(nonzero) if invoke_NN is going to allocate outputptr by itself and return the address via outputptr. Do not change this value after cap negotiation is complete (or the stream has been started). */
+ int run_without_model; /**< TRUE(nonzero) when the neural network framework does not need a model file. Tensor-filter will run invoke_NN without model. */
+ int verify_model_path; /**< TRUE(nonzero) when the NNS framework, not the sub-plugin, should verify the path of model files. */
+
+ int (*invoke_NN) (const GstTensorFilterProperties * prop, void **private_data,
+ const GstTensorMemory * input, GstTensorMemory * output);
/**< Mandatory callback. Invoke the given network model.
*
* @param[in] prop read-only property values
* @return 0 if OK. non-zero if error.
*/
- int (*getInputDimension) (const GstTensorFilterProperties * prop,
- void **private_data, GstTensorsInfo * info);
+ int (*getInputDimension) (const GstTensorFilterProperties * prop,
+ void **private_data, GstTensorsInfo * info);
/**< Optional. Set NULL if not supported. Get dimension of input tensor
* If getInputDimension is NULL, setInputDimension must be defined.
* If getInputDimension is defined, it is recommended to define getOutputDimension
* @return 0 if OK. non-zero if error.
*/
- int (*getOutputDimension) (const GstTensorFilterProperties * prop,
- void **private_data, GstTensorsInfo * info);
+ int (*getOutputDimension) (const GstTensorFilterProperties * prop,
+ void **private_data, GstTensorsInfo * info);
/**< Optional. Set NULL if not supported. Get dimension of output tensor
* If getInputDimension is NULL, setInputDimension must be defined.
* If getInputDimension is defined, it is recommended to define getOutputDimension
* @return 0 if OK. non-zero if error.
*/
- int (*setInputDimension) (const GstTensorFilterProperties * prop,
- void **private_data, const GstTensorsInfo * in_info,
- GstTensorsInfo * out_info);
- /**< Optional. Set Null if not supported. Tensor_Filter::main will
+ int (*setInputDimension) (const GstTensorFilterProperties * prop,
+ void **private_data, const GstTensorsInfo * in_info,
+ GstTensorsInfo * out_info);
+ /**< Optional. Set Null if not supported.Tensor_Filter::main will
* configure input dimension from pad-cap in run-time for the sub-plugin.
* Then, the sub-plugin is required to return corresponding output dimension
* If this is NULL, both getInput/OutputDimension must be non-NULL.
* @return 0 if OK. non-zero if error.
*/
- int (*open) (const GstTensorFilterProperties * prop, void **private_data);
- /**< Optional. tensor_filter.c will call this before any of other callbacks and will call once before calling close
+ void (*destroyNotify) (void **private_data, void * data);
+ /**< Optional. tensor_filter.c will call it when 'allocate_in_invoke' flag of the framework is TRUE and allocateInInvoke also return enabled. Basically, it is called when the data element is destroyed. If it's set as NULL, g_free() will be used as a default. It will be helpful when the data pointer is included as an object of a nnfw. For instance, if the data pointer is removed when the object is gone, it occurs error. In this case, the objects should be maintained for a while first and destroyed when the data pointer is destroyed. Those kinds of logic could be defined at this method.
+ *
+ * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer.
+ * @param[in] data the data element.
+ */
+
+ int (*reloadModel) (const GstTensorFilterProperties * prop, void **private_data);
+ /**< Optional. tensor_filter.c will call it when a model property is newly configured. Also, 'is-updatable' property of the framework should be TRUE. This function reloads a new model specified in the 'prop' argument. Note that it requires extra memory size enough to temporarily hold both old and new models during this function to hide the reload overhead.
*
* @param[in] prop read-only property values
- * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, open() allocates memory for private_data.
+ * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, close() frees private_data and set NULL.
* @return 0 if ok. < 0 if error.
*/
- void (*close) (const GstTensorFilterProperties * prop, void **private_data);
- /**< Optional. tensor_filter.c will not call other callbacks after calling close. Free-ing private_data is this function's responsibility. Set NULL after that.
+ int (*checkAvailability) (accl_hw hw);
+ /**< Optional. Check if the provided hardware accelerator is supported. This check is static or dynamic based on framework support. Positive response of this check does not guarantee successful running of model with this accelerator. The static check can be performed without opening the framework.
+ *
+ * @param[in] hw backend accelerator hardware
+ * @return 0 if supported. -errno if not supported.
+ */
+
+ int (*allocateInInvoke) (void **private_data);
+ /**< Optional. tensor_filter.c will call it when allocate_in_invoke is set to TRUE. This check if the provided model for the framework supports allocation at invoke or not. If this is not defined, then the value of allocate_in_invoke is assumed to be final for all models.
+ *
+ * @param[in] private_data A subplugin may save its internal private data here.
+ * @return 0 if supported. -errno if not supported.
+ */
+ };
+
+ /**
+ * @brief Tensor_Filter Subplugin definition Version 1
+ */
+ struct /** _GstTensorFilterFramework_v1 */
+ {
+ int (*invoke) (const GstTensorFilterProperties * prop, void *private_data,
+ const GstTensorMemory * input, GstTensorMemory * output);
+ /**< Mandatory callback. Invoke the given network model.
*
* @param[in] prop read-only property values
- * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, close() frees private_data and set NULL.
+ * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer.
+ * @param[in] input The array of input tensors. Allocated and filled by tensor_filter/main
+ * @param[out] output The array of output tensors. Allocated by tensor_filter/main and to be filled by invoke_NN. If allocate_in_invoke is TRUE, sub-plugin should allocate the memory block for output tensor. (data in GstTensorMemory)
+ * @return 0 if OK. non-zero if error.
*/
- void (*destroyNotify) (void * data);
- /**< Optional. tensor_filter.c will call it when 'allocate_in_invoke' flag of the framework is TRUE. Basically, it is called when the data element is destroyed. If it's set as NULL, g_free() will be used as a default. It will be helpful when the data pointer is included as an object of a nnfw. For instance, if the data pointer is removed when the object is gone, it occurs error. In this case, the objects should be maintained for a while first and destroyed when the data pointer is destroyed. Those kinds of logic could be defined at this method.
+ int (*getFrameworkInfo) (const GstTensorFilterProperties * prop,
+ void *private_data, GstTensorFilterFrameworkInfo *fw_info);
+ /**< Mandatory callback. Get the framworks statically determined info. Argument 'private_data' can be NULL. If provided 'private_data' is not NULL, then some info, such as 'allocate_in_invoke', can be updated based on the model being used (inferred from the 'private_data' provided). This updated info is useful for custom filter, as some custom filter's ability to support 'allocate_in_invoke' depends on the opened model.
*
- * @param[in] data the data element.
+ * @param[in] prop read-only property values
+ * @param[in] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. This parameter can be NULL.
+ * @param[out] fw_info struct to hold frameworks info. Must be allocated by the caller (return value).
+ * @return 0 if OK. non-zero if error.
+ *
+ * @note CAUTION: private_data can be NULL if the framework is not yet opened by the caller.
*/
- int (*reloadModel) (const GstTensorFilterProperties * prop, void **private_data);
- /**< Optional. tensor_filter.c will call it when a model property is newly configured. Also, 'is-updatable' property of the framework should be TRUE. This function reloads a new model specified in the 'prop' argument. Note that it requires extra memory size enough to temporarily hold both old and new models during this function to hide the reload overhead.
+ int (*getModelInfo) (const GstTensorFilterProperties * prop,
+ void *private_data, model_info_ops ops,
+ GstTensorsInfo *in_info, GstTensorsInfo *out_info);
+ /**< Mandatory callback. Gets the model related tensor info.
+ * If ops == GET_IN_OUT_INFO, in_info would contain the input tensor info, and out_info would contain the output tensor info.
+ * If ops == SET_INPUT_INFO, in_info would contain the provided input tensor info, and out_info would contain the updated output tensor info.
+ * At least one of SET_INPUT_INFO and GET_IN_OUT_INFO operations must be supported.
+ *
+ * Note: Tensor_Filter::main will configure input dimension from pad-cap in
+ * run-time for the sub-plugin. Then, the sub-plugin is required to return
+ * corresponding output dimension, and will call SET_INPUT_INFO operation.
+ *
+ * Note: With SET_INPUT_INFO operation, the caller must NOT allocate or fix
+ * internal data structure based on the return value until invoke is called.
+ * Gstreamer may try different dimensions before settling down.
*
* @param[in] prop read-only property values
- * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer. Normally, close() frees private_data and set NULL.
- * @return 0 if ok. < 0 if error.
+ * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer.
+ * @param[in] ops operation to be performed
+ * @param[in/out] in_info structure of input tensor info
+ * @param[out] out_info structure of output tensor info (return value)
+ * @return 0 if OK. non-zero if error. -ENOENT is operation is not supported.
*/
- int (*checkAvailability) (accl_hw hw);
- /**< Optional. Check if the provided hardware accelerator is supported. This check is static or dynamic based on framework support. Positive response of this check does not guarantee successful running of model with this accelerator. The static check can be performed without opening the framework.
+ int (*eventHandler) (const GstTensorFilterProperties * prop,
+ void *private_data, event_ops ops, GstTensorFilterFrameworkEventData * data);
+ /**< Mandatory callback. Runs the event corresponding to the passed operation.
+ * If ops == DESTROY_NOTIFY: tensor_filter.c will call it when 'allocate_in_invoke' property of the framework is TRUE. Basically, it is called when the data element is destroyed. If it's set as NULL, g_free() will be used as a default. It will be helpful when the data pointer is included as an object of a nnfw. For instance, if the data pointer is removed when the object is gone, it occurs error. In this case, the objects should be maintained for a while first and destroyed when the data pointer is destroyed. Those kinds of logic could be defined at this method.
+ * If ops == RELOAD_MODEL: tensor_filter.c will call it when a model property is newly configured. Also, 'is-updatable' property of the framework should be TRUE. This function reloads a new model passed in as argument via data. Note that it requires extra memory size enough to temporarily hold both old and new models during this function to hide the reload overhead.
+ * If ops == SET_INPUT_PROP: tensor_filter will call to update the property of the subplugin. This function will take tensor info and layout as the argument. This operation can update input tensor shape, type, name and layout.
+ * If ops == SET_OUTPUT_PROP: tensor_filter will call to update the property of the subplugin. This function will take tensor info and layout as the argument. This operation can update output tensor shape, type, name and layout.
+ * If ops == SET_ACCELERATOR: tensor_filter will call to update the property of the subplugin. This function will take accelerator list as the argument. This operation will update the backend to be used by the corresponding subplugin.
+ * List of operations to be supported are optional.
+ * Note: In these operations, the argument 'prop' will not contain the updated information, but will be updated after the corresponding operation is succeeded.
*
- * @param[in] hw backend accelerator hardware
- * @return 0 if supported. -errno if not supported.
+ * @param[in] prop read-only property values
+ * @param[in/out] private_data A subplugin may save its internal private data here. The subplugin is responsible for alloc/free of this pointer.
+ * @param[in] ops operation to be performed
+ * @param[in/out] data user sata for the supported handlers (can be NULL)
+ * @return 0 if OK. non-zero if error. -ENOENT if operation is not supported.
*/
+ };
+ };
} GstTensorFilterFramework;
/* extern functions for subplugin management, exist in tensor_filter.c */
}
/**
+ * @brief Free the data allocated for tensor transform
+ * @details default function for tensor filter framework if not provided by the
+ * framework. The data is in GPtrArray - first element is private data
+ * of framework and second element is the data to be freed.
+ */
+static void
+gst_tensor_filter_destroy_notify (void *data)
+{
+ GPtrArray *array = (GPtrArray *) data;
+ GstTensorFilter *self = (GstTensorFilter *) g_ptr_array_index (array, 0);
+ void *tensor_data = (void *) g_ptr_array_index (array, 1);
+ g_ptr_array_free (array, TRUE);
+
+ if (self->priv.fw->destroyNotify) {
+ self->priv.fw->destroyNotify (&self->priv.privateData, tensor_data);
+ } else {
+ g_free (tensor_data);
+ }
+}
+
+/**
* @brief non-ip transform. required vmethod of GstBaseTransform.
*/
static GstFlowReturn
/* 4. Update result and free map info. */
for (i = 0; i < prop->output_meta.num_tensors; i++) {
if (priv->fw->allocate_in_invoke) {
+ GPtrArray *data_array = g_ptr_array_new ();
+ g_ptr_array_add (data_array, (gpointer) self);
+ g_ptr_array_add (data_array, (gpointer) out_tensors[i].data);
/* filter-subplugin allocated new memory, update this */
out_mem[i] =
gst_memory_new_wrapped (0, out_tensors[i].data, out_tensors[i].size,
- 0, out_tensors[i].size, out_tensors[i].data,
- priv->fw->destroyNotify ? priv->fw->destroyNotify : g_free);
+ 0, out_tensors[i].size, (gpointer) data_array,
+ gst_tensor_filter_destroy_notify);
} else {
gst_memory_unmap (out_mem[i], &out_info[i]);
}
if (ptr->methods->allocate_invoke) {
NNS_support_custom.allocate_in_invoke = TRUE;
- NNS_support_custom.destroyNotify = ptr->methods->destroy_notify;
}
return 0;
}
*private_data = NULL;
}
+/**
+ * @brief The optional callback for GstTensorFilterFramework
+ */
+static void
+custom_destroyNotify (void **private_data, void *data)
+{
+ internal_data *ptr = *private_data;
+
+ if (ptr && ptr->methods->allocate_invoke && ptr->methods->destroy_notify) {
+ ptr->methods->destroy_notify (data);
+ } else {
+ g_free (data);
+ }
+}
+
static gchar filter_subplugin_custom[] = "custom";
static GstTensorFilterFramework NNS_support_custom = {
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.name = filter_subplugin_custom,
.allow_in_place = FALSE, /* custom cannot support in-place (output == input). */
.allocate_in_invoke = FALSE, /* GstTensorFilter allocates output buffers */
.setInputDimension = custom_setInputDim,
.open = custom_open,
.close = custom_close,
- .destroyNotify = NULL, /* default null. if allocate_in_invoke is true, this will be set from custom filter. */
+ .destroyNotify = custom_destroyNotify, /* default null. if allocate_in_invoke is true, this will be set from custom filter. */
};
/** @brief Initialize this object for tensor_filter subplugin runtime register */
static char name_str[] = "custom-easy";
static GstTensorFilterFramework NNS_support_custom_easy = {
+ .version = GST_TENSOR_FILTER_FRAMEWORK_V0,
.name = name_str,
.allow_in_place = FALSE, /* custom cannot support in-place. */
.allocate_in_invoke = FALSE, /* we allocate output buffers for you. */
projects / platform / upstream / nnstreamer.git / commitdiff