From 27e03758272d4ba86834083b62a19ba9360030fb Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Sat, 31 Aug 2019 00:04:24 +0900 Subject: [PATCH] [filter_single] Common functions between tensor_filter and tensor_filter_single Simplified common functions between tensor_filter and tensor_filter_single Added more implementations in tensor_filter_single Signed-off-by: Parichay Kapoor --- api/capi/meson.build | 2 +- api/capi/src/tensor_filter_single.c | 213 ++++++++++- gst/nnstreamer/tensor_filter/meson.build | 1 + gst/nnstreamer/tensor_filter/tensor_filter.c | 392 +------------------ .../tensor_filter/tensor_filter_common.c | 421 +++++++++++++++++++++ .../tensor_filter/tensor_filter_common.h | 107 ++++++ jni/nnstreamer.mk | 1 + 7 files changed, 743 insertions(+), 394 deletions(-) create mode 100644 gst/nnstreamer/tensor_filter/tensor_filter_common.c create mode 100644 gst/nnstreamer/tensor_filter/tensor_filter_common.h diff --git a/api/capi/meson.build b/api/capi/meson.build index b60b603..39ee6b3 100644 --- a/api/capi/meson.build +++ b/api/capi/meson.build @@ -24,7 +24,7 @@ nninc = include_directories('../../gst') # Create dependency for tensor_filter_single for single-shot capi tensor_filter_single_lib = shared_library ('tensor_filter_single_lib', join_paths(meson.current_source_dir(), 'src', 'tensor_filter_single.c'), - dependencies: [glib_dep, gobject_dep], + dependencies: [nnstreamer_dep, glib_dep, gobject_dep, gst_dep], include_directories: [inc, nninc], install: false, version: meson.project_version(), diff --git a/api/capi/src/tensor_filter_single.c b/api/capi/src/tensor_filter_single.c index f43afd6..9fa4abc 100644 --- a/api/capi/src/tensor_filter_single.c +++ b/api/capi/src/tensor_filter_single.c @@ -43,6 +43,10 @@ #include #include +#include +#include +#include + #include "tensor_filter_single.h" /** @@ -52,6 +56,19 @@ #define DBG (!self->silent) #endif +#define silent_debug_info(i,msg) do { \ + if (DBG) { \ + guint info_idx; \ + gchar *dim_str; \ + g_debug (msg " total %d", (i)->num_tensors); \ + for (info_idx = 0; info_idx < (i)->num_tensors; info_idx++) { \ + dim_str = gst_tensor_get_dimension_string ((i)->info[info_idx].dimension); \ + g_debug ("[%d] type=%d dim=%s", info_idx, (i)->info[info_idx].type, dim_str); \ + g_free (dim_str); \ + } \ + } \ +} while (0) + #define g_tensor_filter_single_parent_class parent_class G_DEFINE_TYPE (GTensorFilterSingle, g_tensor_filter_single, G_TYPE_OBJECT); @@ -65,6 +82,8 @@ static void g_tensor_filter_single_get_property (GObject * object, /* GTensorFilterSingle method implementations */ static gboolean g_tensor_filter_single_invoke (GTensorFilterSingle * self, GstTensorMemory * input, GstTensorMemory * output); +static gboolean g_tensor_filter_input_configured (GTensorFilterSingle * self); +static gboolean g_tensor_filter_output_configured (GTensorFilterSingle * self); /* Private functions */ static gboolean g_tensor_filter_single_start (GTensorFilterSingle * self); @@ -84,9 +103,13 @@ g_tensor_filter_single_class_init (GTensorFilterSingleClass * klass) gobject_class->get_property = g_tensor_filter_single_get_property; gobject_class->finalize = g_tensor_filter_single_finalize; - klass->invoke = g_tensor_filter_single_invoke; + gst_tensor_filter_install_properties (gobject_class); - /** TODO: share same setup properties as tensor_filter */ + klass->invoke = g_tensor_filter_single_invoke; + klass->start = g_tensor_filter_single_start; + klass->stop = g_tensor_filter_single_stop; + klass->input_configured = g_tensor_filter_input_configured; + klass->output_configured = g_tensor_filter_output_configured; } /** @@ -95,7 +118,27 @@ g_tensor_filter_single_class_init (GTensorFilterSingleClass * klass) static void g_tensor_filter_single_init (GTensorFilterSingle * self) { - /** TODO: fill this */ + GstTensorFilterProperties *prop; + + prop = &self->prop; + + /* init NNFW properties */ + prop->fwname = NULL; + prop->fw_opened = FALSE; + prop->input_configured = FALSE; + prop->output_configured = FALSE; + prop->model_file = NULL; + prop->custom_properties = NULL; + gst_tensors_info_init (&prop->input_meta); + gst_tensors_info_init (&prop->output_meta); + + /* init internal properties */ + self->fw = NULL; + self->privateData = NULL; + self->silent = TRUE; + self->started = FALSE; + gst_tensors_config_init (&self->in_config); + gst_tensors_config_init (&self->out_config); } /** @@ -106,7 +149,7 @@ g_tensor_filter_single_finalize (GObject * object) { gboolean status; GTensorFilterSingle *self; - /** TODO: fill this */ + GstTensorFilterProperties *prop; self = G_TENSOR_FILTER_SINGLE (object); @@ -115,6 +158,20 @@ g_tensor_filter_single_finalize (GObject * object) status = g_tensor_filter_single_stop (self); g_debug ("Tensor filter single stop status: %d", status); } + + prop = &self->prop; + + g_free_const (prop->fwname); + g_free_const (prop->model_file); + g_free_const (prop->custom_properties); + + gst_tensors_info_free (&prop->input_meta); + gst_tensors_info_free (&prop->output_meta); + + gst_tensors_info_free (&self->in_config.info); + gst_tensors_info_free (&self->out_config.info); + + G_OBJECT_CLASS (parent_class)->finalize (object); } /** @@ -134,7 +191,124 @@ static void g_tensor_filter_single_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - /** TODO: share this with tensor_filter*/ + GTensorFilterSingle *self; + GstTensorFilterProperties *prop; + + self = G_TENSOR_FILTER_SINGLE (object); + prop = &self->prop; + + g_debug ("Getting property for prop %d.\n", prop_id); + + switch (prop_id) { + case PROP_SILENT: + g_value_set_boolean (value, self->silent); + break; + default: + if (!gst_tensor_filter_common_get_property (prop, prop_id, value, pspec)) + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +/** + * @brief Determine if input is configured + * (both input and output tensor) + */ +static gboolean +g_tensor_filter_input_configured (GTensorFilterSingle * self) +{ + if (self->prop.input_configured) + return TRUE; + else + return FALSE; +} + +/** + * @brief Determine if output is configured + * (both input and output tensor) + */ +static gboolean +g_tensor_filter_output_configured (GTensorFilterSingle * self) +{ + if (self->prop.output_configured) + return TRUE; + else + return FALSE; +} + +/** + * @brief Load tensor info from NN model. + * (both input and output tensor) + */ +static void +g_tensor_filter_load_tensor_info (GTensorFilterSingle * self) +{ + GstTensorFilterProperties *prop; + GstTensorsInfo in_info, out_info; + int res = -1; + + prop = &self->prop; + + gst_tensors_info_init (&in_info); + gst_tensors_info_init (&out_info); + + /* supposed fixed in-tensor info if getInputDimension is defined. */ + if (!prop->input_configured) { + res = -1; + if (self->prop.fw_opened && self->fw && self->fw->getInputDimension) { + res = self->fw->getInputDimension + (&self->prop, &self->privateData, &in_info); + } + + if (res == 0) { + g_assert (in_info.num_tensors > 0); + + /** if set-property called and already has info, verify it! */ + if (prop->input_meta.num_tensors > 0) { + if (!gst_tensors_info_is_equal (&in_info, &prop->input_meta)) { + g_critical ("The input tensor is not compatible."); + gst_tensor_filter_compare_tensors (&in_info, &prop->input_meta); + goto done; + } + } else { + gst_tensors_info_copy (&prop->input_meta, &in_info); + } + + prop->input_configured = TRUE; + silent_debug_info (&in_info, "input tensor"); + } + } + + /* supposed fixed out-tensor info if getOutputDimension is defined. */ + if (!prop->output_configured) { + res = -1; + if (self->prop.fw_opened && self->fw && self->fw->getOutputDimension) { + res = self->fw->getOutputDimension + (&self->prop, &self->privateData, &out_info); + } + + if (res == 0) { + g_assert (out_info.num_tensors > 0); + + /** if set-property called and already has info, verify it! */ + if (prop->output_meta.num_tensors > 0) { + if (!gst_tensors_info_is_equal (&out_info, &prop->output_meta)) { + g_critical ("The output tensor is not compatible."); + gst_tensor_filter_compare_tensors (&out_info, &prop->output_meta); + goto done; + } + } else { + gst_tensors_info_copy (&prop->output_meta, &out_info); + } + + prop->output_configured = TRUE; + silent_debug_info (&out_info, "output tensor"); + } + } + +done: + gst_tensors_info_free (&in_info); + gst_tensors_info_free (&out_info); } /** @@ -145,8 +319,24 @@ g_tensor_filter_single_get_property (GObject * object, guint prop_id, static gboolean g_tensor_filter_single_start (GTensorFilterSingle * self) { - /** TODO: fill this */ /** open framework, load model */ + if (self->fw == NULL) + return FALSE; + + if (self->prop.fw_opened == FALSE && self->fw) { + if (self->fw->open != NULL) { + if (self->fw->open (&self->prop, &self->privateData) == 0) + self->prop.fw_opened = TRUE; + } else { + self->prop.fw_opened = TRUE; + } + } + + if (self->prop.fw_opened == FALSE) + return FALSE; + + g_tensor_filter_load_tensor_info (self); + return TRUE; } @@ -158,8 +348,17 @@ g_tensor_filter_single_start (GTensorFilterSingle * self) static gboolean g_tensor_filter_single_stop (GTensorFilterSingle * self) { - /** TODO: fill this */ /** close framework, unload model */ + if (self->prop.fw_opened) { + if (self->fw && self->fw->close) { + self->fw->close (&self->prop, &self->privateData); + } + self->prop.fw_opened = FALSE; + g_free_const (self->prop.fwname); + self->prop.fwname = NULL; + self->fw = NULL; + self->privateData = NULL; + } return TRUE; } diff --git a/gst/nnstreamer/tensor_filter/meson.build b/gst/nnstreamer/tensor_filter/meson.build index 03ee8d7..65dc34f 100644 --- a/gst/nnstreamer/tensor_filter/meson.build +++ b/gst/nnstreamer/tensor_filter/meson.build @@ -1,5 +1,6 @@ tensor_filter_sources = [ 'tensor_filter.c', + 'tensor_filter_common.c', 'tensor_filter_custom.c' ] diff --git a/gst/nnstreamer/tensor_filter/tensor_filter.c b/gst/nnstreamer/tensor_filter/tensor_filter.c index cabddaa..74b582a 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter.c @@ -60,6 +60,7 @@ #include #include "tensor_filter.h" +#include "tensor_filter_common.h" /** * @brief Macro for debug mode. @@ -107,92 +108,12 @@ } \ } while (0) -#define g_free_const(x) g_free((void*)(long)(x)) - #define REGEX_NNAPI_OPTION "(^(true|false)$|^(true|false)([:]?(cpu|gpu|npu)))" -/** - * @brief Validate filter sub-plugin's data. - */ -static gboolean -nnstreamer_filter_validate (const GstTensorFilterFramework * tfsp) -{ - if (!tfsp || !tfsp->name) { - /* invalid fw name */ - return FALSE; - } - - if (!tfsp->invoke_NN) { - /* no invoke function */ - return FALSE; - } - - if (!(tfsp->getInputDimension && tfsp->getOutputDimension) && - !tfsp->setInputDimension) { - /* no method to get tensor info */ - return FALSE; - } - - return TRUE; -} - -/** - * @brief Filter's sub-plugin should call this function to register itself. - * @param[in] tfsp Tensor-Filter Sub-Plugin to be registered. - * @return TRUE if registered. FALSE is failed or duplicated. - */ -int -nnstreamer_filter_probe (GstTensorFilterFramework * tfsp) -{ - g_return_val_if_fail (nnstreamer_filter_validate (tfsp), FALSE); - return register_subplugin (NNS_SUBPLUGIN_FILTER, tfsp->name, tfsp); -} - -/** - * @brief Filter's sub-plugin may call this to unregister itself. - * @param[in] name The name of filter sub-plugin. - */ -void -nnstreamer_filter_exit (const char *name) -{ - unregister_subplugin (NNS_SUBPLUGIN_FILTER, name); -} - -/** - * @brief Find filter sub-plugin with the name. - * @param[in] name The name of filter sub-plugin. - * @return NULL if not found or the sub-plugin object has an error. - */ -const GstTensorFilterFramework * -nnstreamer_filter_find (const char *name) -{ - return get_subplugin (NNS_SUBPLUGIN_FILTER, name); -} - GST_DEBUG_CATEGORY_STATIC (gst_tensor_filter_debug); #define GST_CAT_DEFAULT gst_tensor_filter_debug /** - * @brief GstTensorFilter properties. - */ -enum -{ - PROP_0, - PROP_SILENT, - PROP_FRAMEWORK, - PROP_MODEL, - PROP_INPUT, - PROP_INPUTTYPE, - PROP_INPUTNAME, - PROP_OUTPUT, - PROP_OUTPUTTYPE, - PROP_OUTPUTNAME, - PROP_CUSTOM, - PROP_SUBPLUGINS, - PROP_NNAPI -}; - -/** * @brief Default caps string for both sink and source pad. */ #define CAPS_STRING GST_TENSOR_CAP_DEFAULT "; " GST_TENSORS_CAP_DEFAULT @@ -238,8 +159,6 @@ static gboolean gst_tensor_filter_transform_size (GstBaseTransform * trans, static gboolean gst_tensor_filter_start (GstBaseTransform * trans); static gboolean gst_tensor_filter_stop (GstBaseTransform * trans); -static void gst_tensor_filter_compare_tensors (GstTensorsInfo * info1, - GstTensorsInfo * info2); /** * @brief Open nn framework. @@ -302,52 +221,8 @@ gst_tensor_filter_class_init (GstTensorFilterClass * klass) gobject_class->get_property = gst_tensor_filter_get_property; gobject_class->finalize = gst_tensor_filter_finalize; - g_object_class_install_property (gobject_class, PROP_SILENT, - g_param_spec_boolean ("silent", "Silent", "Produce verbose output", - FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_FRAMEWORK, - g_param_spec_string ("framework", "Framework", - "Neural network framework", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_MODEL, - g_param_spec_string ("model", "Model filepath", - "File path to the model file. Separated with \ - ',' in case of multiple model files(like caffe2)", "", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_INPUT, - g_param_spec_string ("input", "Input dimension", - "Input tensor dimension from inner array, up to 4 dimensions ?", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_INPUTNAME, - g_param_spec_string ("inputname", "Name of Input Tensor", - "The Name of Input Tensor", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_INPUTTYPE, - g_param_spec_string ("inputtype", "Input tensor element type", - "Type of each element of the input tensor ?", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_OUTPUTNAME, - g_param_spec_string ("outputname", "Name of Output Tensor", - "The Name of Output Tensor", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_OUTPUT, - g_param_spec_string ("output", "Output dimension", - "Output tensor dimension from inner array, up to 4 dimensions ?", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_OUTPUTTYPE, - g_param_spec_string ("outputtype", "Output tensor element type", - "Type of each element of the output tensor ?", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_CUSTOM, - g_param_spec_string ("custom", "Custom properties for subplugins", - "Custom properties for subplugins ?", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_SUBPLUGINS, - g_param_spec_string ("sub-plugins", "Sub-plugins", - "Registrable sub-plugins list", "", - G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_NNAPI, - g_param_spec_string ("nnapi", "NNAPI", - "Enable Neural Newtorks API ?", "", G_PARAM_READWRITE)); + gst_tensor_filter_install_properties (gobject_class); + gst_element_class_set_details_simple (gstelement_class, "Tensor_Filter", "Converter/Filter/Tensor", @@ -441,84 +316,6 @@ gst_tensor_filter_finalize (GObject * object) } /** - * @brief Parse the string of model - * @param info tensors info structure - * @param model_file the prediction model path - * @param model_file_sub the initialize model path - * @return number of parsed model path - * @todo Create a struct list to save multiple model files with key, value pair - */ -static guint -gst_tensors_parse_modelpaths_string (GstTensorFilterProperties * prop, - const gchar * model_files) -{ - gchar **models; - gchar **model_0; - gchar **model_1; - guint num_models = 0; - guint num_model_0 = 0; - guint num_model_1 = 0; - - g_return_val_if_fail (prop != NULL, 0); - - if (model_files) { - models = g_strsplit_set (model_files, ",", -1); - num_models = g_strv_length (models); - - if (num_models == 1) { - prop->model_file = g_strdup (models[0]); - } else if (num_models == 2) { - model_0 = g_strsplit_set (models[0], "=", -1); - model_1 = g_strsplit_set (models[1], "=", -1); - - num_model_0 = g_strv_length (model_0); - num_model_1 = g_strv_length (model_1); - - if (num_model_0 == 1 && num_model_1 == 1) { - prop->model_file_sub = g_strdup (model_0[0]); - prop->model_file = g_strdup (model_1[0]); - } else if (g_ascii_strncasecmp (model_0[0], "init", 4) == 0 || - g_ascii_strncasecmp (model_0[0], "Init", 4) == 0) { - prop->model_file_sub = g_strdup (model_0[1]); - - if (num_model_1 == 2) - prop->model_file = g_strdup (model_1[1]); - else - prop->model_file = g_strdup (model_1[0]); - } else if (g_ascii_strncasecmp (model_0[0], "pred", 4) == 0 || - g_ascii_strncasecmp (model_0[0], "Pred", 4) == 0) { - prop->model_file = g_strdup (model_0[1]); - - if (num_model_1 == 2) - prop->model_file_sub = g_strdup (model_1[1]); - else - prop->model_file_sub = g_strdup (model_1[0]); - } else if (g_ascii_strncasecmp (model_1[0], "init", 4) == 0 || - g_ascii_strncasecmp (model_1[0], "Init", 4) == 0) { - prop->model_file_sub = g_strdup (model_1[1]); - - if (num_model_0 == 2) - prop->model_file = g_strdup (model_0[1]); - else - prop->model_file = g_strdup (model_0[0]); - } else if (g_ascii_strncasecmp (model_1[0], "pred", 4) == 0 || - g_ascii_strncasecmp (model_1[0], "Pred", 4) == 0) { - prop->model_file = g_strdup (model_1[1]); - - if (num_model_0 == 2) - prop->model_file_sub = g_strdup (model_0[1]); - else - prop->model_file_sub = g_strdup (model_0[0]); - } - g_strfreev (model_0); - g_strfreev (model_1); - } - g_strfreev (models); - } - return num_models; -} - -/** * @brief Calculate output buffer size. * @param self "this" pointer * @param index index of output tensors (if index < 0, the size of all output tensors will be returned.) @@ -602,7 +399,7 @@ gst_tensor_filter_set_property (GObject * object, guint prop_id, /* Once configures, it cannot be changed in runtime */ /** @todo by using `gst_element_get_state()`, reject configurations in RUNNING or other states */ g_assert (model_files); - model_num = gst_tensors_parse_modelpaths_string (prop, model_files); + model_num = gst_tensor_filter_parse_modelpaths_string (prop, model_files); if (model_num == 1) { silent_debug ("Model = %s\n", prop->model_file); if (!g_file_test (prop->model_file, G_FILE_TEST_IS_REGULAR)) @@ -779,125 +576,9 @@ gst_tensor_filter_get_property (GObject * object, guint prop_id, case PROP_SILENT: g_value_set_boolean (value, self->silent); break; - case PROP_FRAMEWORK: - g_value_set_string (value, prop->fwname); - break; - case PROP_MODEL: - g_value_set_string (value, prop->model_file); - break; - case PROP_INPUT: - if (prop->input_meta.num_tensors > 0) { - gchar *dim_str; - - dim_str = gst_tensors_info_get_dimensions_string (&prop->input_meta); - - g_value_set_string (value, dim_str); - g_free (dim_str); - } else { - g_value_set_string (value, ""); - } - break; - case PROP_OUTPUT: - if (prop->output_meta.num_tensors > 0) { - gchar *dim_str; - - dim_str = gst_tensors_info_get_dimensions_string (&prop->output_meta); - - g_value_set_string (value, dim_str); - g_free (dim_str); - } else { - g_value_set_string (value, ""); - } - break; - case PROP_INPUTTYPE: - if (prop->input_meta.num_tensors > 0) { - gchar *type_str; - - type_str = gst_tensors_info_get_types_string (&prop->input_meta); - - g_value_set_string (value, type_str); - g_free (type_str); - } else { - g_value_set_string (value, ""); - } - break; - case PROP_OUTPUTTYPE: - if (prop->output_meta.num_tensors > 0) { - gchar *type_str; - - type_str = gst_tensors_info_get_types_string (&prop->output_meta); - - g_value_set_string (value, type_str); - g_free (type_str); - } else { - g_value_set_string (value, ""); - } - break; - case PROP_INPUTNAME: - if (prop->input_meta.num_tensors > 0) { - gchar *name_str; - - name_str = gst_tensors_info_get_names_string (&prop->input_meta); - - g_value_set_string (value, name_str); - g_free (name_str); - } else { - g_value_set_string (value, ""); - } - break; - case PROP_OUTPUTNAME: - if (prop->output_meta.num_tensors > 0) { - gchar *name_str; - - name_str = gst_tensors_info_get_names_string (&prop->output_meta); - - g_value_set_string (value, name_str); - g_free (name_str); - } else { - g_value_set_string (value, ""); - } - break; - case PROP_CUSTOM: - g_value_set_string (value, prop->custom_properties); - break; - case PROP_SUBPLUGINS: - { - GString *subplugins; - subplugin_info_s sinfo; - guint i, total; - - subplugins = g_string_new (NULL); - - /* add custom */ - g_string_append (subplugins, "custom"); - - total = nnsconf_get_subplugin_info (NNSCONF_PATH_FILTERS, &sinfo); - - if (total > 0) { - const gchar *prefix_str; - gsize prefix, extension, len; - - prefix_str = nnsconf_get_subplugin_name_prefix (NNSCONF_PATH_FILTERS); - prefix = strlen (prefix_str); - extension = strlen (NNSTREAMER_SO_FILE_EXTENSION); - - for (i = 0; i < total; ++i) { - g_string_append (subplugins, ","); - - /* remove file extension */ - len = strlen (sinfo.names[i]) - prefix - extension; - g_string_append_len (subplugins, sinfo.names[i] + prefix, len); - } - } - - g_value_take_string (value, g_string_free (subplugins, FALSE)); - break; - } - case PROP_NNAPI: - g_value_set_string (value, prop->nnapi); - break; default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + if (!gst_tensor_filter_common_get_property (prop, prop_id, value, pspec)) + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; } } @@ -1093,67 +774,6 @@ done: } /** - * @brief Printout the comparison results of two tensors. - * @param[in] info1 The tensors to be shown on the left hand side - * @param[in] info2 The tensors to be shown on the right hand side - * @todo If this is going to be used by other elements, move this to nnstreamer/tensor_common. - */ -static void -gst_tensor_filter_compare_tensors (GstTensorsInfo * info1, - GstTensorsInfo * info2) -{ - gchar *result = NULL; - gchar *line, *tmp, *left, *right; - guint i; - - for (i = 0; i < NNS_TENSOR_SIZE_LIMIT; i++) { - if (info1->num_tensors <= i && info2->num_tensors <= i) - break; - - if (info1->num_tensors > i) { - tmp = gst_tensor_get_dimension_string (info1->info[i].dimension); - left = g_strdup_printf ("%s [%s]", - gst_tensor_get_type_string (info1->info[i].type), tmp); - g_free (tmp); - } else { - left = g_strdup ("None"); - } - - if (info2->num_tensors > i) { - tmp = gst_tensor_get_dimension_string (info2->info[i].dimension); - right = g_strdup_printf ("%s [%s]", - gst_tensor_get_type_string (info2->info[i].type), tmp); - g_free (tmp); - } else { - right = g_strdup ("None"); - } - - line = - g_strdup_printf ("%2d : %s | %s %s\n", i, left, right, - g_str_equal (left, right) ? "" : "FAILED"); - - g_free (left); - g_free (right); - - if (result) { - tmp = g_strdup_printf ("%s%s", result, line); - g_free (result); - g_free (line); - - result = tmp; - } else { - result = line; - } - } - - if (result) { - /* print warning message */ - g_warning ("Tensor info :\n%s", result); - g_free (result); - } -} - -/** * @brief Configure input and output tensor info from incaps. * @param self "this" pointer * @param incaps received caps for sink pad diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_common.c b/gst/nnstreamer/tensor_filter/tensor_filter_common.c new file mode 100644 index 0000000..b628161 --- /dev/null +++ b/gst/nnstreamer/tensor_filter/tensor_filter_common.c @@ -0,0 +1,421 @@ +/** + * Copyright (C) 2019 Parichay Kapoor + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + */ +/** + * @file tensor_filter_common.c + * @date 28 Aug 2019 + * @brief Common functions for various tensor_filters + * @see http://github.com/nnsuite/nnstreamer + * @author Parichay Kapoor + * @author MyungJoo Ham + * @bug No known bugs except for NYI items + * + */ + +#include + +#include + +#include "tensor_filter_common.h" + +/** + * @brief Validate filter sub-plugin's data. + */ +gboolean +nnstreamer_filter_validate (const GstTensorFilterFramework * tfsp) +{ + if (!tfsp || !tfsp->name) { + /* invalid fw name */ + return FALSE; + } + + if (!tfsp->invoke_NN) { + /* no invoke function */ + return FALSE; + } + + if (!(tfsp->getInputDimension && tfsp->getOutputDimension) && + !tfsp->setInputDimension) { + /* no method to get tensor info */ + return FALSE; + } + + return TRUE; +} + +/** + * @brief Filter's sub-plugin should call this function to register itself. + * @param[in] tfsp Tensor-Filter Sub-Plugin to be registered. + * @return TRUE if registered. FALSE is failed or duplicated. + */ +int +nnstreamer_filter_probe (GstTensorFilterFramework * tfsp) +{ + g_return_val_if_fail (nnstreamer_filter_validate (tfsp), FALSE); + return register_subplugin (NNS_SUBPLUGIN_FILTER, tfsp->name, tfsp); +} + +/** + * @brief Filter's sub-plugin may call this to unregister itself. + * @param[in] name The name of filter sub-plugin. + */ +void +nnstreamer_filter_exit (const char *name) +{ + unregister_subplugin (NNS_SUBPLUGIN_FILTER, name); +} + +/** + * @brief Find filter sub-plugin with the name. + * @param[in] name The name of filter sub-plugin. + * @return NULL if not found or the sub-plugin object has an error. + */ +const GstTensorFilterFramework * +nnstreamer_filter_find (const char *name) +{ + return get_subplugin (NNS_SUBPLUGIN_FILTER, name); +} + + +/** + * @brief Parse the string of model + * @param[out] prop Struct containing the properties of the object + * @param[in] model_files the prediction model paths + * @return number of parsed model path + */ +guint +gst_tensor_filter_parse_modelpaths_string (GstTensorFilterProperties * prop, + const gchar * model_files) +{ + gchar **models; + gchar **model_0; + gchar **model_1; + guint num_models = 0; + guint num_model_0 = 0; + guint num_model_1 = 0; + + g_return_val_if_fail (prop != NULL, 0); + + if (model_files) { + models = g_strsplit_set (model_files, ",", -1); + num_models = g_strv_length (models); + + if (num_models == 1) { + prop->model_file = g_strdup (models[0]); + } else if (num_models == 2) { + model_0 = g_strsplit_set (models[0], "=", -1); + model_1 = g_strsplit_set (models[1], "=", -1); + + num_model_0 = g_strv_length (model_0); + num_model_1 = g_strv_length (model_1); + + if (num_model_0 == 1 && num_model_1 == 1) { + prop->model_file_sub = g_strdup (model_0[0]); + prop->model_file = g_strdup (model_1[0]); + } else if (g_ascii_strncasecmp (model_0[0], "init", 4) == 0 || + g_ascii_strncasecmp (model_0[0], "Init", 4) == 0) { + prop->model_file_sub = g_strdup (model_0[1]); + + if (num_model_1 == 2) + prop->model_file = g_strdup (model_1[1]); + else + prop->model_file = g_strdup (model_1[0]); + } else if (g_ascii_strncasecmp (model_0[0], "pred", 4) == 0 || + g_ascii_strncasecmp (model_0[0], "Pred", 4) == 0) { + prop->model_file = g_strdup (model_0[1]); + + if (num_model_1 == 2) + prop->model_file_sub = g_strdup (model_1[1]); + else + prop->model_file_sub = g_strdup (model_1[0]); + } else if (g_ascii_strncasecmp (model_1[0], "init", 4) == 0 || + g_ascii_strncasecmp (model_1[0], "Init", 4) == 0) { + prop->model_file_sub = g_strdup (model_1[1]); + + if (num_model_0 == 2) + prop->model_file = g_strdup (model_0[1]); + else + prop->model_file = g_strdup (model_0[0]); + } else if (g_ascii_strncasecmp (model_1[0], "pred", 4) == 0 || + g_ascii_strncasecmp (model_1[0], "Pred", 4) == 0) { + prop->model_file = g_strdup (model_1[1]); + + if (num_model_0 == 2) + prop->model_file_sub = g_strdup (model_0[1]); + else + prop->model_file_sub = g_strdup (model_0[0]); + } + g_strfreev (model_0); + g_strfreev (model_1); + } + g_strfreev (models); + } + return num_models; +} + + +/** + * @brief Printout the comparison results of two tensors. + * @param[in] info1 The tensors to be shown on the left hand side + * @param[in] info2 The tensors to be shown on the right hand side + */ +void +gst_tensor_filter_compare_tensors (GstTensorsInfo * info1, + GstTensorsInfo * info2) +{ + gchar *result = NULL; + gchar *line, *tmp, *left, *right; + guint i; + + for (i = 0; i < NNS_TENSOR_SIZE_LIMIT; i++) { + if (info1->num_tensors <= i && info2->num_tensors <= i) + break; + + if (info1->num_tensors > i) { + tmp = gst_tensor_get_dimension_string (info1->info[i].dimension); + left = g_strdup_printf ("%s [%s]", + gst_tensor_get_type_string (info1->info[i].type), tmp); + g_free (tmp); + } else { + left = g_strdup ("None"); + } + + if (info2->num_tensors > i) { + tmp = gst_tensor_get_dimension_string (info2->info[i].dimension); + right = g_strdup_printf ("%s [%s]", + gst_tensor_get_type_string (info2->info[i].type), tmp); + g_free (tmp); + } else { + right = g_strdup ("None"); + } + + line = + g_strdup_printf ("%2d : %s | %s %s\n", i, left, right, + g_str_equal (left, right) ? "" : "FAILED"); + + g_free (left); + g_free (right); + + if (result) { + tmp = g_strdup_printf ("%s%s", result, line); + g_free (result); + g_free (line); + + result = tmp; + } else { + result = line; + } + } + + if (result) { + /* print warning message */ + g_warning ("Tensor info :\n%s", result); + g_free (result); + } +} + +/** + * @brief Installs all the properties for tensor_filter + * @param[in] gobject_class Glib object class whose properties will be set + */ +void +gst_tensor_filter_install_properties (GObjectClass * gobject_class) +{ + g_object_class_install_property (gobject_class, PROP_SILENT, + g_param_spec_boolean ("silent", "Silent", "Produce verbose output", + FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_FRAMEWORK, + g_param_spec_string ("framework", "Framework", + "Neural network framework", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_MODEL, + g_param_spec_string ("model", "Model filepath", + "File path to the model file. Separated with \ + ',' in case of multiple model files(like caffe2)", "", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT, + g_param_spec_string ("input", "Input dimension", + "Input tensor dimension from inner array, up to 4 dimensions ?", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUTNAME, + g_param_spec_string ("inputname", "Name of Input Tensor", + "The Name of Input Tensor", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUTTYPE, + g_param_spec_string ("inputtype", "Input tensor element type", + "Type of each element of the input tensor ?", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_OUTPUTNAME, + g_param_spec_string ("outputname", "Name of Output Tensor", + "The Name of Output Tensor", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_OUTPUT, + g_param_spec_string ("output", "Output dimension", + "Output tensor dimension from inner array, up to 4 dimensions ?", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_OUTPUTTYPE, + g_param_spec_string ("outputtype", "Output tensor element type", + "Type of each element of the output tensor ?", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_CUSTOM, + g_param_spec_string ("custom", "Custom properties for subplugins", + "Custom properties for subplugins ?", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_SUBPLUGINS, + g_param_spec_string ("sub-plugins", "Sub-plugins", + "Registrable sub-plugins list", "", + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_NNAPI, + g_param_spec_string ("nnapi", "NNAPI", + "Enable Neural Newtorks API ?", "", G_PARAM_READWRITE)); +} + +/** + * @brief Get the properties for tensor_filter + * @param[in] prop Struct containing the properties of the object + * @param[in] prop_id Id for the property + * @param[in] value Container to return the asked property + * @param[in] pspec Metadata to specify the parameter + * @return TRUE if prop_id is value, else FALSE + */ +gboolean +gst_tensor_filter_common_get_property (GstTensorFilterProperties * prop, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + gboolean ret = TRUE; + + switch (prop_id) { + case PROP_FRAMEWORK: + g_value_set_string (value, prop->fwname); + break; + case PROP_MODEL: + g_value_set_string (value, prop->model_file); + break; + case PROP_INPUT: + if (prop->input_meta.num_tensors > 0) { + gchar *dim_str; + + dim_str = gst_tensors_info_get_dimensions_string (&prop->input_meta); + + g_value_set_string (value, dim_str); + g_free (dim_str); + } else { + g_value_set_string (value, ""); + } + break; + case PROP_OUTPUT: + if (prop->output_meta.num_tensors > 0) { + gchar *dim_str; + + dim_str = gst_tensors_info_get_dimensions_string (&prop->output_meta); + + g_value_set_string (value, dim_str); + g_free (dim_str); + } else { + g_value_set_string (value, ""); + } + break; + case PROP_INPUTTYPE: + if (prop->input_meta.num_tensors > 0) { + gchar *type_str; + + type_str = gst_tensors_info_get_types_string (&prop->input_meta); + + g_value_set_string (value, type_str); + g_free (type_str); + } else { + g_value_set_string (value, ""); + } + break; + case PROP_OUTPUTTYPE: + if (prop->output_meta.num_tensors > 0) { + gchar *type_str; + + type_str = gst_tensors_info_get_types_string (&prop->output_meta); + + g_value_set_string (value, type_str); + g_free (type_str); + } else { + g_value_set_string (value, ""); + } + break; + case PROP_INPUTNAME: + if (prop->input_meta.num_tensors > 0) { + gchar *name_str; + + name_str = gst_tensors_info_get_names_string (&prop->input_meta); + + g_value_set_string (value, name_str); + g_free (name_str); + } else { + g_value_set_string (value, ""); + } + break; + case PROP_OUTPUTNAME: + if (prop->output_meta.num_tensors > 0) { + gchar *name_str; + + name_str = gst_tensors_info_get_names_string (&prop->output_meta); + + g_value_set_string (value, name_str); + g_free (name_str); + } else { + g_value_set_string (value, ""); + } + break; + case PROP_CUSTOM: + g_value_set_string (value, prop->custom_properties); + break; + case PROP_SUBPLUGINS: + { + GString *subplugins; + subplugin_info_s sinfo; + guint i, total; + + subplugins = g_string_new (NULL); + + /* add custom */ + g_string_append (subplugins, "custom"); + + total = nnsconf_get_subplugin_info (NNSCONF_PATH_FILTERS, &sinfo); + + if (total > 0) { + const gchar *prefix_str; + gsize prefix, extension, len; + + prefix_str = nnsconf_get_subplugin_name_prefix (NNSCONF_PATH_FILTERS); + prefix = strlen (prefix_str); + extension = strlen (NNSTREAMER_SO_FILE_EXTENSION); + + for (i = 0; i < total; ++i) { + g_string_append (subplugins, ","); + + /* remove file extension */ + len = strlen (sinfo.names[i]) - prefix - extension; + g_string_append_len (subplugins, sinfo.names[i] + prefix, len); + } + } + + g_value_take_string (value, g_string_free (subplugins, FALSE)); + break; + } + case PROP_NNAPI: + g_value_set_string (value, prop->nnapi); + break; + default: + ret = FALSE; + break; + } + + return ret; +} diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_common.h b/gst/nnstreamer/tensor_filter/tensor_filter_common.h new file mode 100644 index 0000000..5b86a35 --- /dev/null +++ b/gst/nnstreamer/tensor_filter/tensor_filter_common.h @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2019 Parichay kapoor + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + */ +/** + * @file tensor_filter_common.h + * @date 28 Aug 2019 + * @brief Common functions for various tensor_filters + * @see http://github.com/nnsuite/nnstreamer + * @author Parichay Kapoor + * @author MyungJoo Ham + * @bug No known bugs except for NYI items + */ + +#ifndef __G_TENSOR_FILTER_COMMON_H__ +#define __G_TENSOR_FILTER_COMMON_H__ + +#include + +#include +#include +#include + +/** + * @brief Free memory + */ +#define g_free_const(x) g_free((void*)(long)(x)) + +/** + * @brief GstTensorFilter properties. + */ +enum +{ + PROP_0, + PROP_SILENT, + PROP_FRAMEWORK, + PROP_MODEL, + PROP_INPUT, + PROP_INPUTTYPE, + PROP_INPUTNAME, + PROP_OUTPUT, + PROP_OUTPUTTYPE, + PROP_OUTPUTNAME, + PROP_CUSTOM, + PROP_SUBPLUGINS, + PROP_NNAPI +}; + +/** + * @brief Validate filter sub-plugin's data. + */ +extern gboolean +nnstreamer_filter_validate (const GstTensorFilterFramework * tfsp); + +/** + * @brief Parse the string of model + * @param[out] prop Struct containing the properties of the object + * @param[in] model_files the prediction model paths + * @return number of parsed model path + * @todo Create a struct list to save multiple model files with key, value pair + */ +extern guint +gst_tensor_filter_parse_modelpaths_string (GstTensorFilterProperties * prop, + const gchar * model_files); + +/** + * @brief Printout the comparison results of two tensors. + * @param[in] info1 The tensors to be shown on the left hand side + * @param[in] info2 The tensors to be shown on the right hand side + * @todo If this is going to be used by other elements, move this to nnstreamer/tensor_common. + */ +extern void +gst_tensor_filter_compare_tensors (GstTensorsInfo * info1, + GstTensorsInfo * info2); + + +/** + * @brief Installs all the properties for tensor_filter + * @param[in] gobject_class Glib object class whose properties will be set + */ +extern void +gst_tensor_filter_install_properties (GObjectClass * gobject_class); + + +/** + * @brief Get the properties for tensor_filter + * @param[in] prop Struct containing the properties of the object + * @param[in] prop_id Id for the property + * @param[in] value Container to return the asked property + * @param[in] pspec Metadata to specify the parameter + * @return TRUE if prop_id is value, else FALSE + */ +extern gboolean +gst_tensor_filter_common_get_property (GstTensorFilterProperties *prop, + guint prop_id, GValue *value, GParamSpec *pspec); + +#endif /* __G_TENSOR_FILTER_COMMON_H__ */ diff --git a/jni/nnstreamer.mk b/jni/nnstreamer.mk index 301873b..21ad770 100644 --- a/jni/nnstreamer.mk +++ b/jni/nnstreamer.mk @@ -29,6 +29,7 @@ NNSTREAMER_PLUGINS_SRCS := \ $(NNSTREAMER_GST_HOME)/tensor_decoder/tensordec.c \ $(NNSTREAMER_GST_HOME)/tensor_demux/gsttensordemux.c \ $(NNSTREAMER_GST_HOME)/tensor_filter/tensor_filter.c \ + $(NNSTREAMER_GST_HOME)/tensor_filter/tensor_filter_common.c \ $(NNSTREAMER_GST_HOME)/tensor_filter/tensor_filter_custom.c \ $(NNSTREAMER_GST_HOME)/tensor_merge/gsttensormerge.c \ $(NNSTREAMER_GST_HOME)/tensor_mux/gsttensormux.c \ -- 2.7.4