Plug-and-play sub-plugins for Tensor_Filter

Allow tensor_filter sub-plugins to be independently built
and attached/detached in run-time with subplugin APIs.

This resolves one of "future work" in ATC 2019 submitted paper already :)

Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>

diff --git a/gst/nnstreamer/tensor_decoder/tensordec-plugins.c b/gst/nnstreamer/tensor_decoder/tensordec-plugins.c
index fe1d3bf..1886fe4 100644 (file)
--- a/gst/nnstreamer/tensor_decoder/tensordec-plugins.c
+++ b/gst/nnstreamer/tensor_decoder/tensordec-plugins.c
@@ -24,6 +24,7 @@
  */
 
 #include "tensordec.h"
+#include <nnstreamer_subplugin.h>
 #include <gst/gstinfo.h>
 
 typedef struct _TensorDecDefList TensorDecDefList;
@@ -123,7 +124,7 @@ tensordec_exit (const gchar * name)
  * @brief Find decoders subplugin with the name
  * @param[in] name the name of decoder (modename)
  */
-TensorDecDef *
+const TensorDecDef *
 tensordec_find (const gchar * name)
 {
   TensorDecDefList *list = &listhead;
@@ -142,6 +143,6 @@ tensordec_find (const gchar * name)
     list = list->next;
   } while (list != NULL);
 
-  GST_ERROR ("Cannot find decoder subplugin, \"%s\"\n", name);
-  return NULL;
+  /* If not found, try to search with nnstreamer_subplugin APIs */
+  return get_subplugin (NNS_SUBPLUGIN_DECODER, name);
 }

diff --git a/gst/nnstreamer/tensor_decoder/tensordec.c b/gst/nnstreamer/tensor_decoder/tensordec.c
index 193a0a0..36c3b9b 100644 (file)
--- a/gst/nnstreamer/tensor_decoder/tensordec.c
+++ b/gst/nnstreamer/tensor_decoder/tensordec.c
@@ -397,7 +397,7 @@ gst_tensordec_set_property (GObject * object, guint prop_id,
       break;
     case PROP_MODE:{
       int i;
-      TensorDecDef *decoder;
+      const TensorDecDef *decoder;
       gboolean retval = TRUE;
       temp_string = g_value_dup_string (value);
       decoder = tensordec_find (temp_string);

diff --git a/gst/nnstreamer/tensor_decoder/tensordec.h b/gst/nnstreamer/tensor_decoder/tensordec.h
index 19f7509..ddd874a 100644 (file)
--- a/gst/nnstreamer/tensor_decoder/tensordec.h
+++ b/gst/nnstreamer/tensor_decoder/tensordec.h
@@ -71,7 +71,7 @@ struct _GstTensorDec
   void (*cleanup_plugin_data)(GstTensorDec *self); /**< exit() of subplugin is registered here. If it's null, gfree(plugin_data) is used. */
   GstTensorsConfig tensor_config; /**< configured tensor info @todo support tensors in the future */
 
-  TensorDecDef *decoder; /**< Plugin object */
+  const TensorDecDef *decoder; /**< Plugin object */
 };
 
 /**
@@ -158,7 +158,7 @@ struct _TensorDecDef
 
 extern gboolean tensordec_probe (TensorDecDef *decoder);
 extern void tensordec_exit (const gchar *name);
-extern TensorDecDef *tensordec_find (const gchar *name);
+extern const TensorDecDef *tensordec_find (const gchar *name);
 
 
 G_END_DECLS

diff --git a/gst/nnstreamer/tensor_filter/tensor_filter.c b/gst/nnstreamer/tensor_filter/tensor_filter.c
index 710bd67..e3bc53a 100644 (file)
--- a/gst/nnstreamer/tensor_filter/tensor_filter.c
+++ b/gst/nnstreamer/tensor_filter/tensor_filter.c
@@ -59,6 +59,7 @@
 #include <glib.h>
 #include <string.h>
 
+#include <nnstreamer_subplugin.h>
 #include "tensor_filter.h"
 
 /**
@@ -107,6 +108,117 @@
   } \
 } while (0)
 
+
+typedef struct _TensorFilterSPList TensorFilterSPList;
+/**
+ * @brief Linked list having all registered filter subplugins
+ */
+struct _TensorFilterSPList
+{
+  TensorFilterSPList *next; /**< "Next" in the list */
+  GstTensorFilterFramework *body; /**< "Data" in the list element */
+};
+static GstTensorFilterFramework unknown = {
+  .name = "unknown",
+};
+static TensorFilterSPList listhead = {.next = NULL,.body = &unknown };
+
+/**
+ * @brief Filter subplugin should call this to register itself
+ * @param[in] tfsp Tensor-Filter Sub-Plugin to be registered
+ * @return TRUE if registered. FALSE is failed or duplicated.
+ */
+gboolean
+tensor_filter_probe (GstTensorFilterFramework * tfsp)
+{
+  TensorFilterSPList *list;
+
+  if (!tfsp || !tfsp->name || !tfsp->name[0]) {
+    GST_ERROR ("Cannot register invalid filter subplugin.\n");
+    return FALSE;
+  }
+
+  list = &listhead;
+  do {
+    if (0 == g_strcmp0 (list->body->name, tfsp->name)) {
+      /* Duplicated! */
+      GST_ERROR ("DUplicated filter sub-plugin name found: %s\n", tfsp->name);
+      return FALSE;
+    }
+    if (list->next == NULL) {
+      TensorFilterSPList *next = g_malloc (sizeof (TensorFilterSPList));
+      next->next = NULL;
+      next->body = tfsp;
+      list->next = next;
+      break;
+    }
+    list = list->next;
+  } while (list != NULL);
+
+  GST_INFO ("A new sub-plugin, \"%s\" is registered for tensor_filter.\n",
+      tfsp->name);
+  return TRUE;
+}
+
+/**
+ * @brief filter sub-plugin may call this to unregister itself
+ * @param[in] name the name of filter sub-plugin
+ */
+void
+tensor_filter_exit (const gchar * name)
+{
+  TensorFilterSPList *list = &listhead;
+
+  if (!name || !name[0]) {
+    GST_ERROR ("Cannot unregister without a proper name.");
+    return;
+  }
+
+  list = &listhead;
+  do {
+    if (list->next != NULL && 0 == g_strcmp0 (list->next->body->name, name)) {
+      TensorFilterSPList *found = list->next;
+      list->next = found->next;
+      g_free (found);
+      GST_INFO ("A tensor_filter sub-plugin \"%s\" is removed.", name);
+      return;
+    }
+    list = list->next;
+  } while (list != NULL);
+
+  GST_ERROR ("Cannot find a tensor_filter sub-plugin \"%s\".", name);
+  return;
+}
+
+/**
+ * @brief Find filter sub-plugin with the name
+ * @param[in] name The name of tensor_filter sub-plugin
+ * @return NULL if not found or the sub-plugin object.
+ */
+static const GstTensorFilterFramework *
+tensor_filter_find (const gchar * name)
+{
+  TensorFilterSPList *list = &listhead;
+
+  if (!name || !name[0]) {
+    GST_ERROR ("The name of tensor_filter sub-plugin is not given.");
+    return NULL;
+  }
+
+  do {
+    g_assert (list->body);
+
+    if (0 == g_strcmp0 (list->body->name, name)) {
+      return list->body;
+    }
+    list = list->next;
+  } while (list != NULL);
+
+  /* If not found, try to search with nnstremer_subplugin APIs */
+  return get_subplugin (NNS_SUBPLUGIN_FILTER, name);
+}
+
+/** @todo Obsolete. To be removed soon. */
 GstTensorFilterFramework *tensor_filter_supported[] = {
   [_T_F_UNDEFINED] = NULL,
 
@@ -127,17 +239,6 @@ GstTensorFilterFramework *tensor_filter_supported[] = {
   0,
 };
 
-const char *nnfw_names[] = {
-  [_T_F_UNDEFINED] = "Not supported",
-
-  [_T_F_CUSTOM] = "custom",
-  [_T_F_TENSORFLOW_LITE] = "tensorflow-lite",
-  [_T_F_TENSORFLOW] = "tensorflow",
-  [_T_F_CAFFE2] = "caffe2",
-
-  0,
-};
-
 GST_DEBUG_CATEGORY_STATIC (gst_tensor_filter_debug);
 #define GST_CAT_DEFAULT gst_tensor_filter_debug
 
@@ -228,7 +329,7 @@ static gboolean gst_tensor_filter_stop (GstBaseTransform * trans);
         if (filter->fw && filter->fw->close) \
           filter->fw->close (filter, &filter->privateData); \
         filter->prop.fw_opened = FALSE; \
-        filter->prop.nnfw = _T_F_UNDEFINED; \
+        g_free ((char *) filter->prop.fwname); \
         filter->fw = NULL; \
       } \
     } while (0)
@@ -349,7 +450,7 @@ gst_tensor_filter_init (GstTensorFilter * self)
   prop = &self->prop;
 
   /* init NNFW properties */
-  prop->nnfw = _T_F_UNDEFINED;
+  prop->fwname = NULL;
   prop->fw_opened = FALSE;
   prop->input_configured = FALSE;
   prop->output_configured = FALSE;
@@ -422,27 +523,21 @@ gst_tensor_filter_set_property (GObject * object, guint prop_id,
     {
       const gchar *fw_name = g_value_get_string (value);
 
-      if (prop->nnfw != _T_F_UNDEFINED) {
+      if (self->fw != NULL) {
         gst_tensor_filter_close_fw (self);
       }
-      prop->nnfw = find_key_strv (nnfw_names, fw_name);
+      self->fw = tensor_filter_find (fw_name);
+
       silent_debug ("Framework = %s\n", fw_name);
-      if (prop->nnfw == -1 || prop->nnfw == _T_F_UNDEFINED) {
+      if (self->fw == NULL) {
         GST_WARNING_OBJECT (self,
             "Cannot identify the given neural network framework, %s\n",
             fw_name);
-        prop->nnfw = _T_F_UNDEFINED;
         break;
       }
 
-      self->fw = tensor_filter_supported[prop->nnfw];
-      if (self->fw == NULL) {
-        GST_WARNING_OBJECT (self,
-            "The given neural network framework is identified but not supported, %s\n",
-            fw_name);
-        prop->nnfw = _T_F_UNDEFINED;
-        break;
-      }
+      g_free ((char *) prop->fwname);
+      prop->fwname = g_strdup (fw_name);
 
       /* See if mandatory methods are filled in */
       g_assert (self->fw->invoke_NN);
@@ -626,7 +721,7 @@ gst_tensor_filter_get_property (GObject * object, guint prop_id,
       g_value_set_boolean (value, self->silent);
       break;
     case PROP_FRAMEWORK:
-      g_value_set_string (value, nnfw_names[prop->nnfw]);
+      g_value_set_string (value, prop->fwname);
       break;
     case PROP_MODEL:
       g_value_set_string (value, prop->model_file);

diff --git a/gst/nnstreamer/tensor_filter/tensor_filter.h b/gst/nnstreamer/tensor_filter/tensor_filter.h
index 83527f3..e4a763e 100644 (file)
--- a/gst/nnstreamer/tensor_filter/tensor_filter.h
+++ b/gst/nnstreamer/tensor_filter/tensor_filter.h
@@ -53,8 +53,6 @@ typedef struct _GstTensorFilter GstTensorFilter;
 typedef struct _GstTensorFilterClass GstTensorFilterClass;
 typedef struct _GstTensorFilterFramework GstTensorFilterFramework;
 
-extern const char *nnfw_names[];
-
 /**
  * @brief Internal data structure for tensor_filter instances.
  */
@@ -64,7 +62,7 @@ struct _GstTensorFilter
 
   void *privateData; /**< NNFW plugin's private data is stored here */
   GstTensorFilterProperties prop; /**< NNFW plugin's properties */
-  GstTensorFilterFramework *fw; /**< The implementation core of the NNFW. NULL if not configured */
+  const GstTensorFilterFramework *fw; /**< The implementation core of the NNFW. NULL if not configured */
 
   /** internal properties for tensor-filter */
   int silent; /**< Verbose mode if FALSE. int instead of gboolean for non-glib custom plugins */
@@ -188,6 +186,10 @@ extern GstTensorFilterFramework NNS_support_custom;
 
 extern GstTensorFilterFramework *tensor_filter_supported[];
 
+/* extern functions for subplugin management */
+extern gboolean tensor_filter_probe (GstTensorFilterFramework *tfsp);
+extern void tensor_filter_exit (const gchar *name);
+
 G_END_DECLS
 
 #endif /* __GST_TENSOR_FILTER_H__ */

diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_custom.c b/gst/nnstreamer/tensor_filter/tensor_filter_custom.c
index 11c7701..a3de460 100644 (file)
--- a/gst/nnstreamer/tensor_filter/tensor_filter_custom.c
+++ b/gst/nnstreamer/tensor_filter/tensor_filter_custom.c
@@ -244,3 +244,17 @@ GstTensorFilterFramework NNS_support_custom = {
   .open = custom_open,
   .close = custom_close,
 };
+
+/** @brief Initialize this object for tensor_filter subplugin runtime register */
+__attribute__ ((constructor))
+     void init_filter_custom (void)
+{
+  tensor_filter_probe (&NNS_support_custom);
+}
+
+/** @brief Destruct the subplugin */
+__attribute__ ((destructor))
+     void fini_filter_custom (void)
+{
+  tensor_filter_exit (NNS_support_custom.name);
+}

diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow.c b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow.c
index 4612be9..e3c9951 100644 (file)
--- a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow.c
+++ b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow.c
@@ -158,3 +158,17 @@ GstTensorFilterFramework NNS_support_tensorflow = {
   .open = tf_open,
   .close = tf_close,
 };
+
+/** @brief Initialize this object for tensor_filter subplugin runtime register */
+__attribute__ ((constructor))
+     void init_filter_tf (void)
+{
+  tensor_filter_probe (&NNS_support_tensorflow);
+}
+
+/** @brief Destruct the subplugin */
+__attribute__ ((destructor))
+     void fini_filter_tf (void)
+{
+  tensor_filter_exit (NNS_support_tensorflow.name);
+}

diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c
index f0e0821..7e3fb9e 100644 (file)
--- a/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c
+++ b/gst/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c
@@ -157,3 +157,17 @@ GstTensorFilterFramework NNS_support_tensorflow_lite = {
   .open = tflite_open,
   .close = tflite_close,
 };
+
+/** @brief Initialize this object for tensor_filter subplugin runtime register */
+__attribute__ ((constructor))
+     void init_filter_tflite (void)
+{
+  tensor_filter_probe (&NNS_support_tensorflow_lite);
+}
+
+/** @brief Destruct the subplugin */
+__attribute__ ((destructor))
+     void fini_filter_tflite (void)
+{
+  tensor_filter_exit (NNS_support_tensorflow_lite.name);
+}

diff --git a/gst/nnstreamer/tensor_typedef.h b/gst/nnstreamer/tensor_typedef.h
index e1c4eab..294ba06 100644 (file)
--- a/gst/nnstreamer/tensor_typedef.h
+++ b/gst/nnstreamer/tensor_typedef.h
@@ -145,7 +145,7 @@ typedef struct
  */
 typedef struct _GstTensorFilterProperties
 {
-  nnfw_type nnfw; /**< The enum value of corresponding NNFW. _T_F_UNDEFINED if not configured */
+  const char *fwname; /**< The name of NN Framework */
   int fw_opened; /**< TRUE IF open() is called or tried. Use int instead of gboolean because this is refered by custom plugins. */
   const char *model_file; /**< Filepath to the model file (as an argument for NNFW). char instead of gchar for non-glib custom plugins */