[FILTER] add nnapi property to handle

Add nnapi property in the tensor_filter element.
Use nnapi=(true|false)(:(cpu|gpu|npu)) option for tensor_filter
element. For example,

gst-launch-1.0 --gst-plugin-path=${PATH_TO_PLUGIN} filesrc
location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze !
videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter
! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} nnapi=true:gpu ! filesink location=tensorfilter.out.log

**Self evaluation:**
1. Build test:	 [X]Passed [ ]Failed [ ]Skipped
2. Run test:	 [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: jijoong.moon <jijoong.moon@samsung.com>

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c
index 49d9ae4..2b5ca12 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c
@@ -30,6 +30,17 @@
 #include <string.h>
 
 #include "tensor_filter_tensorflow_lite_core.h"
+#include "tensor_common.h"
+
+/**
+ * @brief nnapi hw type string
+ */
+static const char *nnapi_hw_string[] = {
+  [NNAPI_CPU] = "cpu",
+  [NNAPI_GPU] = "gpu",
+  [NNAPI_NPU] = "npu",
+  [NNAPI_UNKNOWN] = "unknown"
+};
 
 void init_filter_tflite (void) __attribute__ ((constructor));
 void fini_filter_tflite (void) __attribute__ ((destructor));
@@ -58,6 +69,17 @@ tflite_close (const GstTensorFilterProperties * prop, void **private_data)
 }
 
 /**
+ * @brief return nnapi_hw type
+ */
+static nnapi_hw
+get_nnapi_hw_type (const gchar * str)
+{
+  gint index = 0;
+  index = find_key_strv (nnapi_hw_string, str);
+  return (index < 0) ? NNAPI_UNKNOWN : index;
+}
+
+/**
  * @brief Load tensorflow lite modelfile
  * @param prop property of tensor_filter instance
  * @param private_data : tensorflow lite plugin's private data
@@ -70,6 +92,18 @@ tflite_loadModelFile (const GstTensorFilterProperties * prop,
     void **private_data)
 {
   tflite_data *tf;
+  nnapi_hw hw = NNAPI_UNKNOWN;
+  if (prop->nnapi) {
+    gchar **strv = NULL;
+    strv = g_strsplit (prop->nnapi, ":", 2);
+    if (!g_strcmp0 (strv[0], "true") && g_strv_length (strv) >= 2) {
+      hw = get_nnapi_hw_type (strv[1]);
+    } else if (!g_strcmp0 (strv[0], "true") && g_strv_length (strv) == 1) {
+      /** defalut hw for nnapi is CPU */
+      hw = NNAPI_CPU;
+    }
+  }
+
   if (*private_data != NULL) {
     /** @todo : Check the integrity of filter->data and filter->model_file, nnfw */
     tf = *private_data;
@@ -82,7 +116,7 @@ tflite_loadModelFile (const GstTensorFilterProperties * prop,
   }
   tf = g_new0 (tflite_data, 1); /** initialize tf Fill Zero! */
   *private_data = tf;
-  tf->tflite_private_data = tflite_core_new (prop->model_file);
+  tf->tflite_private_data = tflite_core_new (prop->model_file, hw);
   if (tf->tflite_private_data) {
     if (tflite_core_init (tf->tflite_private_data)) {
       g_printerr ("failed to initialize the object: Tensorflow-lite");

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.cc b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.cc
index 3030815..558cd65 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.cc
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.cc
@@ -41,10 +41,15 @@
  * @note       the model of _model_path will be loaded simultaneously
  * @return     Nothing
  */
-TFLiteCore::TFLiteCore (const char * _model_path)
+TFLiteCore::TFLiteCore (const char * _model_path, nnapi_hw hw)
 {
   model_path = _model_path;
-  use_nnapi = nnsconf_get_custom_value_bool ("tensorflowlite", "enable_nnapi", FALSE);
+  if(hw == NNAPI_UNKNOWN){
+    use_nnapi = nnsconf_get_custom_value_bool ("tensorflowlite", "enable_nnapi", FALSE);
+  } else {
+    use_nnapi = TRUE;
+    accel = hw;
+  }
 
   gst_tensors_info_init (&inputTensorMeta);
   gst_tensors_info_init (&outputTensorMeta);
@@ -381,9 +386,9 @@ TFLiteCore::invoke (const GstTensorMemory * input, GstTensorMemory * output)
  * @return     TFLiteCore class
  */
 void *
-tflite_core_new (const char * _model_path)
+tflite_core_new (const char * _model_path, nnapi_hw hw)
 {
-  return new TFLiteCore (_model_path);
+  return new TFLiteCore (_model_path, hw);
 }
 
 /**

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.h b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.h
index 5ee21e9..d38a4ce 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.h
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.h
@@ -43,7 +43,7 @@
 class TFLiteCore
 {
 public:
-  TFLiteCore (const char * _model_path);
+  TFLiteCore (const char *_model_path, nnapi_hw hw);
   ~TFLiteCore ();
 
   int init ();
@@ -59,6 +59,7 @@ private:
 
   const char *model_path;
   bool use_nnapi;
+  nnapi_hw accel;
 
   GstTensorsInfo inputTensorMeta;  /**< The tensor info of input tensors */
   GstTensorsInfo outputTensorMeta;  /**< The tensor info of output tensors */
@@ -81,7 +82,7 @@ extern "C"
 {
 #endif
 
-  void *tflite_core_new (const char *_model_path);
+  void *tflite_core_new (const char *_model_path, nnapi_hw hw);
   void tflite_core_delete (void * tflite);
   int tflite_core_init (void * tflite);
   const char *tflite_core_getModelPath (void * tflite);

diff --git a/gst/nnstreamer/nnstreamer_plugin_api_filter.h b/gst/nnstreamer/nnstreamer_plugin_api_filter.h
index 121b2a7..234cd5c 100644 (file)
--- a/gst/nnstreamer/nnstreamer_plugin_api_filter.h
+++ b/gst/nnstreamer/nnstreamer_plugin_api_filter.h
@@ -31,6 +31,18 @@ extern "C" {
 #endif
 
 /**
+ * @brief nnapi hw properties.
+ */
+typedef enum
+{
+  NNAPI_CPU = 0,
+  NNAPI_GPU = 1,
+  NNAPI_NPU = 2,
+
+  NNAPI_UNKNOWN
+} nnapi_hw;
+
+/**
  * @brief GstTensorFilter's properties for NN framework (internal data structure)
  *
  * Because custom filters of tensor_filter may need to access internal data
@@ -50,6 +62,8 @@ typedef struct _GstTensorFilterProperties
   GstTensorsInfo output_meta; /**< configured output tensor info */
 
   const char *custom_properties; /**< sub-plugin specific custom property values in string */
+  const char *nnapi;
+
 } GstTensorFilterProperties;
 
 /**

diff --git a/gst/nnstreamer/tensor_filter/tensor_filter.c b/gst/nnstreamer/tensor_filter/tensor_filter.c
index 6fc4ab6..e0c11e6 100644 (file)
--- a/gst/nnstreamer/tensor_filter/tensor_filter.c
+++ b/gst/nnstreamer/tensor_filter/tensor_filter.c
@@ -109,6 +109,8 @@
 
 #define g_free_const(x) g_free((void*)(long)(x))
 
+#define REGEX_NNAPI_OPTION "(^(true|false)$|^(true|false)([:]?(cpu|gpu|npu)))"
+
 /**
  * @brief Validate filter sub-plugin's data.
  */
@@ -186,7 +188,8 @@ enum
   PROP_OUTPUTTYPE,
   PROP_OUTPUTNAME,
   PROP_CUSTOM,
-  PROP_SUBPLUGINS
+  PROP_SUBPLUGINS,
+  PROP_NNAPI
 };
 
 /**
@@ -342,7 +345,9 @@ gst_tensor_filter_class_init (GstTensorFilterClass * klass)
       g_param_spec_string ("sub-plugins", "Sub-plugins",
           "Registrable sub-plugins list", "",
           G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
-
+  g_object_class_install_property (gobject_class, PROP_NNAPI,
+      g_param_spec_string ("nnapi", "NNAPI",
+          "Enable Neural Newtorks API ?", "", G_PARAM_READWRITE));
   gst_element_class_set_details_simple (gstelement_class,
       "Tensor_Filter",
       "Converter/Filter/Tensor",
@@ -394,6 +399,7 @@ gst_tensor_filter_init (GstTensorFilter * self)
   prop->input_configured = FALSE;
   prop->output_configured = FALSE;
   prop->model_file = NULL;
+  prop->nnapi = NULL;
   prop->custom_properties = NULL;
   gst_tensors_info_init (&prop->input_meta);
   gst_tensors_info_init (&prop->output_meta);
@@ -749,6 +755,17 @@ gst_tensor_filter_set_property (GObject * object, guint prop_id,
       prop->custom_properties = g_value_dup_string (value);
       silent_debug ("Custom Option = %s\n", prop->custom_properties);
       break;
+    case PROP_NNAPI:
+      prop->nnapi = g_value_dup_string (value);
+      if (!g_regex_match_simple (REGEX_NNAPI_OPTION, prop->nnapi, 0, 0)) {
+        gchar *name = gst_object_get_name ((GstObject *) object);
+        g_critical
+            ("%s: nnapi: \'%s\' is not valid string: it should be in the form of BOOL:ACCELLERATOR or BOOL with a regex, "
+            REGEX_NNAPI_OPTION "\n", name, prop->nnapi);
+        g_free (name);
+        break;
+      }
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -887,6 +904,9 @@ gst_tensor_filter_get_property (GObject * object, guint prop_id,
       g_value_take_string (value, g_string_free (subplugins, FALSE));
       break;
     }
+    case PROP_NNAPI:
+      g_value_set_string (value, prop->nnapi);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;

diff --git a/meson.build b/meson.build
index f3dd45b..3152067 100644 (file)
--- a/meson.build
+++ b/meson.build
@@ -226,7 +226,7 @@ if get_option('enable-tizen')
   add_project_arguments('-D__TIZEN__=1', language: ['c', 'cpp'])
 endif
 
-# nnfw
+# nnfw ( details in https://review.tizen.org/gerrit/p/platform/core/ml/nnfw )
 if get_option('enable-nnfw')
     add_project_arguments('-DENABLE_NNFW=1', language: ['c', 'cpp'])
 endif