[Filter/nnfw] Tizen-nnfw skeleton

Skeleton code for Tizen-nnfw is added.
New nnfw of Tizen has its own interface; thus, we can no longer
rely on tensorflow-lite/nnapi for this.

Changes v2:
- Enabled nnfw by default for Tizen
- Applied jaeyun's comments (meson script style fix, nnfw subplugin explicitness of properties)

Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>

diff --git a/ext/nnstreamer/tensor_filter/meson.build b/ext/nnstreamer/tensor_filter/meson.build
index b0de81b..74a71db 100644 (file)
--- a/ext/nnstreamer/tensor_filter/meson.build
+++ b/ext/nnstreamer/tensor_filter/meson.build
@@ -1,3 +1,39 @@
+if get_option('enable-nnfw-runtime')
+  filter_sub_nnfw_sources = ['tensor_filter_nnfw.c']
+
+  nnstreamer_filter_nnfw_sources = []
+  foreach s : filter_sub_nnfw_sources
+    nnstreamer_filter_nnfw_sources += join_paths(meson.current_source_dir(), s)
+  endforeach
+
+  nnfw_dep = dependency('nnfw', required: false)
+  add_args = ''
+  nnstreamer_filter_nnfw_deps = [glib_dep, gst_dep, nnstreamer_dep]
+
+  if nnfw_dep.found()
+    nnstreamer_filter_nnfw_deps += nnfw_dep
+  else
+    # Until nnfw supports pkg-config, we need to do this primitively.
+    add_args = '-lnnfw-dev'
+  endif
+
+
+  shared_library('nnstreamer_filter_nnfw',
+    nnstreamer_filter_nnfw_sources,
+    dependencies: nnstreamer_filter_nnfw_deps,
+    c_args : add_args,
+    install: true,
+    install_dir: filter_subplugin_install_dir
+  )
+  static_library('nnstreamer_filter_nnfw',
+    nnstreamer_filter_nnfw_sources,
+    dependencies: nnstreamer_filter_nnfw_deps,
+    c_args : add_args,
+    install: false,
+    install_dir: filter_subplugin_install_dir
+  )
+endif
+
 if get_option('enable-tensorflow')
   filter_sub_tf_sources = [
     'tensor_filter_tensorflow.c',

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_nnfw.c b/ext/nnstreamer/tensor_filter/tensor_filter_nnfw.c
new file mode 100644 (file)
index 0000000..8ed2eb9
--- /dev/null
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_nnfw.c
@@ -0,0 +1,111 @@
+/**
+ * GStreamer Tensor_Filter, Tizen NNFW Module
+ * Copyright (C) 2019 MyungJoo Ham <myungjoo.ham@samsung.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ */
+/**
+ * @file       tensor_filter_nnfw.c
+ * @date       24 Sep 2019
+ * @brief      Tizen-NNFW module for tensor_filter gstreamer plugin
+ * @see                http://github.com/nnsuite/nnstreamer
+ * @author     MyungJoo Ham <myungjoo.ham@samsung.com>
+ * @bug                No known bugs except for NYI items
+ *
+ * This is the per-NN-framework plugin (Tizen nnfw) for tensor_filter.
+ *
+ * @todo Check if nnfw supports dynamic input dimension. (if so, we need to supply setInputDim)
+ * @todo Decide whether to let nnfw allocate output buffers or we will feed output buffers to nnfw.
+ *
+ */
+
+#include <glib.h>
+#include <tensor_common.h>
+#include <nnstreamer_plugin_api_filter.h>
+
+#include <nnfw.h>
+
+void init_filter_nnfw (void) __attribute__ ((constructor));
+void fini_filter_nnfw (void) __attribute__ ((destructor));
+
+/**
+ * @brief The standard tensor_filter callback
+ */
+static int nnfw_open (const GstTensorFilterProperties * prop,
+    void **private_data)
+{
+  return 0;
+}
+
+/**
+ * @brief The standard tensor_filter callback
+ */
+static void nnfw_close (const GstTensorFilterProperties * prop,
+    void **private_data)
+{
+}
+
+/**
+ * @brief The standard tensor_filter callback
+ */
+static int nnfw_getInputDim (const GstTensorFilterProperties * prop,
+      void **private_data, GstTensorsInfo * info)
+{
+  return 0;
+}
+
+/**
+ * @brief The standard tensor_filter callback
+ */
+static int nnfw_getOutputDim (const GstTensorFilterProperties * prop,
+      void **private_data, GstTensorsInfo * info)
+{
+  return 0;
+}
+
+/**
+ * @brief The standard tensor_filter callback
+ */
+static int nnfw_invoke (const GstTensorFilterProperties * prop,
+    void **private_data, const GstTensorMemory * input,
+    GstTensorMemory * output)
+{
+  return 0;
+}
+
+static gchar filter_subplugin_nnfw[] = "nnfw";
+
+static GstTensorFilterFramework NNS_support_nnfw = {
+  .name = filter_subplugin_nnfw,
+  .allow_in_place = FALSE,
+  .allocate_in_invoke = FALSE,
+  .run_without_model = FALSE,
+  .invoke_NN = nnfw_invoke,
+  .getInputDimension = nnfw_getInputDim,
+  .getOutputDimension = nnfw_getOutputDim,
+  .open = nnfw_open,
+  .close = nnfw_close,
+};
+
+/**@brief Initialize this object for tensor_filter subplugin runtime register */
+void
+init_filter_nnfw (void)
+{
+  nnstreamer_filter_probe (&NNS_support_nnfw);
+}
+
+/** @brief Destruct the subplugin */
+void
+fini_filter_nnfw (void)
+{
+  nnstreamer_filter_exit (NNS_support_nnfw.name);
+}

diff --git a/meson.build b/meson.build
index dd6d610..af1ccde 100644 (file)
--- a/meson.build
+++ b/meson.build
@@ -226,6 +226,8 @@ endif
 # nnfw ( details in https://review.tizen.org/gerrit/p/platform/core/ml/nnfw )
 if get_option('enable-nnfw')
   add_project_arguments('-DENABLE_NNFW=1', language: ['c', 'cpp'])
+  # For tf-lite/nnapi, enable-nnfw allows to use nnfw as a backend.
+  # This also enables nnfw as a subplugin for Tizen.
 endif
 
 # Patch for non-tizen build

diff --git a/meson_options.txt b/meson_options.txt
index 341a2ef..0d6be82 100644 (file)
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -18,3 +18,4 @@ option('enable-tizen', type: 'boolean', value: false)
 option('enable-element-restriction', type: 'boolean', value: false) # true to restrict gst-elements in api
 option('restricted-elements', type: 'string', value: '')
 option('enable-nnfw', type: 'boolean', value: false)
+option('enable-nnfw-runtime', type: 'boolean', value: false)

diff --git a/packaging/nnstreamer.spec b/packaging/nnstreamer.spec
index 0c74cda..58b64c1 100644 (file)
--- a/packaging/nnstreamer.spec
+++ b/packaging/nnstreamer.spec
@@ -3,6 +3,7 @@
 %define                gstlibdir       %{_libdir}/%{gstpostfix}
 %define                nnstexampledir  /usr/lib/nnstreamer/bin
 %define                tensorflow_support      0
+%define                enable_nnfw     1
 
 # If it is tizen, we can export Tizen API packages.
 %bcond_with tizen
@@ -63,6 +64,9 @@ BuildRequires: tensorflow-devel
 BuildRequires: lcov
 # BuildRequires:       taos-ci-unittest-coverage-assessment
 %endif
+
+%define                enable_nnfw_runtime     -Denable-nnfw-runtime=false
+%define                enable_nnfw_r   0
 %if %{with tizen}
 BuildRequires: pkgconfig(mm-resource-manager)
 BuildRequires: pkgconfig(mm-camcorder)
@@ -72,6 +76,16 @@ BuildRequires:       pkgconfig(capi-base-common)
 BuildRequires: pkgconfig(dlog)
 BuildRequires: gst-plugins-bad-devel
 BuildRequires: gst-plugins-base-devel
+
+%if 0%{?enable_nnfw}
+%ifarch %arm aarch64
+# Tizen 5.5 M2+ support nn-runtime (nnfw)
+# As of 2019-09-24, unfortunately, nnfw does not support pkg-config
+BuildRequires:  nnfw-devel
+%define                enable_nnfw_runtime     -Denable-nnfw-runtime=true
+%define                enable_nnfw_r   1
+%endif
+%endif
 %endif
 
 # Unit Testing Uses SSAT (hhtps://github.com/myungjoo/SSAT.git)
@@ -114,6 +128,12 @@ Requires:  nnstreamer = %{version}-%{release}
 %description tensorflow-lite
 NNStreamer's tensor_fliter subplugin of TensorFlow Lite.
 
+%package nnfw
+Summary:       NNStreamer Tizen-nnfw runtime support
+Requires:      nnfw
+%description nnfw
+NNStreamer's tensor_filter subplugin of Tizen-NNFW Runtime. (5.5 M2 +)
+
 %package -n nnstreamer-python2
 Summary:  NNStreamer Python Custom Filter Support
 Requires: nnstreamer = %{version}-%{release}
@@ -214,7 +234,7 @@ CFLAGS="${CFLAGS} -fprofile-arcs -ftest-coverage"
 
 mkdir -p build
 
-meson --buildtype=plain --prefix=%{_prefix} --sysconfdir=%{_sysconfdir} --libdir=%{_libdir} --bindir=%{nnstexampledir} --includedir=%{_includedir} -Dinstall-example=true %{enable_tf} -Denable-pytorch=false -Denable-caffe2=false -Denable-env-var=false -Denable-symbolic-link=false %{enable_api} %{enable_tizen} %{restriction} build
+meson --buildtype=plain --prefix=%{_prefix} --sysconfdir=%{_sysconfdir} --libdir=%{_libdir} --bindir=%{nnstexampledir} --includedir=%{_includedir} -Dinstall-example=true %{enable_tf} -Denable-pytorch=false -Denable-caffe2=false -Denable-env-var=false -Denable-symbolic-link=false %{enable_api} %{enable_tizen} %{restriction} %{enable_nnfw_runtime} build
 
 ninja -C build %{?_smp_mflags}
 
@@ -322,6 +342,13 @@ cp -r result %{buildroot}%{_datadir}/nnstreamer/unittest/
 %defattr(-,root,root,-)
 %{_prefix}/lib/nnstreamer/filters/libnnstreamer_filter_tensorflow-lite.so
 
+%if 0%{?enable_nnfw_r}
+%files nnfw
+%manifest nnstreamer.manifest
+%defattr(-,root,root,-)
+%{_prefix}/lib/nnstreamer/filters/libnnstreamer_filter_nnfw.so
+%endif
+
 %files -n nnstreamer-python2
 %manifest capi-nnstreamer.manifest
 %defattr(-,root,root,-)