From 8ddfcccec3b639173991320f4674d65739908d31 Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Thu, 29 Aug 2019 19:51:50 +0900 Subject: [PATCH] [filter_single] Added tensor_filter_single prototype Added prototype for tensor_filter_single tensor_filter_single is going to form the backend for singleshot api to minimize the latency. This is made to be independent of GStreamer, supporting all the existing tensor_filter plugins Signed-off-by: Parichay Kapoor --- api/capi/include/tensor_filter_single.h | 94 ++++++++++++++++ api/capi/meson.build | 21 +++- api/capi/src/tensor_filter_single.c | 186 ++++++++++++++++++++++++++++++++ meson.build | 1 + 4 files changed, 298 insertions(+), 4 deletions(-) create mode 100644 api/capi/include/tensor_filter_single.h create mode 100644 api/capi/src/tensor_filter_single.c diff --git a/api/capi/include/tensor_filter_single.h b/api/capi/include/tensor_filter_single.h new file mode 100644 index 0000000..8756db9 --- /dev/null +++ b/api/capi/include/tensor_filter_single.h @@ -0,0 +1,94 @@ +/** + * Copyright (C) 2019 Parichay kapoor + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + */ +/** + * @file tensor_filter_single.h + * @date 28 Aug 2019 + * @brief Element to use general neural network framework individually without gstreamer pipeline + * @see http://github.com/nnsuite/nnstreamer + * @author Parichay Kapoor + * @author MyungJoo Ham + * @bug No known bugs except for NYI items + */ + +#ifndef __G_TENSOR_FILTER_SINGLE_H__ +#define __G_TENSOR_FILTER_SINGLE_H__ + +#include +#include + +#include +#include + +G_BEGIN_DECLS +#define G_TYPE_TENSOR_FILTER_SINGLE \ + (g_tensor_filter_single_get_type()) +#define G_TENSOR_FILTER_SINGLE(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),G_TYPE_TENSOR_FILTER_SINGLE,GTensorFilterSingle)) +#define G_TENSOR_FILTER_SINGLE_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),G_TYPE_TENSOR_FILTER_SINGLE,GTensorFilterSingleClass)) +#define G_IS_TENSOR_FILTER_SINGLE(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),G_TYPE_TENSOR_FILTER_SINGLE)) +#define G_IS_TENSOR_FILTER_SINGLE_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),G_TYPE_TENSOR_FILTER_SINGLE)) +#define G_TENSOR_FILTER_SINGLE_CAST(obj) ((GTensorFilterSingle *)(obj)) + +typedef struct _GTensorFilterSingle GTensorFilterSingle; +typedef struct _GTensorFilterSingleClass GTensorFilterSingleClass; + +/** + * @brief Internal data structure for tensor_filter_single instances. + */ +struct _GTensorFilterSingle +{ + GObject element; /**< This is the parent object */ + + void *privateData; /**< NNFW plugin's private data is stored here */ + GstTensorFilterProperties prop; /**< NNFW plugin's properties */ + const GstTensorFilterFramework *fw; /**< The implementation core of the NNFW. NULL if not configured */ + + /* internal properties for tensor_filter_single */ + gboolean silent; /**< Verbose mode if FALSE. int instead of gboolean for non-glib custom plugins */ + gboolean started; /**< filter has been started */ + GstTensorsConfig in_config; /**< input tensor info */ + GstTensorsConfig out_config; /**< output tensor info */ +}; + +/** + * @brief GTensorFilterSingleClass inherits GObjectClass. + */ +struct _GTensorFilterSingleClass +{ + GObjectClass parent; /**< inherits GObjectClass */ + + + /** Invoke the filter for execution. */ + gboolean (*invoke) (GTensorFilterSingle * self, GstTensorMemory * input, GstTensorMemory * output); + /** Start the filter, must be called before invoke. */ + gboolean (*start) (GTensorFilterSingle * self); + /** Stop the filter.*/ + gboolean (*stop) (GTensorFilterSingle * self); + /** Check if the input is already configured */ + gboolean (*input_configured) (GTensorFilterSingle * self); + /** Check if the output is already configured */ + gboolean (*output_configured) (GTensorFilterSingle * self); +}; + +/** + * @brief Get Type function required for gst elements + */ +GType g_tensor_filter_single_get_type (void); + +G_END_DECLS +#endif /* __G_TENSOR_FILTER_SINGLE_H__ */ diff --git a/api/capi/meson.build b/api/capi/meson.build index 8e70c7e..b60b603 100644 --- a/api/capi/meson.build +++ b/api/capi/meson.build @@ -18,6 +18,22 @@ if meson.project_name() != 'nnstreamer' endif +inc = include_directories('include') +nninc = include_directories('../../gst') + +# Create dependency for tensor_filter_single for single-shot capi +tensor_filter_single_lib = shared_library ('tensor_filter_single_lib', + join_paths(meson.current_source_dir(), 'src', 'tensor_filter_single.c'), + dependencies: [glib_dep, gobject_dep], + include_directories: [inc, nninc], + install: false, + version: meson.project_version(), +) + +tensor_filter_single_dep = declare_dependency(link_with: tensor_filter_single_lib, + include_directories: [inc], +) + capi_main = [] capi_main += join_paths(meson.current_source_dir(), 'src', 'nnstreamer-capi-pipeline.c') capi_main += join_paths(meson.current_source_dir(), 'src', 'nnstreamer-capi-util.c') @@ -27,9 +43,6 @@ capi_devel_main = [] capi_devel_main += join_paths(meson.current_source_dir(), 'include', 'nnstreamer.h') capi_devel_main += join_paths(meson.current_source_dir(), 'include', 'nnstreamer-single.h') -inc = include_directories('include') -nninc = include_directories('../../gst') - tizen_deps = [] if (get_option('enable-tizen')) @@ -48,7 +61,7 @@ else endif capi_deps = [ - nnstreamer_dep, glib_dep, gst_dep, gst_app_dep, tizen_deps + nnstreamer_dep, glib_dep, gst_dep, gst_app_dep, tizen_deps, tensor_filter_single_dep ] nnstreamer_capi_lib = shared_library ('capi-nnstreamer', diff --git a/api/capi/src/tensor_filter_single.c b/api/capi/src/tensor_filter_single.c new file mode 100644 index 0000000..f43afd6 --- /dev/null +++ b/api/capi/src/tensor_filter_single.c @@ -0,0 +1,186 @@ +/** + * Copyright (C) 2019 Parichay Kapoor + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + */ +/** + * @file tensor_filter_single.c + * @date 28 Aug 2019 + * @brief Element to use general neural network framework directly without gstreamer pipeline + * @see http://github.com/nnsuite/nnstreamer + * @author Parichay Kapoor + * @bug No known bugs except for NYI items + * + * This is the main element for per-NN-framework plugins. + * Specific implementations for each NN framework must be written + * in each framework specific files; e.g., tensor_filter_tensorflow_lite.c + * + */ + +/** + * SECTION:element-tensor_filter_single + * + * An element that invokes neural network models and their framework or + * an independent shared object implementing tensor_filter_custom.h. + * The input and output are always in the format of other/tensor or + * other/tensors. This element is going to be the basis of single shot api. + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "tensor_filter_single.h" + +/** + * @brief Macro for debug mode. + */ +#ifndef DBG +#define DBG (!self->silent) +#endif + +#define g_tensor_filter_single_parent_class parent_class +G_DEFINE_TYPE (GTensorFilterSingle, g_tensor_filter_single, G_TYPE_OBJECT); + +/* GObject vmethod implementations */ +static void g_tensor_filter_single_finalize (GObject * object); +static void g_tensor_filter_single_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void g_tensor_filter_single_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); + +/* GTensorFilterSingle method implementations */ +static gboolean g_tensor_filter_single_invoke (GTensorFilterSingle * self, + GstTensorMemory * input, GstTensorMemory * output); + +/* Private functions */ +static gboolean g_tensor_filter_single_start (GTensorFilterSingle * self); +static gboolean g_tensor_filter_single_stop (GTensorFilterSingle * self); + +/** + * @brief initialize the tensor_filter's class + */ +static void +g_tensor_filter_single_class_init (GTensorFilterSingleClass * klass) +{ + GObjectClass *gobject_class; + + gobject_class = (GObjectClass *) klass; + + gobject_class->set_property = g_tensor_filter_single_set_property; + gobject_class->get_property = g_tensor_filter_single_get_property; + gobject_class->finalize = g_tensor_filter_single_finalize; + + klass->invoke = g_tensor_filter_single_invoke; + + /** TODO: share same setup properties as tensor_filter */ +} + +/** + * @brief initialize the new element + */ +static void +g_tensor_filter_single_init (GTensorFilterSingle * self) +{ + /** TODO: fill this */ +} + +/** + * @brief Function to finalize instance. + */ +static void +g_tensor_filter_single_finalize (GObject * object) +{ + gboolean status; + GTensorFilterSingle *self; + /** TODO: fill this */ + + self = G_TENSOR_FILTER_SINGLE (object); + + /** stop if not already stopped */ + if (self->started == TRUE) { + status = g_tensor_filter_single_stop (self); + g_debug ("Tensor filter single stop status: %d", status); + } +} + +/** + * @brief Setter for tensor_filter_single properties. + */ +static void +g_tensor_filter_single_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + /** TODO: share this with tensor_filter*/ +} + +/** + * @brief Getter for tensor_filter_single properties. + */ +static void +g_tensor_filter_single_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + /** TODO: share this with tensor_filter*/ +} + +/** + * @brief Called when the element starts processing, if fw not laoded + * @param self "this" pointer + * @return TRUE if there is no error. + */ +static gboolean +g_tensor_filter_single_start (GTensorFilterSingle * self) +{ + /** TODO: fill this */ + /** open framework, load model */ + return TRUE; +} + +/** + * @brief Called when the element stops processing, if fw loaded + * @param self "this" pointer + * @return TRUE if there is no error. + */ +static gboolean +g_tensor_filter_single_stop (GTensorFilterSingle * self) +{ + /** TODO: fill this */ + /** close framework, unload model */ + return TRUE; +} + + +/** + * @brief Called when an input supposed to be invoked + * @param self "this" pointer + * @param input memory containing input data to run processing on + * @param output memory to put output data into after processing + * @return TRUE if there is no error. + */ +static gboolean +g_tensor_filter_single_invoke (GTensorFilterSingle * self, + GstTensorMemory * input, GstTensorMemory * output) +{ + gboolean status = TRUE; + /** TODO: fill this */ + + /** start if not already started */ + if (self->started == FALSE) + status = g_tensor_filter_single_start (self); + + return status; +} diff --git a/meson.build b/meson.build index 91989dc..e76803c 100644 --- a/meson.build +++ b/meson.build @@ -108,6 +108,7 @@ add_project_arguments('-DNNSTREAMER_CONF_FILE="' + join_paths(nnstreamer_inidir, # Dependencies glib_dep = dependency('glib-2.0') +gobject_dep = dependency('gobject-2.0') gst_dep = dependency('gstreamer-' + gst_api_verision) gst_base_dep = dependency('gstreamer-base-' + gst_api_verision) gst_controller_dep = dependency('gstreamer-controller-' + gst_api_verision) -- 2.7.4