From 6ee3088349253292253963e10b192b0bc5ed168e Mon Sep 17 00:00:00 2001 From: Dongju Chae Date: Mon, 16 Dec 2019 14:24:58 +0900 Subject: [PATCH] [Refactor/TF] Refactor tf tensor-filter subplugin to remove .c/.cc wrapper This commit refactors tf tensor-filter subplugin to remove .c/.cc wrapper. Signed-off-by: Dongju Chae --- ext/nnstreamer/tensor_filter/meson.build | 5 +- .../tensor_filter/tensor_filter_tensorflow.c | 205 ---------------- ...sorflow_core.cc => tensor_filter_tensorflow.cc} | 266 ++++++++++++++++----- .../tensor_filter/tensor_filter_tensorflow_core.h | 112 --------- jni/nnstreamer.mk | 3 +- 5 files changed, 210 insertions(+), 381 deletions(-) delete mode 100644 ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.c rename ext/nnstreamer/tensor_filter/{tensor_filter_tensorflow_core.cc => tensor_filter_tensorflow.cc} (70%) delete mode 100644 ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.h diff --git a/ext/nnstreamer/tensor_filter/meson.build b/ext/nnstreamer/tensor_filter/meson.build index c454e5c..e26eb39 100644 --- a/ext/nnstreamer/tensor_filter/meson.build +++ b/ext/nnstreamer/tensor_filter/meson.build @@ -31,10 +31,7 @@ if get_option('enable-nnfw-runtime') endif if get_option('enable-tensorflow') - filter_sub_tf_sources = [ - 'tensor_filter_tensorflow.c', - 'tensor_filter_tensorflow_core.cc' - ] + filter_sub_tf_sources = ['tensor_filter_tensorflow.cc'] nnstreamer_filter_tf_sources = [] foreach s : filter_sub_tf_sources diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.c b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.c deleted file mode 100644 index 1dc2ac4..0000000 --- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.c +++ /dev/null @@ -1,205 +0,0 @@ -/** - * GStreamer Tensor_Filter, Tensorflow Module - * Copyright (C) 2018 Jijoong Moon - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - */ -/** - * @file tensor_filter_tensorflow.c - * @date 02 Aug 2018 - * @brief Tensorflow module for tensor_filter gstreamer plugin - * @see http://github.com/nnsuite/nnstreamer - * @author Jijoong Moon - * @bug No known bugs except for NYI items - * - * This is the per-NN-framework plugin (tensorflow) for tensor_filter. - * Fill in "GstTensorFilterFramework" for tensor_filter.h/c - * - */ - -#include -#include - -#include "nnstreamer_conf.h" -#include "tensor_filter_tensorflow_core.h" - -void init_filter_tf (void) __attribute__ ((constructor)); -void fini_filter_tf (void) __attribute__ ((destructor)); - -/** - * @brief internal data of tensorflow - */ -struct _Tf_data -{ - void *tf_private_data; -}; -typedef struct _Tf_data tf_data; - - -/** - * @brief Free privateData and move on. - */ -static void -tf_close (const GstTensorFilterProperties * prop, void **private_data) -{ - tf_data *tf; - tf = *private_data; - tf_core_delete (tf->tf_private_data); - g_free (tf); - *private_data = NULL; -} - -/** - * @brief Load tensorflow modelfile - * @param prop: property of tensor_filter instance - * @param private_data : tensorflow plugin's private data - * @return 0 if successfully loaded. 1 if skipped (already loaded). - * -1 if the object construction is failed. - * -2 if the object initialization if failed - */ -static int -tf_loadModelFile (const GstTensorFilterProperties * prop, void **private_data) -{ - tf_data *tf; - - if (*private_data != NULL) { - tf = *private_data; - if (g_strcmp0 (prop->model_files[0], - tf_core_getModelPath (tf->tf_private_data)) != 0) { - tf_close (prop, private_data); - } else { - return 1; - } - } - - tf = *private_data = g_new0 (tf_data, 1); - if (tf == NULL) { - g_printerr ("Failed to allocate memory for filter subplugin."); - return -1; - } - - tf->tf_private_data = tf_core_new (prop->model_files[0]); - - if (tf->tf_private_data) { - if (tf_core_init (tf->tf_private_data, prop)) { - g_printerr ("failed to initailize the object: tensorflow"); - return -2; - } - return 0; - } else { - g_printerr ("failed to create the object: tensorflow"); - return -1; - } -} - -/** - * @brief The open callback for GstTensorFilterFramework. Called before anything else - * @param prop: property of tensor_filter instance - * @param private_data : tensorflow plugin's private data - */ -static int -tf_open (const GstTensorFilterProperties * prop, void **private_data) -{ - int retval = tf_loadModelFile (prop, private_data); - g_assert (retval >= 0); /** This must be called only once */ - return retval; -} - -/** - * @brief The mandatory callback for GstTensorFilterFramework - * @param prop: property of tensor_filter instance - * @param private_data : tensorflow plugin's private data - * @param[in] input The array of input tensors - * @param[out] output The array of output tensors - */ -static int -tf_run (const GstTensorFilterProperties * prop, void **private_data, - const GstTensorMemory * input, GstTensorMemory * output) -{ - int retval; - tf_data *tf; - tf = *private_data; - g_assert (*private_data); - retval = tf_core_run (tf->tf_private_data, input, output); - g_assert (retval == 0); - return retval; -} - -/** - * @brief The optional callback for GstTensorFilterFramework - * @param prop: property of tensor_filter instance - * @param private_data : tensorflow plugin's private data - * @param[out] info The dimesions and types of input tensors - */ -static int -tf_getInputDim (const GstTensorFilterProperties * prop, void **private_data, - GstTensorsInfo * info) -{ - tf_data *tf; - tf = *private_data; - g_assert (*private_data); - return tf_core_getInputDim (tf->tf_private_data, info); -} - -/** - * @brief The optional callback for GstTensorFilterFramework - * @param prop: property of tensor_filter instance - * @param private_data : tensorflow plugin's private data - * @param[out] info The dimesions and types of output tensors - */ -static int -tf_getOutputDim (const GstTensorFilterProperties * prop, void **private_data, - GstTensorsInfo * info) -{ - tf_data *tf; - tf = *private_data; - g_assert (*private_data); - return tf_core_getOutputDim (tf->tf_private_data, info); -} - -/** - * @brief The optional callback for GstTensorFilterFramework - * @param[in] data The data element. - */ -static void -tf_destroyNotify (void *data) -{ - tf_core_destroyNotify (data); -} - -static gchar filter_subplugin_tensorflow[] = "tensorflow"; - -static GstTensorFilterFramework NNS_support_tensorflow = { - .name = filter_subplugin_tensorflow, - .allow_in_place = FALSE, /** @todo: support this to optimize performance later. */ - .allocate_in_invoke = TRUE, - .destroyNotify = tf_destroyNotify, - .invoke_NN = tf_run, - .getInputDimension = tf_getInputDim, - .getOutputDimension = tf_getOutputDim, - .open = tf_open, - .close = tf_close, -}; - -/** @brief Initialize this object for tensor_filter subplugin runtime register */ -void -init_filter_tf (void) -{ - nnstreamer_filter_probe (&NNS_support_tensorflow); -} - -/** @brief Destruct the subplugin */ -void -fini_filter_tf (void) -{ - nnstreamer_filter_exit (NNS_support_tensorflow.name); -} diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.cc b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc similarity index 70% rename from ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.cc rename to ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc index 78a7f25..fd65fb9 100644 --- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.cc +++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc @@ -1,4 +1,5 @@ /** + * GStreamer Tensor_Filter, Tensorflow Module * Copyright (C) 2018 Samsung Electronics Co., Ltd. All rights reserved. * Copyright (C) 2018 HyoungJoo Ahn * Copyright (C) 2018 Jijoong Moon @@ -15,17 +16,27 @@ * */ /** - * @file tensor_filter_tensorflow_core.cc + * @file tensor_filter_tensorflow.cc + * @date 02 Aug 2018 + * @brief Tensorflow module for tensor_filter gstreamer plugin + * @see http://github.com/nnsuite/nnstreamer * @author HyoungJoo Ahn * @author Jijoong Moon - * @date 08/02/2018 - * @brief connection with tensorflow libraries. + * @bug No known bugs except for NYI items * - * @bug No known bugs. + * This is the per-NN-framework plugin (tensorflow) for tensor_filter. */ #include -#include "tensor_filter_tensorflow_core.h" +#include + +#include +#include +#include +#include +#include + +#include /** * @brief Macro for debug mode. @@ -34,6 +45,58 @@ #define DBG FALSE #endif +/** + * @brief Internal data structure for tensorflow + */ +typedef struct +{ + TF_DataType type; + int rank; + std::vector < std::int64_t > dims; +} tf_tensor_info_s; + +/** + * @brief ring cache structure + */ +class TFCore +{ +public: + /** + * member functions. + */ + TFCore (const char * _model_path); + ~TFCore (); + + int init (const GstTensorFilterProperties * prop); + int loadModel (); + const char *getModelPath (); + + int getInputTensorDim (GstTensorsInfo * info); + int getOutputTensorDim (GstTensorsInfo * info); + int run (const GstTensorMemory * input, GstTensorMemory * output); + + static std::map < void *, TF_Tensor * >outputTensorMap; + +private: + + char *model_path; + + GstTensorsInfo inputTensorMeta; /**< The tensor info of input tensors from user input */ + GstTensorsInfo outputTensorMeta; /**< The tensor info of output tensors from user input */ + + std::vector < tf_tensor_info_s > input_tensor_info; /* hold information for TF */ + + TF_Graph *graph; + TF_Session *session; + + tensor_type getTensorTypeFromTF (TF_DataType tfType); + TF_DataType getTensorTypeToTF (tensor_type tType); + int validateTensor (const GstTensorsInfo * tensorInfo, int is_input); +}; + +void init_filter_tf (void) __attribute__ ((constructor)); +void fini_filter_tf (void) __attribute__ ((destructor)); + std::map TFCore::outputTensorMap; /** @@ -526,95 +589,182 @@ failed: return ret; } -void * -tf_core_new (const char *_model_path) +/** + * @brief the destroy notify method for tensorflow. it will free the output tensor + * @param[in] data : the data element destroyed at the pipeline + */ +void +tf_core_destroyNotify (void * data) { - return new TFCore (_model_path); + TF_DeleteTensor ( (TFCore::outputTensorMap.find (data))->second); + TFCore::outputTensorMap.erase (data); } /** - * @brief delete the TFCore class. - * @param tf : the class object - * @return Nothing + * @brief Free privateData and move on. */ -void -tf_core_delete (void * tf) +static void +tf_close (const GstTensorFilterProperties * prop, void **private_data) { - TFCore *c = (TFCore *) tf; - delete c; + TFCore *core = static_cast(*private_data); + + g_assert (core); + + delete core; + + *private_data = NULL; } /** - * @brief initialize the object with tf model - * @param tf : the class object - * @return 0 if OK. non-zero if error. + * @brief Load tensorflow modelfile + * @param prop: property of tensor_filter instance + * @param private_data : tensorflow plugin's private data + * @return 0 if successfully loaded. 1 if skipped (already loaded). + * -1 if the object construction is failed. + * -2 if the object initialization if failed */ -int -tf_core_init (void * tf, const GstTensorFilterProperties * prop) +static int +tf_loadModelFile (const GstTensorFilterProperties * prop, void **private_data) { - TFCore *c = (TFCore *) tf; - return c->init (prop); + TFCore *core; + const gchar *model_file; + + if (prop->num_models != 1) + return -1; + + core = static_cast(*private_data); + model_file = prop->model_files[0]; + + if (core != NULL) { + if (g_strcmp0 (model_file, core->getModelPath ()) == 0) + return 1; /* skipped */ + + tf_close (prop, private_data); + } + + core = new TFCore (model_file); + if (core == NULL) { + g_printerr ("Failed to allocate memory for filter subplugin: tensorflow\n"); + return -1; + } + + if (core->init (prop) != 0) { + *private_data = NULL; + delete core; + + g_printerr ("failed to initailize the object: tensorflow\n"); + return -2; + } + + *private_data = core; + + return 0; } /** - * @brief get model path - * @param tf : the class object - * @return model path + * @brief The open callback for GstTensorFilterFramework. Called before anything else + * @param prop: property of tensor_filter instance + * @param private_data : tensorflow plugin's private data */ -const char * -tf_core_getModelPath (void * tf) +static int +tf_open (const GstTensorFilterProperties * prop, void **private_data) { - TFCore *c = (TFCore *) tf; - return c->getModelPath (); + int status = tf_loadModelFile (prop, private_data); + + g_assert (status >= 0); /** This must be called only once */ + + return status; } /** - * @brief get the Dimension of Input Tensor of model - * @param tf the class object - * @param[out] info Structure for tensor info. - * @return 0 if OK. non-zero if error. + * @brief The mandatory callback for GstTensorFilterFramework + * @param prop: property of tensor_filter instance + * @param private_data : tensorflow plugin's private data + * @param[in] input The array of input tensors + * @param[out] output The array of output tensors */ -int -tf_core_getInputDim (void * tf, GstTensorsInfo * info) +static int +tf_run (const GstTensorFilterProperties * prop, void **private_data, + const GstTensorMemory * input, GstTensorMemory * output) { - TFCore *c = (TFCore *) tf; - return c->getInputTensorDim (info); + TFCore *core = static_cast(*private_data); + + g_assert (core); + + return core->run (input, output); } /** - * @brief get the Dimension of Output Tensor of model - * @param tf the class object - * @param[out] info Structure for tensor info. - * @return 0 if OK. non-zero if error. + * @brief The optional callback for GstTensorFilterFramework + * @param prop: property of tensor_filter instance + * @param private_data : tensorflow plugin's private data + * @param[out] info The dimesions and types of input tensors */ -int -tf_core_getOutputDim (void * tf, GstTensorsInfo * info) +static int +tf_getInputDim (const GstTensorFilterProperties * prop, void **private_data, + GstTensorsInfo * info) { - TFCore *c = (TFCore *) tf; - return c->getOutputTensorDim (info); + TFCore *core = static_cast(*private_data); + + g_assert (core); + + return core->getInputTensorDim (info); } /** - * @brief run the model - * @param tf : the class object - * @param[in] input : The array of input tensors - * @param[out] output : The array of output tensors - * @return 0 if OK. non-zero if error. + * @brief The optional callback for GstTensorFilterFramework + * @param prop: property of tensor_filter instance + * @param private_data : tensorflow plugin's private data + * @param[out] info The dimesions and types of output tensors */ -int -tf_core_run (void * tf, const GstTensorMemory * input, GstTensorMemory * output) +static int +tf_getOutputDim (const GstTensorFilterProperties * prop, void **private_data, + GstTensorsInfo * info) { - TFCore *c = (TFCore *) tf; - return c->run (input, output); + TFCore *core = static_cast(*private_data); + + g_assert (core); + + return core->getOutputTensorDim (info); } /** - * @brief the destroy notify method for tensorflow. it will free the output tensor - * @param[in] data : the data element destroyed at the pipeline + * @brief The optional callback for GstTensorFilterFramework + * @param[in] data The data element. */ -void -tf_core_destroyNotify (void * data) +static void +tf_destroyNotify (void *data) { TF_DeleteTensor ( (TFCore::outputTensorMap.find (data))->second); TFCore::outputTensorMap.erase (data); } + +static gchar filter_subplugin_tensorflow[] = "tensorflow"; + +static GstTensorFilterFramework NNS_support_tensorflow = { + .name = filter_subplugin_tensorflow, + .allow_in_place = FALSE, /** @todo: support this to optimize performance later. */ + .allocate_in_invoke = TRUE, + .run_without_model = FALSE, + .invoke_NN = tf_run, + .getInputDimension = tf_getInputDim, + .getOutputDimension = tf_getOutputDim, + .setInputDimension = NULL, + .open = tf_open, + .close = tf_close, + .destroyNotify = tf_destroyNotify, +}; + +/** @brief Initialize this object for tensor_filter subplugin runtime register */ +void +init_filter_tf (void) +{ + nnstreamer_filter_probe (&NNS_support_tensorflow); +} + +/** @brief Destruct the subplugin */ +void +fini_filter_tf (void) +{ + nnstreamer_filter_exit (NNS_support_tensorflow.name); +} diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.h b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.h deleted file mode 100644 index f71469c..0000000 --- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.h +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Copyright (C) 2018 Samsung Electronics Co., Ltd. All rights reserved. - * Copyright (C) 2018 HyoungJoo Ahn - * Copyright (C) 2018 Jijoong Moon - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - */ -/** - * @file tensor_filter_tensorflow_core.h - * @author HyoungJoo Ahn - * @author Jijoong Moon - * @date 08/02/2018 - * @brief connection with tensorflow libraries. - * - * @bug No known bugs. - */ -#ifndef TENSOR_FILTER_TENSORFLOW_CORE_H -#define TENSOR_FILTER_TENSORFLOW_CORE_H - -#include - -#include "nnstreamer_plugin_api_filter.h" - -#ifdef __cplusplus -#include -#include -#include -#include -#include - -#include - -/** - * @brief Internal data structure for tensorflow - */ -typedef struct -{ - TF_DataType type; - int rank; - std::vector < std::int64_t > dims; -} tf_tensor_info_s; - -/** - * @brief ring cache structure - */ -class TFCore -{ -public: - /** - * member functions. - */ - TFCore (const char * _model_path); - ~TFCore (); - - int init (const GstTensorFilterProperties * prop); - int loadModel (); - const char *getModelPath (); - - int getInputTensorDim (GstTensorsInfo * info); - int getOutputTensorDim (GstTensorsInfo * info); - int run (const GstTensorMemory * input, GstTensorMemory * output); - - static std::map < void *, TF_Tensor * >outputTensorMap; - -private: - - char *model_path; - - GstTensorsInfo inputTensorMeta; /**< The tensor info of input tensors from user input */ - GstTensorsInfo outputTensorMeta; /**< The tensor info of output tensors from user input */ - - std::vector < tf_tensor_info_s > input_tensor_info; /* hold information for TF */ - - TF_Graph *graph; - TF_Session *session; - - tensor_type getTensorTypeFromTF (TF_DataType tfType); - TF_DataType getTensorTypeToTF (tensor_type tType); - int validateTensor (const GstTensorsInfo * tensorInfo, int is_input); -}; - -/** - * @brief the definition of functions to be used at C files. - */ -extern "C" -{ -#endif - - void *tf_core_new (const char * _model_path); - void tf_core_delete (void * tf); - int tf_core_init (void * tf, const GstTensorFilterProperties * prop); - const char *tf_core_getModelPath (void * tf); - int tf_core_getInputDim (void * tf, GstTensorsInfo * info); - int tf_core_getOutputDim (void * tf, GstTensorsInfo * info); - int tf_core_run (void * tf, const GstTensorMemory * input, - GstTensorMemory * output); - void tf_core_destroyNotify (void * data); - -#ifdef __cplusplus -} -#endif - -#endif /* TENSOR_FILTER_TENSORFLOW_CORE_H */ diff --git a/jni/nnstreamer.mk b/jni/nnstreamer.mk index cff98d6..bcbbc48 100644 --- a/jni/nnstreamer.mk +++ b/jni/nnstreamer.mk @@ -57,8 +57,7 @@ NNSTREAMER_CAPI_SRCS := \ # filter tensorflow NNSTREAMER_FILTER_TF_SRCS := \ - $(NNSTREAMER_EXT_HOME)/tensor_filter/tensor_filter_tensorflow.c \ - $(NNSTREAMER_EXT_HOME)/tensor_filter/tensor_filter_tensorflow_core.cc + $(NNSTREAMER_EXT_HOME)/tensor_filter/tensor_filter_tensorflow.cc # filter tensorflow-lite NNSTREAMER_FILTER_TFLITE_SRCS := \ -- 2.7.4