From a74061cf2138f040e64c1d102341d0cdc121e50d Mon Sep 17 00:00:00 2001 From: Hyoung Joo Ahn Date: Wed, 5 Jun 2019 18:02:45 +0900 Subject: [PATCH] [Filter/Caffe2] initial commit for caffe2 it will be updated after packaging pytorch properly and example & test also Signed-off-by: Hyoung Joo Ahn --- .../tensor_filter/tensor_filter_caffe2.c | 194 +++++++++ .../tensor_filter/tensor_filter_caffe2_core.cc | 439 +++++++++++++++++++++ .../tensor_filter/tensor_filter_caffe2_core.h | 90 +++++ 3 files changed, 723 insertions(+) create mode 100644 ext/nnstreamer/tensor_filter/tensor_filter_caffe2.c create mode 100644 ext/nnstreamer/tensor_filter/tensor_filter_caffe2_core.cc create mode 100644 ext/nnstreamer/tensor_filter/tensor_filter_caffe2_core.h diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_caffe2.c b/ext/nnstreamer/tensor_filter/tensor_filter_caffe2.c new file mode 100644 index 0000000..199f152 --- /dev/null +++ b/ext/nnstreamer/tensor_filter/tensor_filter_caffe2.c @@ -0,0 +1,194 @@ +/** + * GStreamer Tensor_Filter, caffe2 Module + * Copyright (C) 2019 Hyoung Joo Ahn + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + */ +/** + * @file tensor_filter_caffe2.c + * @date 27 May 2019 + * @brief Caffe2 for tensor_filter gstreamer plugin + * @see http://github.com/nnsuite/nnstreamer + * @author HyoungJoo Ahn + * @bug No known bugs except for NYI items + * + * This is the per-NN-framework plugin (caffe2) for tensor_filter. + * Fill in "GstTensorFilterFramework" for tensor_filter.h/c + * + */ + +#include +#include + +#include "tensor_filter_caffe2_core.h" + +void init_filter_caffe2 (void) __attribute__ ((constructor)); +void fini_filter_caffe2 (void) __attribute__ ((destructor)); + +/** + * @brief internal data of caffe2 + */ +struct _Caffe2_data +{ + void *caffe2_private_data; +}; +typedef struct _Caffe2_data caffe2_data; + + +/** + * @brief Free privateData and move on. + */ +static void +caffe2_close (const GstTensorFilterProperties * prop, void **private_data) +{ + caffe2_data *cf2; + cf2 = *private_data; + caffe2_core_delete (cf2->caffe2_private_data); + g_free (cf2); + *private_data = NULL; +} + +/** + * @brief Load caffe2 modelfile + * @param prop property of tensor_filter instance + * @param private_data : caffe2 plugin's private data + * @return 0 if successfully loaded. 1 if skipped (already loaded). + * -1 if the object construction is failed. + * -2 if the object initialization if failed + */ +static int +caffe2_loadModelFile (const GstTensorFilterProperties * prop, + void **private_data) +{ + caffe2_data *cf2; + if (*private_data != NULL) { + /** @todo : Check the integrity of filter->data and filter->model_file, nnfw */ + cf2 = *private_data; + if (g_strcmp0 (prop->model_file, + caffe2_core_getModelPath (cf2->caffe2_private_data)) != 0 || + g_strcmp0 (prop->model_file_sub, + caffe2_core_getInitModelPath (cf2->caffe2_private_data)) != 0) { + caffe2_close (prop, private_data); + } else { + return 1; + } + } + cf2 = g_new0 (caffe2_data, 1); /** initialize cf2 Fill Zero! */ + *private_data = cf2; + cf2->caffe2_private_data = caffe2_core_new (prop->model_file, + prop->model_file_sub); + if (cf2->caffe2_private_data) { + if (caffe2_core_init (cf2->caffe2_private_data)) { + g_printerr ("failed to initialize the object: Caffe2"); + g_free (cf2); + return -2; + } + return 0; + } else { + g_printerr ("failed to create the object: Caffe2"); + g_free (cf2); + return -1; + } +} + +/** + * @brief The open callback for GstTensorFilterFramework. Called before anything else + * @param prop property of tensor_filter instance + * @param private_data : caffe2 plugin's private data + */ +static int +caffe2_open (const GstTensorFilterProperties * prop, void **private_data) +{ + int ret = caffe2_loadModelFile (prop, private_data); + g_assert (ret == 0); /** This must be called only once */ + return ret; +} + +/** + * @brief The mandatory callback for GstTensorFilterFramework + * @param prop property of tensor_filter instance + * @param private_data : caffe2 plugin's private data + * @param[in] input The array of input tensors + * @param[out] output The array of output tensors + * @return 0 if OK. non-zero if error. + */ +static int +caffe2_run (const GstTensorFilterProperties * prop, void **private_data, + const GstTensorMemory * input, GstTensorMemory * output) +{ + int retval; + caffe2_data *cf2; + cf2 = *private_data; + g_assert (*private_data); + retval = caffe2_core_run (cf2->caffe2_private_data, input, output); + g_assert (retval == 0); + return retval; +} + +/** + * @brief The optional callback for GstTensorFilterFramework + * @param prop property of tensor_filter instance + * @param private_data : caffe2 plugin's private data + * @param[out] info The dimesions and types of input tensors + */ +static int +caffe2_getInputDim (const GstTensorFilterProperties * prop, void **private_data, + GstTensorsInfo * info) +{ + caffe2_data *cf2; + cf2 = *private_data; + g_assert (*private_data); + return caffe2_core_getInputDim (cf2->caffe2_private_data, info); +} + +/** + * @brief The optional callback for GstTensorFilterFramework + * @param prop property of tensor_filter instance + * @param private_data : caffe2 plugin's private data + * @param[out] info The dimesions and types of output tensors + */ +static int +caffe2_getOutputDim (const GstTensorFilterProperties * prop, + void **private_data, GstTensorsInfo * info) +{ + caffe2_data *cf2; + cf2 = *private_data; + g_assert (*private_data); + return caffe2_core_getOutputDim (cf2->caffe2_private_data, info); +} + +static gchar filter_subplugin_caffe2[] = "caffe2"; + +static GstTensorFilterFramework NNS_support_caffe2 = { + .name = filter_subplugin_caffe2, + .allow_in_place = FALSE, /** @todo: support this to optimize performance later. */ + .allocate_in_invoke = TRUE, + .invoke_NN = caffe2_run, + .getInputDimension = caffe2_getInputDim, + .getOutputDimension = caffe2_getOutputDim, + .open = caffe2_open, + .close = caffe2_close, +}; + +/** @brief Initialize this object for tensor_filter subplugin runtime register */ +void +init_filter_caffe2 (void) +{ + nnstreamer_filter_probe (&NNS_support_caffe2); +} + +/** @brief Destruct the subplugin */ +void +fini_filter_caffe2 (void) +{ + nnstreamer_filter_exit (NNS_support_caffe2.name); +} diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_caffe2_core.cc b/ext/nnstreamer/tensor_filter/tensor_filter_caffe2_core.cc new file mode 100644 index 0000000..995f811 --- /dev/null +++ b/ext/nnstreamer/tensor_filter/tensor_filter_caffe2_core.cc @@ -0,0 +1,439 @@ +/** + * Copyright (C) 2019 Samsung Electronics Co., Ltd. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + */ +/** + * @file tensor_filter_caffe2_core.cc + * @author HyoungJoo Ahn + * @date 31/5/2019 + * @brief connection with caffe2 libraries. + * + * @bug No known bugs. + */ + +#include +#include + +#include +#include "tensor_filter_caffe2_core.h" + +/** + * @brief Macro for debug mode. + */ +#ifndef DBG +#define DBG FALSE +#endif + +std::map Caffe2Core::inputTensorMap; + +/** + * @brief Caffe2Core creator + * @param _model_path : the logical path to '{model_name}.tffile' file + * @note the model of _model_path will be loaded simultaneously + * @return Nothing + */ +Caffe2Core::Caffe2Core (const char * _model_path, const char *_model_path_sub) +{ + pred_model_path = _model_path; + init_model_path = _model_path_sub; + + gst_tensors_info_init (&inputTensorMeta); + gst_tensors_info_init (&outputTensorMeta); +} + +/** + * @brief Caffe2Core Destructor + * @return Nothing + */ +Caffe2Core::~Caffe2Core () +{ + gst_tensors_info_free (&inputTensorMeta); + gst_tensors_info_free (&outputTensorMeta); +} + +/** + * @brief initialize the object with caffe2 model + * @return 0 if OK. non-zero if error. + * -1 if the model is not loaded. + * -2 if the initialization of input tensor is failed. + * -3 if the initialization of output tensor is failed. + */ +int +Caffe2Core::init (const GstTensorFilterProperties * prop) +{ + if (loadModels ()) { + g_critical ("Failed to load model\n"); + return -1; + } + + gst_tensors_info_copy (&inputTensorMeta, &prop->input_meta); + gst_tensors_info_copy (&outputTensorMeta, &prop->output_meta); + + if (initInputTensor ()) { + g_critical ("Failed to initialize input tensor\n"); + return -2; + } + return 0; +} + +#define initializeTensor(type)\ +do {\ + ReinitializeTensor (\ + inputTensor,\ + {\ + inputTensorMeta.info[i].dimension[0],\ + inputTensorMeta.info[i].dimension[1],\ + inputTensorMeta.info[i].dimension[2],\ + inputTensorMeta.info[i].dimension[3]\ + },\ + at::dtype ().device (CPU)\ + );\ +} while (0); + +/** + * @brief initialize the input tensor + */ +int +Caffe2Core::initInputTensor () +{ + int i = 0; + inputTensorMap.clear (); + for (; i < inputTensorMeta.num_tensors; i++) { + Tensor *inputTensor = workSpace.CreateBlob (inputTensorMeta.info[i].name) + ->GetMutable (); + + switch (inputTensorMeta.info[i].type){ + case _NNS_INT32: + initializeTensor (int32_t); + break; + case _NNS_UINT32: + g_critical ("invalid data type is used"); + return -1; + case _NNS_INT16: + initializeTensor (int16_t); + break; + case _NNS_UINT16: + initializeTensor (uint16_t); + break; + case _NNS_INT8: + initializeTensor (int8_t); + break; + case _NNS_UINT8: + initializeTensor (uint8_t); + break; + case _NNS_FLOAT64: + initializeTensor (double); + break; + case _NNS_FLOAT32: + initializeTensor (float); + break; + case _NNS_INT64: + initializeTensor (int64_t); + break; + case _NNS_UINT64: + g_critical ("invalid data type is used"); + return -1; + default: + g_critical ("invalid data type is used"); + return -1; + } + + inputTensorMap.insert ( + std::make_pair (inputTensorMeta.info[i].name, inputTensor) + ); + } + return 0; +} + +/** + * @brief get the model path + * @return the model path. + */ +const char * +Caffe2Core::getPredModelPath () +{ + return pred_model_path; +} + +/** + * @brief get the model path + * @return the model path. + */ +const char * +Caffe2Core::getInitModelPath () +{ + return init_model_path; +} + +/** + * @brief load the caffe2 model + * @note the model will be loaded + * @return 0 if OK. non-zero if error. + */ +int +Caffe2Core::loadModels () +{ +#if (DBG) + gint64 start_time = g_get_real_time (); +#endif + if (!g_file_test (init_model_path, G_FILE_TEST_IS_REGULAR)) { + g_critical ("the file of init_model_path is not valid\n"); + return -1; + } + if (!g_file_test (pred_model_path, G_FILE_TEST_IS_REGULAR)) { + g_critical ("the file of pred_model_path is not valid\n"); + return -1; + } + + CAFFE_ENFORCE (ReadProtoFromFile (init_model_path, &initNet)); + CAFFE_ENFORCE (ReadProtoFromFile (pred_model_path, &predictNet)); + + CAFFE_ENFORCE (workSpace.RunNetOnce (initNet)); + CAFFE_ENFORCE (workSpace.CreateNet (predictNet)); +#if (DBG) + gint64 stop_time = g_get_real_time (); + g_message ("Model is loaded: %" G_GINT64_FORMAT, (stop_time - start_time)); +#endif + return 0; +} + +/** + * @brief return the Dimension of Input Tensor. + * @param[out] info Structure for tensor info. + * @todo return whole array rather than index 0 + * @return 0 if OK. non-zero if error. + */ +int +Caffe2Core::getInputTensorDim (GstTensorsInfo * info) +{ + gst_tensors_info_copy (info, &inputTensorMeta); + return 0; +} + +/** + * @brief return the Dimension of Tensor. + * @param[out] info Structure for tensor info. + * @todo return whole array rather than index 0 + * @return 0 if OK. non-zero if error. + */ +int +Caffe2Core::getOutputTensorDim (GstTensorsInfo * info) +{ + gst_tensors_info_copy (info, &outputTensorMeta); + return 0; +} + +/** + * @brief run the model with the input. + * @param[in] input : The array of input tensors + * @param[out] output : The array of output tensors + * @return 0 if OK. non-zero if error. + */ +int +Caffe2Core::run (const GstTensorMemory * input, GstTensorMemory * output) +{ + int i; +#if (DBG) + gint64 start_time = g_get_real_time (); +#endif + for (i = 0; i < inputTensorMeta.num_tensors; i++){ + Tensor *inputTensor = inputTensorMap. + find(inputTensorMeta.info[i].name)->second; + inputTensor->ShareExternalPointer ((float*) input[i].data); + switch (outputTensorMeta.info[i].type){ + case _NNS_INT32: + inputTensor->ShareExternalPointer ((int32_t*) input[i].data); + break; + case _NNS_UINT32: + g_critical ("invalid data type is used"); + return -1; + case _NNS_INT16: + inputTensor->ShareExternalPointer ((int16_t*) input[i].data); + break; + case _NNS_UINT16: + inputTensor->ShareExternalPointer ((uint16_t*) input[i].data); + break; + case _NNS_INT8: + inputTensor->ShareExternalPointer ((int8_t*) input[i].data); + break; + case _NNS_UINT8: + inputTensor->ShareExternalPointer ((uint8_t*) input[i].data); + break; + case _NNS_FLOAT64: + inputTensor->ShareExternalPointer ((double*) input[i].data); + break; + case _NNS_FLOAT32: + inputTensor->ShareExternalPointer ((float*) input[i].data); + break; + case _NNS_INT64: + inputTensor->ShareExternalPointer ((int64_t*) input[i].data); + break; + case _NNS_UINT64: + g_critical ("invalid data type is used"); + return -1; + default: + g_critical ("invalid data type is used"); + return -1; + } + } + + workSpace.RunNet (predictNet.name ()); + + for (i = 0; i < outputTensorMeta.num_tensors; i++) { + const auto& out = workSpace.GetBlob (outputTensorMeta.info[i].name) + ->Get (); + switch (outputTensorMeta.info[i].type){ + case _NNS_INT32: + output[i].data = out.data(); + break; + case _NNS_UINT32: + g_critical ("invalid data type is used"); + return -1; + case _NNS_INT16: + output[i].data = out.data(); + break; + case _NNS_UINT16: + output[i].data = out.data(); + break; + case _NNS_INT8: + output[i].data = out.data(); + break; + case _NNS_UINT8: + output[i].data = out.data(); + break; + case _NNS_FLOAT64: + output[i].data = out.data(); + break; + case _NNS_FLOAT32: + output[i].data = out.data(); + break; + case _NNS_INT64: + output[i].data = out.data(); + break; + case _NNS_UINT64: + g_critical ("invalid data type is used"); + return -1; + default: + g_critical ("invalid data type is used"); + return -1; + } + } + +#if (DBG) + gint64 stop_time = g_get_real_time (); + g_message ("Run() is finished: %" G_GINT64_FORMAT, + (stop_time - start_time)); +#endif + + return 0; +} + +/** + * @brief call the creator of Caffe2Core class. + * @param _model_path : the logical path to '{model_name}.tffile' file + * @return Caffe2Core class + */ +void * +caffe2_core_new (const char *_model_path, const char *_model_path_sub) +{ + return new Caffe2Core (_model_path, _model_path_sub); +} + +/** + * @brief delete the Caffe2Core class. + * @param caffe2 : the class object + * @return Nothing + */ +void +caffe2_core_delete (void * caffe2) +{ + Caffe2Core *c = (Caffe2Core *) caffe2; + delete c; +} + +/** + * @brief initialize the object with caffe2 model + * @param caffe2 : the class object + * @return 0 if OK. non-zero if error. + */ +int +caffe2_core_init (void * caffe2, const GstTensorFilterProperties * prop) +{ + Caffe2Core *c = (Caffe2Core *) caffe2; + return c->init (prop); +} + +/** + * @brief get the model path + * @param caffe2 : the class object + * @return the model path. + */ +const char * +caffe2_core_getInitModelPath (void * caffe2) +{ + Caffe2Core *c = (Caffe2Core *) caffe2; + return c->getInitModelPath (); +} + +/** + * @brief get the model path + * @param caffe2 : the class object + * @return the model path. + */ +const char * +caffe2_core_getPredModelPath (void * caffe2) +{ + Caffe2Core *c = (Caffe2Core *) caffe2; + return c->getPredModelPath (); +} + +/** + * @brief get the Dimension of Input Tensor of model + * @param caffe2 : the class object + * @param[out] info Structure for tensor info. + * @return 0 if OK. non-zero if error. + */ +int +caffe2_core_getInputDim (void * caffe2, GstTensorsInfo * info) +{ + Caffe2Core *c = (Caffe2Core *) caffe2; + return c->getInputTensorDim (info); +} + +/** + * @brief get the Dimension of Output Tensor of model + * @param caffe2 : the class object + * @param[out] info Structure for tensor info. + * @return 0 if OK. non-zero if error. + */ +int +caffe2_core_getOutputDim (void * caffe2, GstTensorsInfo * info) +{ + Caffe2Core *c = (Caffe2Core *) caffe2; + return c->getOutputTensorDim (info); +} + +/** + * @brief run the model + * @param caffe2 : the class object + * @param[in] input : The array of input tensors + * @param[out] output : The array of output tensors + * @return 0 if OK. non-zero if error. + */ +int +caffe2_core_run (void * caffe2, const GstTensorMemory * input, + GstTensorMemory * output) +{ + Caffe2Core *c = (Caffe2Core *) caffe2; + return c->run (input, output); +} diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_caffe2_core.h b/ext/nnstreamer/tensor_filter/tensor_filter_caffe2_core.h new file mode 100644 index 0000000..18c0dd2 --- /dev/null +++ b/ext/nnstreamer/tensor_filter/tensor_filter_caffe2_core.h @@ -0,0 +1,90 @@ +/** + * Copyright (C) 2019 Samsung Electronics Co., Ltd. All rights reserved. + * Copyright (C) 2019 HyoungJoo Ahn + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + */ +/** + * @file tensor_filter_caffe2_core.h + * @author HyoungJoo Ahn + * @date 05/31/2019 + * @brief connection with caffe2 libraries. + * + * @bug No known bugs. + */ +#ifndef TENSOR_FILTER_CAFFE2_CORE_H +#define TENSOR_FILTER_CAFFE2_CORE_H + +#include "nnstreamer_plugin_api_filter.h" + +#ifdef __cplusplus +#include + +#include "caffe2/core/workspace.h" +#include "caffe2/core/init.h" + +using namespace caffe2; + +/** + * @brief ring cache structure + */ +class Caffe2Core +{ +public: + Caffe2Core (const char * _model_path, const char * _model_path_sub); + ~Caffe2Core (); + + int init (const GstTensorFilterProperties * prop); + int loadModels (); + const char* getPredModelPath (); + const char* getInitModelPath (); + int getInputTensorDim (GstTensorsInfo * info); + int getOutputTensorDim (GstTensorsInfo * info); + int run (const GstTensorMemory * input, GstTensorMemory * output); + +private: + + const char *init_model_path; + const char *pred_model_path; + + GstTensorsInfo inputTensorMeta; /**< The tensor info of input tensors */ + GstTensorsInfo outputTensorMeta; /**< The tensor info of output tensors */ + + Workspace workSpace; + NetDef initNet, predictNet; + static std::map inputTensorMap; + + int initInputTensor (); +}; + +/** + * @brief the definition of functions to be used at C files. + */ +extern "C" +{ +#endif + + void *caffe2_core_new (const char *_model_path, const char *_model_path_sub); + void caffe2_core_delete (void * caffe2); + int caffe2_core_init (void * caffe2, const GstTensorFilterProperties * prop); + const char *caffe2_core_getInitModelPath (void * caffe2); + const char *caffe2_core_getPredModelPath (void * caffe2); + int caffe2_core_getInputDim (void * caffe2, GstTensorsInfo * info); + int caffe2_core_getOutputDim (void * caffe2, GstTensorsInfo * info); + int caffe2_core_run (void * caffe2, const GstTensorMemory * input, + GstTensorMemory * output); + +#ifdef __cplusplus +} +#endif + +#endif /* TENSOR_FILTER_CAFFE2_CORE_H */ -- 2.7.4