From 7c26946608a00e872846a4184d73223d29acf767 Mon Sep 17 00:00:00 2001 From: Yongjoo Ahn Date: Tue, 24 Mar 2020 17:48:03 +0900 Subject: [PATCH] [Decoder/protobuf] Add protobuf tensor_decoder sub-plugin - Add tensor_decoder sub-plugin "protobuf" to support Protocol Buffers Signed-off-by: Yongjoo Ahn --- ext/nnstreamer/include/nnstreamer.proto | 33 ++++ ext/nnstreamer/meson.build | 12 ++ ext/nnstreamer/tensor_decoder/meson.build | 25 +++ .../tensor_decoder/tensordec-protobuf.cc | 176 +++++++++++++++++++++ gst/nnstreamer/include/tensor_typedef.h | 9 ++ meson.build | 6 + meson_options.txt | 1 + 7 files changed, 262 insertions(+) create mode 100644 ext/nnstreamer/include/nnstreamer.proto create mode 100644 ext/nnstreamer/tensor_decoder/tensordec-protobuf.cc diff --git a/ext/nnstreamer/include/nnstreamer.proto b/ext/nnstreamer/include/nnstreamer.proto new file mode 100644 index 0000000..034e5a9 --- /dev/null +++ b/ext/nnstreamer/include/nnstreamer.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package NNStreamer; + +message Tensor { + string name = 1; + enum Tensor_type { + NNS_INT32 = 0; + NNS_UINT32 = 1; + NNS_INT16 = 2; + NNS_UINT16 = 3; + NNS_INT8 = 4; + NNS_UINT8 = 5; + NNS_FLOAT64 = 6; + NNS_FLOAT32 = 7; + NNS_INT64 = 8; + NNS_UINT64 = 9; + } + Tensor_type type = 2; + repeated uint32 dimension = 3; + bytes data = 4; +} + +message Tensors { + uint32 num_tensor = 1; + // frame_rate := rate_n / rate_d and it may be 0 + message frame_rate { + int32 rate_n = 1; + int32 rate_d = 2; + } + frame_rate fr = 2; + repeated Tensor tensor = 3; +} diff --git a/ext/nnstreamer/meson.build b/ext/nnstreamer/meson.build index 56a761a..b3e9467 100644 --- a/ext/nnstreamer/meson.build +++ b/ext/nnstreamer/meson.build @@ -1,3 +1,15 @@ +if protobuf_support_is_available + pb_gen = generator(pb_comp, + output: ['@BASENAME@.pb.h', '@BASENAME@.pb.cc'], + arguments : [ + '--proto_path=@CURRENT_SOURCE_DIR@/include', + '--cpp_out=@BUILD_DIR@', + '@INPUT@' + ] + ) + pb_gen_src = pb_gen.process('./include/nnstreamer.proto') +endif + subdir('tensor_decoder') subdir('tensor_filter') subdir('tensor_source') diff --git a/ext/nnstreamer/tensor_decoder/meson.build b/ext/nnstreamer/tensor_decoder/meson.build index 4009371..c6ffbc1 100644 --- a/ext/nnstreamer/tensor_decoder/meson.build +++ b/ext/nnstreamer/tensor_decoder/meson.build @@ -117,3 +117,28 @@ static_library('nnstreamer_decoder_image_segment', install: true, install_dir: nnstreamer_libdir ) + +# protobuf decoder +if protobuf_support_is_available + decoder_sub_protobuf_sources = [ + 'tensordec-protobuf.cc', + ] + + nnstreamer_decoder_protobuf_sources = [pb_gen_src] + foreach s : decoder_sub_protobuf_sources + nnstreamer_decoder_protobuf_sources += join_paths(meson.current_source_dir(), s) + endforeach + + shared_library('nnstreamer_decoder_protobuf', + nnstreamer_decoder_protobuf_sources, + dependencies: [nnstreamer_dep, glib_dep, gst_dep, protobuf_dep, thread_dep], + install: true, + install_dir: decoder_subplugin_install_dir + ) + static_library('nnstreamer_decoder_protobuf', + nnstreamer_decoder_protobuf_sources, + dependencies: [nnstreamer_dep, glib_dep, gst_dep, protobuf_dep, thread_dep], + install: true, + install_dir: nnstreamer_libdir + ) +endif diff --git a/ext/nnstreamer/tensor_decoder/tensordec-protobuf.cc b/ext/nnstreamer/tensor_decoder/tensordec-protobuf.cc new file mode 100644 index 0000000..dfbfd52 --- /dev/null +++ b/ext/nnstreamer/tensor_decoder/tensordec-protobuf.cc @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: LGPL-2.1-only */ +/** + * GStreamer / NNStreamer tensor_decoder subplugin, "protobuf" + * Copyright (C) 2020 Yongjoo Ahn + */ +/** + * @file tensordec-protobuf.cc + * @date 25 Mar 2020 + * @brief NNStreamer tensor-decoder subplugin, "protobuf", + * which converts tensor or tensors to Protocol Buffers. + * @see https://github.com/nnstreamer/nnstreamer + * @author Yongjoo Ahn + * @bug No known bugs except for NYI items + * + */ + +#include +#include +#include +#include +#include +#include + +#include "nnstreamer.pb.h" /* Generated by `protoc` */ + +void init_pb (void) __attribute__ ((constructor)); +void fini_pb (void) __attribute__ ((destructor)); + +/** + * @brief tensordec-plugin's GstTensorDecoderDef callback + */ +static int +pb_init (void **pdata) +{ + /** + * Verify that the version of the library we linked is + * compatibile with the headers. + */ + GOOGLE_PROTOBUF_VERIFY_VERSION; + *pdata = NULL; /* no private data are needed for this sub-plugin */ + return TRUE; +} + +/** + * @brief tensordec-plugin's GstTensorDecoderDef callback + */ +static void +pb_exit (void **pdata) +{ + google::protobuf::ShutdownProtobufLibrary (); + return; +} + +/** + * @brief tensordec-plugin's GstTensorDecoderDef callback + */ +static int +pb_setOption (void **pdata, int opNum, const char *param) +{ + return TRUE; +} + +/** + * @brief tensordec-plugin's GstTensorDecoderDef callback + */ +static GstCaps * +pb_getOutCaps (void **pdata, const GstTensorsConfig * config) +{ + return gst_caps_from_string (GST_PROTOBUF_TENSOR_CAP_DEFAULT); +} + +/** @brief tensordec-plugin's GstTensorDecoderDef callback */ +static GstFlowReturn +pb_decode (void **pdata, const GstTensorsConfig * config, + const GstTensorMemory * input, GstBuffer * outbuf) +{ + GstMapInfo out_info; + GstMemory *out_mem; + size_t size, outbuf_size; + + NNStreamer::Tensors tensors; + NNStreamer::Tensors::frame_rate *fr = NULL; + + const unsigned int num_tensors = config->info.num_tensors; + g_assert (num_tensors > 0); + tensors.set_num_tensor (num_tensors); + + fr = tensors.mutable_fr (); + if (!fr) { + nns_loge ("Failed to get pointer of tensors / tensordec-protobuf"); + return GST_FLOW_ERROR; + } + + fr->set_rate_n (config->rate_n); + fr->set_rate_d (config->rate_d); + + for (unsigned int i = 0; i < num_tensors; ++i) { + NNStreamer::Tensor *tensor = tensors.add_tensor (); + gchar *name = NULL; + + name = config->info.info[i].name; + if (name == NULL) { + tensor->set_name ("Anonymous"); + } else { + tensor->set_name (name); + } + + tensor->set_type ((NNStreamer::Tensor::Tensor_type) + config->info.info[i].type); + + for (int j = 0; j < NNS_TENSOR_RANK_LIMIT; ++j) { + tensor->add_dimension (config->info.info[i].dimension[j]); + } + + tensor->set_data (input[i].data, (int) input[i].size); + } + + g_assert (outbuf); + + size = tensors.ByteSizeLong (); + outbuf_size = gst_buffer_get_size (outbuf); + + if (outbuf_size == 0) { + out_mem = gst_allocator_alloc (NULL, size, NULL); + } else { + if (outbuf_size < size) { + gst_buffer_set_size (outbuf, size); + } + out_mem = gst_buffer_get_all_memory (outbuf); + } + + if (FALSE == gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) { + nns_loge ("Cannot map aoutput memory / tensordec-protobuf"); + return GST_FLOW_ERROR; + } + + tensors.SerializeToArray (out_info.data, size); + + gst_memory_unmap (out_mem, &out_info); + + if (gst_buffer_get_size (outbuf) == 0) { + gst_buffer_append_memory (outbuf, out_mem); + } + + return GST_FLOW_OK; +} + +static gchar decoder_subplugin_protobuf[] = "protobuf"; + +/** + * @brief protocol buffers tensordec-plugin GstTensorDecoderDef instance + */ +static GstTensorDecoderDef protobuf = { + .modename = decoder_subplugin_protobuf, + .init = pb_init, + .exit = pb_exit, + .setOption = pb_setOption, + .getOutCaps = pb_getOutCaps, + .decode = pb_decode +}; + +/** + * @brief Initialize this object for tensordec-plugin + */ +void +init_pb (void) +{ + nnstreamer_decoder_probe (&protobuf); +} + +/** @brief Destruct this object for tensordec-plugin */ +void +fini_pb (void) +{ + nnstreamer_decoder_exit (protobuf.modename); +} diff --git a/gst/nnstreamer/include/tensor_typedef.h b/gst/nnstreamer/include/tensor_typedef.h index 5087370..96bab2c 100644 --- a/gst/nnstreamer/include/tensor_typedef.h +++ b/gst/nnstreamer/include/tensor_typedef.h @@ -80,6 +80,15 @@ */ /** + * @brief Default static capibility for Protocol Buffers + * protobuf converter will convert this capability to other/tensor(s) + * @todo Move this definition to proper header file + */ +#define GST_PROTOBUF_TENSOR_CAP_DEFAULT \ + "other/protobuf-tensor, " \ + "framerate = " GST_TENSOR_RATE_RANGE + +/** * @brief Possible data element types of other/tensor. */ typedef enum _nns_tensor_type diff --git a/meson.build b/meson.build index 241aa7d..5a7eea7 100644 --- a/meson.build +++ b/meson.build @@ -129,6 +129,9 @@ thread_dep = dependency('threads') # pthread for tensorflow-lite # Protobuf protobuf_dep = dependency('protobuf', version: '>= 3.6.1', required: false) +# Protobuf compiler +pb_comp = find_program('protoc', required: get_option('protobuf-support')) + json_glib_dep = dependency('json-glib-1.0', required: get_option('nnfw-runtime-support')) #orc @@ -223,6 +226,9 @@ features = { 'extra_deps': [ snpe_dep ], 'project_args': { 'ENABLE_SNPE' : 1 }, 'extra_args': { 'SNPE_ROOT': SNPE_ROOT } + }, + 'protobuf-support': { + 'extra_deps': [ pb_comp ] } } diff --git a/meson_options.txt b/meson_options.txt index 544ae68..6b3a4df 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -15,6 +15,7 @@ option('tflite-nnapi-delegation', type: 'feature', value: 'auto') option('armnn-support', type: 'feature', value: 'auto') option('orcc-support', type: 'feature', value: 'auto') option('snpe-support', type: 'feature', value: 'auto') +option('protobuf-support', type: 'feature', value: 'auto') # booleans & other options option('enable-test', type: 'boolean', value: true) -- 2.7.4