[Decoder/protobuf] Add protobuf tensor_decoder sub-plugin

- Add tensor_decoder sub-plugin "protobuf" to support Protocol Buffers

Signed-off-by: Yongjoo Ahn <yongjoo1.ahn@samsung.com>

diff --git a/ext/nnstreamer/include/nnstreamer.proto b/ext/nnstreamer/include/nnstreamer.proto
new file mode 100644 (file)
index 0000000..034e5a9
--- /dev/null
+++ b/ext/nnstreamer/include/nnstreamer.proto
@@ -0,0 +1,33 @@
+syntax = "proto3";
+
+package NNStreamer;
+
+message Tensor {
+  string name = 1;
+  enum Tensor_type {
+    NNS_INT32 = 0;
+    NNS_UINT32 = 1;
+    NNS_INT16 = 2;
+    NNS_UINT16 = 3;
+    NNS_INT8 = 4;
+    NNS_UINT8 = 5;
+    NNS_FLOAT64 = 6;
+    NNS_FLOAT32 = 7;
+    NNS_INT64 = 8;
+    NNS_UINT64 = 9;
+  }
+  Tensor_type type = 2;
+  repeated uint32 dimension = 3;
+  bytes data = 4;
+}
+
+message Tensors {
+  uint32 num_tensor = 1;
+  // frame_rate := rate_n / rate_d and it may be 0
+  message frame_rate {
+    int32 rate_n = 1;
+    int32 rate_d = 2;
+  }
+  frame_rate fr = 2;
+  repeated Tensor tensor = 3;
+}

diff --git a/ext/nnstreamer/meson.build b/ext/nnstreamer/meson.build
index 56a761a..b3e9467 100644 (file)
--- a/ext/nnstreamer/meson.build
+++ b/ext/nnstreamer/meson.build
@@ -1,3 +1,15 @@
+if protobuf_support_is_available
+  pb_gen = generator(pb_comp,
+      output: ['@BASENAME@.pb.h', '@BASENAME@.pb.cc'],
+      arguments : [
+        '--proto_path=@CURRENT_SOURCE_DIR@/include',
+        '--cpp_out=@BUILD_DIR@',
+        '@INPUT@'
+      ]
+  )
+  pb_gen_src = pb_gen.process('./include/nnstreamer.proto')
+endif
+
 subdir('tensor_decoder')
 subdir('tensor_filter')
 subdir('tensor_source')

diff --git a/ext/nnstreamer/tensor_decoder/meson.build b/ext/nnstreamer/tensor_decoder/meson.build
index 4009371..c6ffbc1 100644 (file)
--- a/ext/nnstreamer/tensor_decoder/meson.build
+++ b/ext/nnstreamer/tensor_decoder/meson.build
@@ -117,3 +117,28 @@ static_library('nnstreamer_decoder_image_segment',
   install: true,
   install_dir: nnstreamer_libdir
 )
+
+# protobuf decoder
+if protobuf_support_is_available
+  decoder_sub_protobuf_sources = [
+    'tensordec-protobuf.cc',
+  ]
+
+  nnstreamer_decoder_protobuf_sources = [pb_gen_src]
+  foreach s : decoder_sub_protobuf_sources
+    nnstreamer_decoder_protobuf_sources += join_paths(meson.current_source_dir(), s)
+  endforeach
+  
+  shared_library('nnstreamer_decoder_protobuf',
+    nnstreamer_decoder_protobuf_sources,
+    dependencies: [nnstreamer_dep, glib_dep, gst_dep, protobuf_dep, thread_dep],
+    install: true,
+    install_dir: decoder_subplugin_install_dir
+  )
+  static_library('nnstreamer_decoder_protobuf',
+    nnstreamer_decoder_protobuf_sources,
+    dependencies: [nnstreamer_dep, glib_dep, gst_dep, protobuf_dep, thread_dep],
+    install: true,
+    install_dir: nnstreamer_libdir
+  )
+endif

diff --git a/ext/nnstreamer/tensor_decoder/tensordec-protobuf.cc b/ext/nnstreamer/tensor_decoder/tensordec-protobuf.cc
new file mode 100644 (file)
index 0000000..dfbfd52
--- /dev/null
+++ b/ext/nnstreamer/tensor_decoder/tensordec-protobuf.cc
@@ -0,0 +1,176 @@
+/* SPDX-License-Identifier: LGPL-2.1-only */
+/**
+ * GStreamer / NNStreamer tensor_decoder subplugin, "protobuf"
+ * Copyright (C) 2020 Yongjoo Ahn <yongjoo1.ahn@samsung.com>
+ */
+/**
+ * @file        tensordec-protobuf.cc
+ * @date        25 Mar 2020
+ * @brief       NNStreamer tensor-decoder subplugin, "protobuf",
+ *              which converts tensor or tensors to Protocol Buffers.
+ * @see         https://github.com/nnstreamer/nnstreamer
+ * @author      Yongjoo Ahn <yongjoo1.ahn@samsung.com>
+ * @bug         No known bugs except for NYI items
+ *
+ */
+
+#include <glib.h>
+#include <gst/gstinfo.h>
+#include <tensor_typedef.h>
+#include <nnstreamer_plugin_api_decoder.h>
+#include <nnstreamer_plugin_api.h>
+#include <nnstreamer_log.h>
+
+#include "nnstreamer.pb.h"    /* Generated by `protoc` */
+
+void init_pb (void) __attribute__ ((constructor));
+void fini_pb (void) __attribute__ ((destructor));
+
+/**
+ * @brief tensordec-plugin's GstTensorDecoderDef callback
+ */
+static int
+pb_init (void **pdata)
+{
+  /**
+   * Verify that the version of the library we linked is
+   * compatibile with the headers.
+   */
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
+  *pdata = NULL; /* no private data are needed for this sub-plugin */
+  return TRUE;
+}
+
+/** 
+ * @brief tensordec-plugin's GstTensorDecoderDef callback 
+ */
+static void
+pb_exit (void **pdata)
+{
+  google::protobuf::ShutdownProtobufLibrary ();
+  return;
+}
+
+/**
+ * @brief tensordec-plugin's GstTensorDecoderDef callback
+ */
+static int
+pb_setOption (void **pdata, int opNum, const char *param)
+{
+  return TRUE;
+}
+
+/**
+ * @brief tensordec-plugin's GstTensorDecoderDef callback
+ */
+static GstCaps *
+pb_getOutCaps (void **pdata, const GstTensorsConfig * config)
+{
+  return gst_caps_from_string (GST_PROTOBUF_TENSOR_CAP_DEFAULT);
+}
+
+/** @brief tensordec-plugin's GstTensorDecoderDef callback */
+static GstFlowReturn
+pb_decode (void **pdata, const GstTensorsConfig * config,
+    const GstTensorMemory * input, GstBuffer * outbuf)
+{
+  GstMapInfo out_info;
+  GstMemory *out_mem;
+  size_t size, outbuf_size;
+
+  NNStreamer::Tensors tensors;
+  NNStreamer::Tensors::frame_rate *fr = NULL;
+
+  const unsigned int num_tensors = config->info.num_tensors;
+  g_assert (num_tensors > 0);
+  tensors.set_num_tensor (num_tensors);
+
+  fr = tensors.mutable_fr ();
+  if (!fr) {
+    nns_loge ("Failed to get pointer of tensors / tensordec-protobuf");
+    return GST_FLOW_ERROR;
+  }
+
+  fr->set_rate_n (config->rate_n);
+  fr->set_rate_d (config->rate_d);
+  
+  for (unsigned int i = 0; i < num_tensors; ++i) {
+    NNStreamer::Tensor *tensor = tensors.add_tensor ();
+    gchar *name = NULL;
+
+    name = config->info.info[i].name;
+    if (name == NULL) {
+      tensor->set_name ("Anonymous");
+    } else {
+      tensor->set_name (name);
+    }
+
+    tensor->set_type ((NNStreamer::Tensor::Tensor_type)
+                          config->info.info[i].type);
+
+    for (int j = 0; j < NNS_TENSOR_RANK_LIMIT; ++j) {
+      tensor->add_dimension (config->info.info[i].dimension[j]);
+    }
+
+    tensor->set_data (input[i].data, (int) input[i].size);
+  }
+
+  g_assert (outbuf);
+
+  size = tensors.ByteSizeLong ();
+  outbuf_size = gst_buffer_get_size (outbuf);
+  
+  if (outbuf_size == 0) {
+    out_mem = gst_allocator_alloc (NULL, size, NULL);
+  } else {
+    if (outbuf_size < size) {
+      gst_buffer_set_size (outbuf, size);
+    }
+    out_mem = gst_buffer_get_all_memory (outbuf);
+  }
+
+  if (FALSE == gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) {
+    nns_loge ("Cannot map aoutput memory / tensordec-protobuf");
+    return GST_FLOW_ERROR;
+  }
+
+  tensors.SerializeToArray (out_info.data, size);
+
+  gst_memory_unmap (out_mem, &out_info);
+
+  if (gst_buffer_get_size (outbuf) == 0) {
+    gst_buffer_append_memory (outbuf, out_mem);
+  }
+  
+  return GST_FLOW_OK;
+}
+
+static gchar decoder_subplugin_protobuf[] = "protobuf";
+
+/**
+ * @brief protocol buffers tensordec-plugin GstTensorDecoderDef instance
+ */
+static GstTensorDecoderDef protobuf = {
+  .modename = decoder_subplugin_protobuf,
+  .init = pb_init,
+  .exit = pb_exit,
+  .setOption = pb_setOption,
+  .getOutCaps = pb_getOutCaps,
+  .decode = pb_decode
+};
+
+/**
+ * @brief Initialize this object for tensordec-plugin
+ */
+void
+init_pb (void)
+{
+  nnstreamer_decoder_probe (&protobuf);
+}
+
+/** @brief Destruct this object for tensordec-plugin */
+void
+fini_pb (void)
+{
+  nnstreamer_decoder_exit (protobuf.modename);
+}

diff --git a/gst/nnstreamer/include/tensor_typedef.h b/gst/nnstreamer/include/tensor_typedef.h
index 5087370..96bab2c 100644 (file)
--- a/gst/nnstreamer/include/tensor_typedef.h
+++ b/gst/nnstreamer/include/tensor_typedef.h
@@ -80,6 +80,15 @@
      */
 
 /**
+ * @brief Default static capibility for Protocol Buffers
+ * protobuf converter will convert this capability to other/tensor(s)
+ * @todo Move this definition to proper header file
+ */
+#define GST_PROTOBUF_TENSOR_CAP_DEFAULT \
+    "other/protobuf-tensor, " \
+    "framerate = " GST_TENSOR_RATE_RANGE
+
+/**
  * @brief Possible data element types of other/tensor.
  */
 typedef enum _nns_tensor_type

diff --git a/meson.build b/meson.build
index 241aa7d..5a7eea7 100644 (file)
--- a/meson.build
+++ b/meson.build
@@ -129,6 +129,9 @@ thread_dep = dependency('threads') # pthread for tensorflow-lite
 # Protobuf
 protobuf_dep = dependency('protobuf', version: '>= 3.6.1', required: false)
 
+# Protobuf compiler
+pb_comp = find_program('protoc', required: get_option('protobuf-support'))
+
 json_glib_dep = dependency('json-glib-1.0', required: get_option('nnfw-runtime-support')) 
 
 #orc
@@ -223,6 +226,9 @@ features = {
     'extra_deps': [ snpe_dep ],
     'project_args': { 'ENABLE_SNPE' : 1 },
     'extra_args': { 'SNPE_ROOT': SNPE_ROOT }
+  },
+  'protobuf-support': {
+    'extra_deps': [ pb_comp ]
   }
 }
 

diff --git a/meson_options.txt b/meson_options.txt
index 544ae68..6b3a4df 100644 (file)
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -15,6 +15,7 @@ option('tflite-nnapi-delegation', type: 'feature', value: 'auto')
 option('armnn-support', type: 'feature', value: 'auto')
 option('orcc-support', type: 'feature', value: 'auto')
 option('snpe-support', type: 'feature', value: 'auto')
+option('protobuf-support', type: 'feature', value: 'auto')
 
 # booleans & other options
 option('enable-test', type: 'boolean', value: true)