libgtest-dev, ssat, libpng-dev, libopencv-dev, liborc-0.4-dev,
python, python-numpy, python3, python3-dev, python3-numpy,
tensorflow-lite-dev, pytorch, libedgetpu1-std (>=12), libedgetpu-dev (>=12),
- libflatbuffers-dev, flatbuffers-compiler,
- tensorflow-dev [amd64], python2.7-dev, libprotobuf-dev [amd64 arm64 armhf]
+ libflatbuffers-dev, flatbuffers-compiler, protobuf-compiler17,
+ tensorflow-dev [amd64], python2.7-dev, libprotobuf-dev [amd64 arm64 armhf],
Standards-Version: 3.9.6
Homepage: https://github.com/nnstreamer/nnstreamer
syntax = "proto3";
-package NNStreamer;
+package nnstreamer.protobuf;
message Tensor {
string name = 1;
install_dir: converter_subplugin_install_dir
)
endif
+
+# protocol buffer
+if protobuf_support_is_available
+ converter_sub_protobuf_sources = [
+ 'tensor_converter_protobuf.cc'
+ ]
+
+ nnstreamer_converter_protobuf_sources = [pb_gen_src]
+ foreach s : converter_sub_protobuf_sources
+ nnstreamer_converter_protobuf_sources += join_paths(meson.current_source_dir(), s)
+ endforeach
+
+ shared_library('nnstreamer_converter_protobuf',
+ nnstreamer_converter_protobuf_sources,
+ dependencies: [nnstreamer_dep, glib_dep, gst_dep, protobuf_dep],
+ install: true,
+ install_dir: converter_subplugin_install_dir
+ )
+endif
--- /dev/null
+/* SPDX-License-Identifier: LGPL-2.1-only */
+/**
+ * GStreamer / NNStreamer tensor_converter subplugin, "protobuf"
+ * Copyright (C) 2020 Gichan Jang <gichan2.jang@samsung.com>
+ */
+ /**
+ * @file tensor_converter_protobuf.cc
+ * @date 2 June 2020
+ * @brief NNStreamer tensor-converter subplugin, "protobuf",
+ * which converts protobuf byte stream to tensors.
+ * @see https://github.com/nnsuite/nnstreamer
+ * @author Gichan Jang <gichan2.jang@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+ /**
+ * Install protobuf
+ * We assume that you use Ubuntu linux distribution.
+ * You may simply download binary packages from PPA
+ *
+ * $ sudo apt-add-repository ppa:nnstreamer
+ * $ sudo apt update
+ * $ sudo apt install libprotobuf-dev libprotobuf-lite17 libprotobuf17 protobuf-compiler17
+ */
+
+#include <iostream>
+#include <fstream>
+#include <typeinfo>
+#include <glib.h>
+#include <gst/gstinfo.h>
+#include <nnstreamer_plugin_api.h>
+#include <string>
+#include "nnstreamer_plugin_api_converter.h"
+#include <nnstreamer_log.h>
+#include "nnstreamer.pb.h" /* Generated by `protoc` */
+
+namespace nnstreamer {
+namespace protobuf {
+
+ void init_pbc (void) __attribute__ ((constructor));
+ void fini_pbc (void) __attribute__ ((destructor));
+
+/** @brief tensor converter plugin's NNStreamerExternalConverter callback */
+static GstCaps *pbc_query_caps (const GstTensorsConfig * config)
+{
+ return gst_caps_from_string (GST_PROTOBUF_TENSOR_CAP_DEFAULT);
+}
+
+/** @brief tensor converter plugin's NNStreamerExternalConverter callback */
+static gboolean
+ pbc_get_out_config (const GstCaps * in_cap, GstTensorsConfig * config)
+{
+ GstStructure *structure;
+
+ g_return_val_if_fail (config != NULL, FALSE);
+ gst_tensors_config_init (config);
+ g_return_val_if_fail (in_cap != NULL, FALSE);
+
+ structure = gst_caps_get_structure (in_cap, 0);
+ g_return_val_if_fail (structure != NULL, FALSE);
+
+ /* All tensor info should be updated later in chain function. */
+ config->info.info[0].type = _NNS_UINT8;
+ config->info.num_tensors = 1;
+ if (gst_tensor_parse_dimension ("1:1:1:1",
+ config->info.info[0].dimension) == 0) {
+ ml_loge ("Failed to set initial dimension for subplugin");
+ return FALSE;
+ }
+
+ if (gst_structure_has_field (structure, "framerate"))
+ {
+ gst_structure_get_fraction (structure, "framerate", &config->rate_n,
+ &config->rate_d);
+ } else {
+ /* cannot get the framerate */
+ config->rate_n = 0;
+ config->rate_d = 1;
+ }
+ return TRUE;
+}
+
+/** @brief tensor converter plugin's NNStreamerExternalConverter callback */
+ static GstBuffer *pbc_convert (GstBuffer * in_buf, gsize * frame_size,
+ guint * frames_in, GstTensorsConfig * config)
+ {
+ Tensors tensors;
+ Tensors::frame_rate * fr = NULL;
+ GstMemory *in_mem, *out_mem;
+ GstMapInfo in_info;
+ GstBuffer *out_buf;
+ guint mem_size;
+ gpointer mem_data;
+
+ in_mem = gst_buffer_peek_memory (in_buf, 0);
+ g_assert (gst_memory_map (in_mem, &in_info, GST_MAP_READ));
+
+ tensors.ParseFromArray (in_info.data, in_info.size);
+
+ config->info.num_tensors = tensors.num_tensor ();
+ fr = tensors.mutable_fr ();
+ config->rate_n = fr->rate_n ();
+ config->rate_d = fr->rate_d ();
+
+ out_buf = gst_buffer_new ();
+
+ for (guint i = 0; i < config->info.num_tensors; i++) {
+ const Tensor *tensor = &tensors.tensor (i);
+ config->info.info[i].name = g_strdup (tensor->name ().c_str ());
+ config->info.info[i].type = (tensor_type) tensor->type ();
+ for (guint j = 0; j < NNS_TENSOR_RANK_LIMIT; j++) {
+ config->info.info[i].dimension[j] = tensor->dimension (j);
+ }
+ *frames_in = 1;
+ *frame_size = mem_size = tensor->data ().length ();
+
+ mem_data = g_memdup (tensor->data ().c_str (), mem_size);
+
+ out_mem = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY,
+ mem_data, mem_size, 0, mem_size, NULL, NULL);
+
+ gst_buffer_append_memory (out_buf, out_mem);
+ }
+
+ /** copy timestamps */
+ gst_buffer_copy_into (out_buf, in_buf,
+ (GstBufferCopyFlags) GST_BUFFER_COPY_METADATA, 0, -1);
+ gst_memory_unmap (in_mem, &in_info);
+
+ return out_buf;
+ }
+
+ static gchar converter_subplugin_protobuf[] =
+ "libnnstreamer_converter_protobuf";
+
+/** @brief protobuf tensor converter sub-plugin NNStreamerExternalConverter instance */
+ static NNStreamerExternalConverter protobuf = {
+ .name = converter_subplugin_protobuf,
+ .convert = pbc_convert,
+ .get_out_config = pbc_get_out_config,
+ .query_caps = pbc_query_caps
+ };
+
+/** @brief Initialize this object for tensor converter sub-plugin */
+ void init_pbc (void)
+ {
+ /**
+ * Verify that the version of the library we linked is
+ * compatibile with the headers.
+ */
+ GOOGLE_PROTOBUF_VERIFY_VERSION;
+ registerExternalConverter (&protobuf);
+ }
+
+/** @brief Destruct this object for tensor converter sub-plugin */
+ void fini_pbc (void)
+ {
+ unregisterExternalConverter (protobuf.name);
+ google::protobuf::ShutdownProtobufLibrary ();
+ }
+
+}; /* Namespace protobuf */
+}; /* Namespace nnstreamer */
install: true,
install_dir: decoder_subplugin_install_dir
)
+<<<<<<< c5239c0bfc9a1af65cecc494f1ffc1daa121d6fd
endif
# flatbuffer
install: true,
install_dir: decoder_subplugin_install_dir,
)
+=======
+>>>>>>> [Converter] Add tensor converter sub-plugin protobuf
endif
#include "nnstreamer.pb.h" /* Generated by `protoc` */
+namespace nnstreamer {
+namespace protobuf {
+
void init_pb (void) __attribute__ ((constructor));
void fini_pb (void) __attribute__ ((destructor));
GstMemory *out_mem;
size_t size, outbuf_size;
- NNStreamer::Tensors tensors;
- NNStreamer::Tensors::frame_rate *fr = NULL;
+ Tensors tensors;
+ Tensors::frame_rate *fr = NULL;
const unsigned int num_tensors = config->info.num_tensors;
g_assert (num_tensors > 0);
fr->set_rate_d (config->rate_d);
for (unsigned int i = 0; i < num_tensors; ++i) {
- NNStreamer::Tensor *tensor = tensors.add_tensor ();
+ Tensor *tensor = tensors.add_tensor ();
gchar *name = NULL;
name = config->info.info[i].name;
tensor->set_name (name);
}
- tensor->set_type ((NNStreamer::Tensor::Tensor_type)
+ tensor->set_type ((Tensor::Tensor_type)
config->info.info[i].type);
for (int j = 0; j < NNS_TENSOR_RANK_LIMIT; ++j) {
{
nnstreamer_decoder_exit (protobuf.modename);
}
+
+}; /* Namespace protobuf */
+}; /* Namespace nnstreamer */
%define gstpostfix gstreamer-1.0
%define gstlibdir %{_libdir}/%{gstpostfix}
%define nnstexampledir /usr/lib/nnstreamer/bin
-%define tensorflow_support 0
+%define tensorflow_support 0
%define tensorflow_lite_support 1
%define armnn_support 0
%define vivante_support 0
%define flatbuf_support 1
+%define protobuf_support 1
%if 0%{tizen_version_major} >= 5
%define python_support 1
BuildRequires: opencv-devel
# For './testAll.sh' time limit.
BuildRequires: procps
+# for protobuf
+%if 0%{?protobuf_support}
+BuildRequires: protobuf-devel >= 3.4.0
+%endif
# for tensorflow
%if 0%{?tensorflow_support}
-BuildRequires: protobuf-devel >= 3.4.0
BuildRequires: tensorflow
BuildRequires: tensorflow-devel
%endif
%{_prefix}/lib/nnstreamer/decoders/libnnstreamer_decoder_*.so
%{_prefix}/lib/nnstreamer/converters/libnnstreamer_converter_*.so
%{_prefix}/lib/nnstreamer/filters/libnnstreamer_filter_cpp.so
+%{_prefix}/lib/nnstreamer/converters/libnnstreamer_converter_*.so
%{gstlibdir}/libnnstreamer.so
%{_libdir}/libnnstreamer.so
%{_sysconfdir}/nnstreamer.ini
--- /dev/null
+#!/usr/bin/env bash
+##
+## SPDX-License-Identifier: LGPL-2.1-only
+##
+## @file runTest.sh
+## @author Gichan Jang <gichan2.jang@samsung.com>
+## @date June 08 2020
+## @brief SSAT Test Cases for protobuf subplugin of tensor converter and decoder
+## @details After decoding the tensor into protobuf, convert it to tensor again to check if it matches the original
+##
+if [[ "$SSATAPILOADED" != "1" ]]; then
+ SILENT=0
+ INDEPENDENT=1
+ search="ssat-api.sh"
+ source $search
+ printf "${Blue}Independent Mode${NC}
+"
+fi
+
+# This is compatible with SSAT (https://github.com/myungjoo/SSAT)
+testInit $1
+
+if [ "$SKIPGEN" == "YES" ]; then
+ echo "Test Case Generation Skipped"
+ sopath=$2
+else
+ echo "Test Case Generation Started"
+ python ../nnstreamer_converter/generateGoldenTestResult.py
+ python ../nnstreamer_merge/generateTest.py
+ sopath=$1
+fi
+convertBMP2PNG
+
+PATH_TO_PLUGIN="../../build"
+
+gst-launch-1.0 -m -v --gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=280,height=40,framerate=0/1 ! tee name=t ! queue ! videoconvert ! video/x-raw, format=BGRx ! tensor_converter silent=TRUE ! filesink location="test.bgrx.log" sync=true t. ! queue ! filesink location="test.rgb.log" sync=true
+
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=280,height=40,framerate=0/1 ! tee name=t ! queue ! videoconvert ! video/x-raw, format=BGRx ! tensor_converter silent=TRUE ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! filesink location=\"test.bgrx.log\" sync=true t. ! queue ! filesink location=\"test.rgb.log\" sync=true" 1R 0 0 $PERFORMANCE
+
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=GRAY8,width=280,height=40,framerate=0/1 ! queue ! tensor_converter silent=TRUE ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! filesink location=\"test.gray8.log\" sync=true" 1G 0 0 $PERFORMANCE
+
+callCompareTest testcase01.bgrx.golden test.bgrx.log 1-1 "BGRX Golden Test" 1 0
+callCompareTest testcase01.rgb.golden test.rgb.log 1-2 "RGB Golden Test" 1 0
+callCompareTest testcase01.gray8.golden test.gray8.log 1-3 "Gray8 Golden Test" 1 0
+
+##
+## @brief Execute gstreamer pipeline and do golden-test with the output of the pipeline.
+## @param $1 Colorspace
+## @param $2 Width
+## @param $3 Height
+## @param $4 Test Case Number
+function do_test() {
+ gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=testcase02_${1}_${2}x${3}.png ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=${1},width=${2},height=${3},framerate=0/1 ! tensor_converter silent=TRUE ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! filesink location=\"testcase02_${1}_${2}x${3}.log\" sync=true" ${4} 0 0 $PERFORMANCE
+
+ callCompareTest testcase02_${1}_${2}x${3}.golden testcase02_${1}_${2}x${3}.log ${4} "PNG Golden Testing ${4}" 1 0
+}
+
+do_test BGRx 640 480 2-1
+do_test RGB 640 480 2-2
+do_test GRAY8 640 480 2-3
+do_test BGRx 642 480 2-4
+do_test RGB 642 480 2-5
+do_test GRAY8 642 480 2-6
+
+# Fail Test: YUV is given
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=YUV,width=280,height=40,framerate=0/1 ! videoconvert ! video/x-raw, format=YUV ! tensor_converter silent=TRUE ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! filesink location=\"test.yuv.fail.log\" sync=true" 5F_n 0 1 $PERFORMANCE
+
+# Fail Test: Unknown property is given
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} videotestsrc num-buffers=1 ! video/x-raw,format=RGB,width=280,height=40,framerate=0/1 ! videoconvert ! video/x-raw, format=RGB ! tensor_converter silent=TRUE whatthehell=isthis ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! filesink location=\"test.yuv.fail.log\" sync=true" 6F_n 0 1 $PERFORMANCE
+
+# audio format S16LE, 8k sample rate, samples per buffer 8000
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} audiotestsrc num-buffers=1 samplesperbuffer=8000 ! audioconvert ! audio/x-raw,format=S16LE,rate=8000 ! tee name=t ! queue ! audioconvert ! tensor_converter frames-per-tensor=8000 ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! filesink location=\"test.audio8k.s16le.log\" sync=true t. ! queue ! filesink location=\"test.audio8k.s16le.origin.log\" sync=true" 7-1 0 0 $PERFORMANCE
+callCompareTest test.audio8k.s16le.origin.log test.audio8k.s16le.log 7-2 "Audio8k-s16le Golden Test" 0 0
+
+# audio format U8, 16k sample rate, samples per buffer 8000
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} audiotestsrc num-buffers=1 samplesperbuffer=8000 ! audioconvert ! audio/x-raw,format=U8,rate=16000 ! tee name=t ! queue ! audioconvert ! tensor_converter frames-per-tensor=8000 ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! filesink location=\"test.audio16k.u8.log\" sync=true t. ! queue ! filesink location=\"test.audio16k.u8.origin.log\" sync=true" 7-3 0 0 $PERFORMANCE
+callCompareTest test.audio16k.u8.origin.log test.audio16k.u8.log 7-4 "Audio16k-u8 Golden Test" 0 0
+
+# audio format U16LE, 16k sample rate, 2 channels, samples per buffer 8000
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} audiotestsrc num-buffers=1 samplesperbuffer=8000 ! audioconvert ! audio/x-raw,format=U16LE,rate=16000,channels=2 ! tee name=t ! queue ! audioconvert ! tensor_converter frames-per-tensor=8000 ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! filesink location=\"test.audio16k2c.u16le.log\" sync=true t. ! queue ! filesink location=\"test.audio16k2c.u16le.origin.log\" sync=true" 7-5 0 0 $PERFORMANCE
+callCompareTest test.audio16k2c.u16le.origin.log test.audio16k2c.u16le.log 7-6 "Audio16k2c-u16le Golden Test" 0 0
+
+# Stream test case (genCase08 in generateGoldenTestResult.py)
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} multifilesrc location=\"testsequence_%1d.png\" index=0 caps=\"image/png,framerate=\(fraction\)30/1\" ! pngdec ! videoconvert ! tensor_converter ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! filesink location=\"testcase08.log\"" 8 0 0 $PERFORMANCE
+callCompareTest testcase08.golden testcase08.log 8 "PNG Stream Test" 0 0
+
+# tensor merge test (The output is always in the format of other/tensor)
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} tensor_merge name=merge mode=linear option=2 silent=true sync_mode=basepad sync_option=0:0 ! multifilesink location=testsynch08_%1d.log multifilesrc location=\"testsequence03_%1d.png\" index=0 caps=\"image/png, framerate=(fraction)10/1\" ! pngdec ! tensor_converter ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! merge.sink_0 multifilesrc location=\"testsequence03_%1d.png\" index=0 caps=\"image/png, framerate=(fraction)30/1\" ! pngdec ! tensor_converter ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! merge.sink_1" 9 0 0 $PERFORMANCE
+callCompareTest testsynch08_0.golden testsynch08_0.log 9-1 "Tensor merge Compare 9-1" 1 0
+callCompareTest testsynch08_1.golden testsynch08_1.log 9-2 "Tensor merge Compare 9-2" 1 0
+callCompareTest testsynch08_2.golden testsynch08_2.log 9-3 "Tensor merge Compare 9-3" 1 0
+callCompareTest testsynch08_3.golden testsynch08_3.log 9-4 "Tensor merge Compare 9-4" 1 0
+
+# tensor mux test (The output is always in the format of other/tensors)
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} tensor_mux name=tensors_mux sync_mode=basepad sync_option=1:50000000 ! multifilesink location=testsynch19_%1d.log \
+ tensor_mux name=tensor_mux0 sync_mode=slowest ! tensors_mux.sink_0 \
+ tensor_mux name=tensor_mux1 sync_mode=slowest ! tensors_mux.sink_1 \
+ multifilesrc location=\"testsequence03_%1d.png\" index=0 caps=\"image/png, framerate=(fraction)10/1\" ! pngdec ! \
+ tensor_converter ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! tensor_mux0.sink_0 \
+ multifilesrc location=\"testsequence03_%1d.png\" index=0 caps=\"image/png, framerate=(fraction)20/1\" ! pngdec ! \
+ tensor_converter ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! tensor_mux0.sink_1 \
+ multifilesrc location=\"testsequence03_%1d.png\" index=0 caps=\"image/png, framerate=(fraction)30/1\" ! pngdec ! \
+ tensor_converter ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! tensor_mux1.sink_0 \
+ multifilesrc location=\"testsequence03_%1d.png\" index=0 caps=\"image/png, framerate=(fraction)20/1\" ! pngdec ! \
+ tensor_converter ! tensor_decoder mode=protobuf ! other/protobuf-tensor ! tensor_converter ! tensor_mux1.sink_1" 10 0 0 $PERFORMANCE
+callCompareTest testsynch19_0.golden testsynch19_0.log 10-1 "Tensor mux Compare 10-1" 1 0
+callCompareTest testsynch19_1.golden testsynch19_1.log 10-2 "Tensor mux Compare 10-2" 1 0
+callCompareTest testsynch19_2.golden testsynch19_2.log 10-3 "Tensor mux Compare 10-3" 1 0
+callCompareTest testsynch19_3.golden testsynch19_3.log 10-4 "Tensor mux Compare 10-4" 1 0
+callCompareTest testsynch19_4.golden testsynch19_4.log 10-5 "Tensor mux Compare 10-5" 1 0
+
+report
projects / platform / upstream / nnstreamer.git / commitdiff