[Filter/TF] support string type

In tensorflow sub-plugin, support DT_STRING type.
Added testcase with speech command model, which classifies wav input to keyword.
This model has input type DT_STRING.

Signed-off-by: Jaeyun Jung <jy1210.jung@samsung.com>

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.cc b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.cc
index d70b311..71ab597 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.cc
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_core.cc
@@ -249,8 +249,13 @@ TFCore::validateInputTensor (const GraphDef &graph_def)
     }
 
     if (inputTensorMeta.info[i].type != getTensorTypeFromTF (dtype)) {
-      GST_ERROR ("Input Tensor is not valid: the type of input tensor is different\n");
-      return -3;
+      /* consider the input data as bytes if tensor type is string */
+      if (dtype == DT_STRING) {
+        GST_WARNING ("Input type is string, ignore type comparision.");
+      } else {
+        GST_ERROR ("Input Tensor is not valid: the type of input tensor is different\n");
+        return -3;
+      }
     }
 
     gchar **str_dims;
@@ -373,8 +378,12 @@ TFCore::run (const GstTensorMemory * input, GstTensorMemory * output)
     Tensor in (input_tensor_info[i].type, input_tensor_info[i].shape);
 
     /* copy data */
-    std::copy_n ((char *) input[i].data, input[i].size,
-        const_cast<char *>(in.tensor_data().data()));
+    if (input_tensor_info[i].type == DT_STRING) {
+      in.scalar<string>()() = string ((char *) input[i].data, input[i].size);
+    } else {
+      std::copy_n ((char *) input[i].data, input[i].size,
+          const_cast<char *>(in.tensor_data().data()));
+    }
 
     input_feeds.push_back ({inputTensorMeta.info[i].name, in});
   }
@@ -394,6 +403,9 @@ TFCore::run (const GstTensorMemory * input, GstTensorMemory * output)
   }
 
   for (int i = 0; i < outputTensorMeta.num_tensors; ++i) {
+    /**
+     * @todo support DT_STRING output tensor
+     */
     output[i].data = const_cast<char *>(outputs[i].tensor_data().data());
     outputTensorMap.insert (std::make_pair (output[i].data, outputs[i]));
   }

diff --git a/tests/nnstreamer_filter_tensorflow/data/2.wav b/tests/nnstreamer_filter_tensorflow/data/2.wav
new file mode 100644 (file)
index 0000000..36d48dc
Binary files /dev/null and b/tests/nnstreamer_filter_tensorflow/data/2.wav differ

diff --git a/tests/nnstreamer_filter_tensorflow/runTest.sh b/tests/nnstreamer_filter_tensorflow/runTest.sh
index eb7089c..f12aaa8 100755 (executable)
--- a/tests/nnstreamer_filter_tensorflow/runTest.sh
+++ b/tests/nnstreamer_filter_tensorflow/runTest.sh
@@ -21,13 +21,21 @@ if [[ -z "${TEST_TENSORFLOW}" ]]; then
     report
 fi
 
-# Test constant passthrough custom filter (1, 2)
+# Test with mnist model
 PATH_TO_PLUGIN="../../build"
 PATH_TO_MODEL="../test_models/models/mnist.pb"
 PATH_TO_DATA="data/9.raw"
 
-gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_DATA} ! application/octet-stream ! tensor_converter input-dim=784:1 input-type=uint8 ! tensor_transform mode=arithmetic option=typecast:float32,add:-127.5,div:127.5 ! tensor_filter framework=tensorflow model=${PATH_TO_MODEL} input=784:1:1:1 inputtype=float32 inputname=input output=10:1:1:1 outputtype=float32 outputname=softmax ! filesink location=tensorfilter.out.log " 1 0 0 $PERFORMANCE
-python checkLabel.py tensorfilter.out.log ${PATH_TO_DATA}
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_DATA} ! application/octet-stream ! tensor_converter input-dim=784:1 input-type=uint8 ! tensor_transform mode=arithmetic option=typecast:float32,add:-127.5,div:127.5 ! tensor_filter framework=tensorflow model=${PATH_TO_MODEL} input=784:1:1:1 inputtype=float32 inputname=input output=10:1:1:1 outputtype=float32 outputname=softmax ! filesink location=tensorfilter.out.1.log " 1 0 0 $PERFORMANCE
+python checkLabel.py tensorfilter.out.1.log ${PATH_TO_DATA}
 testResult $? 1 "Golden test comparison" 0 1
 
+# Test with speech command model (.wav file, answer is 'yes', this model has a input type DT_STRING.)
+PATH_TO_MODEL="../test_models/models/conv_actions_frozen.pb"
+PATH_TO_DATA="data/2.wav"
+
+gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_DATA} blocksize=-1 ! application/octet-stream ! tensor_converter input-dim=1:16022 input-type=int16 ! tensor_filter framework=tensorflow model=${PATH_TO_MODEL} input=1:16022 inputtype=int16 inputname=wav_data output=12 outputtype=float32 outputname=labels_softmax ! filesink location=tensorfilter.out.2.log " 2 0 0 $PERFORMANCE
+python checkLabel.py tensorfilter.out.2.log ${PATH_TO_DATA}
+testResult $? 2 "Golden test comparison" 0 1
+
 report

diff --git a/tests/test_models/models/conv_actions_frozen.pb b/tests/test_models/models/conv_actions_frozen.pb
new file mode 100644 (file)
index 0000000..0c07767
Binary files /dev/null and b/tests/test_models/models/conv_actions_frozen.pb differ