tensor-filter subplugins: add FLOAT16 support

Add float16 support for a few subplugins
with possible float16 usages.

Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_armnn.cc b/ext/nnstreamer/tensor_filter/tensor_filter_armnn.cc
index b1762a5..9fbd844 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_armnn.cc
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_armnn.cc
@@ -461,7 +461,11 @@ ArmNNCore::getGstTensorType (armnn::DataType armType)
     /** Supported with tf, tflite and caffe */
     return _NNS_FLOAT32;
   case armnn::DataType::Float16:
-    ml_logw ("Unsupported armnn datatype Float16.");
+#ifdef FLOAT16_SUPPORT
+    return _NNS_FLOAT16;
+#else
+    ml_logw ("Unsupported armnn datatype Float16. Recompile with -DSUPPORT_FLOAT16 option.");
+#endif
     break;
   case armnn::DataType::QAsymmU8:
     /** Supported with tflite */

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_deepview_rt.cc b/ext/nnstreamer/tensor_filter/tensor_filter_deepview_rt.cc
index 9fce6ab..6ee7fb6 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_deepview_rt.cc
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_deepview_rt.cc
@@ -268,6 +268,10 @@ dvrt_subplugin::getTensorType (gsize index, tensor_type *type)
     res = _NNS_FLOAT64;
     break;
   case NNTensorType_F16:
+#ifdef FLOAT16_SUPPORT
+    res = _NNS_FLOAT16;
+    break;
+#endif
   default:
     nns_logw ("Tensor type not supported: %d", (gint)_type);
     return -EINVAL;

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_lua.cc b/ext/nnstreamer/tensor_filter/tensor_filter_lua.cc
index b781c1e..742b692 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_lua.cc
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_lua.cc
@@ -145,6 +145,15 @@ tensor_index (lua_State *L)
     case _NNS_FLOAT32:
       value = (double) ((float *) lt->data)[tidx];
       break;
+    case _NNS_FLOAT16:
+#ifdef FLOAT16_SUPPORT
+      value = (double) ((float16 *) lt->data)[tidx];
+#else
+      nns_loge
+          ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+      throw std::runtime_error ("Float16 not supported. Recompile with -DFLOAT16_SUPPORT.");
+#endif
+      break;
     case _NNS_INT64:
       value = (double) ((int64_t *) lt->data)[tidx];
       break;
@@ -207,6 +216,15 @@ tensor_newindex (lua_State* L)
     case _NNS_FLOAT32:
       ((float *) lt->data)[tidx] = (float) value;
       break;
+    case _NNS_FLOAT16:
+#ifdef FLOAT16_SUPPORT
+      ((float16 *) lt->data)[tidx] = (float16) value;
+#else
+      nns_loge
+          ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+      throw std::runtime_error ("Float16 not supported. Recompile with -DFLOAT16_SUPPORT.");
+#endif
+      break;
     case _NNS_INT64:
       ((int64_t *) lt->data)[tidx] = (int64_t) value;
       break;

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_pytorch.cc b/ext/nnstreamer/tensor_filter/tensor_filter_pytorch.cc
index 3b22b6c..640ea0f 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_pytorch.cc
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_pytorch.cc
@@ -253,6 +253,13 @@ TorchCore::getTensorTypeFromTorch (torch::Dtype torchType)
   case torch::kF64:
     return _NNS_FLOAT64;
   case torch::kF16:
+#ifdef FLOAT16_SUPPORT
+    return _NNS_FLOAT16;
+#else
+    ml_loge
+        ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+    break;
+#endif
   default:
     break;
   }
@@ -284,6 +291,14 @@ TorchCore::getTensorTypeToTorch (tensor_type tensorType, torch::Dtype *torchType
   case _NNS_INT64:
     *torchType = torch::kI64;
     break;
+  case _NNS_FLOAT16:
+#ifdef FLOAT16_SUPPORT
+    *torchType = torch::kF16;
+#else
+    ml_loge
+        ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+#endif
+    break;
   case _NNS_FLOAT32:
     *torchType = torch::kF32;
     break;

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc
index 7fb7d2d..8d3d05c 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow.cc
@@ -306,6 +306,14 @@ TFCore::getTensorTypeFromTF (TF_DataType tfType)
     return _NNS_FLOAT32;
   case TF_DOUBLE:
     return _NNS_FLOAT64;
+  case TF_HALF:
+#ifdef FLOAT16_SUPPORT
+    return _NNS_FLOAT16;
+#else
+    ml_loge
+        ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+    break;
+#endif
   default:
     /** @todo Support other types */
     break;
@@ -339,6 +347,14 @@ TFCore::getTensorTypeToTF (tensor_type tType)
     return TF_INT64;
   case _NNS_UINT64:
     return TF_UINT64;
+  case _NNS_FLOAT16:
+#ifdef FLOAT16_SUPPORT
+    return TF_HALF;
+#else
+    ml_loge
+        ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+    break;
+#endif
   case _NNS_FLOAT32:
     return TF_FLOAT;
   case _NNS_FLOAT64:

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.cc b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.cc
index d85df9b..b18d1ae 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.cc
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.cc
@@ -592,13 +592,20 @@ TFLiteInterpreter::getTensorType (TfLiteType tfType)
 #endif
   case kTfLiteInt64:
     return _NNS_INT64;
+#ifdef TFLITE_FLOAT16
+  case kTfLiteFloat16:
+#ifdef FLOAT16_SUPPORT
+    return _NNS_FLOAT16;
+#else
+    ml_loge
+        ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+    break;
+#endif
+#endif
   case kTfLiteString:
 #ifdef TFLITE_COMPLEX64
   case kTfLiteComplex64:
 #endif
-#ifdef TFLITE_FLOAT16
-  case kTfLiteFloat16:
-#endif
   default:
     ml_loge ("Not supported Tensorflow Data Type: [%d].", tfType);
     /** @todo Support other types */

diff --git a/ext/nnstreamer/tensor_filter/vivante/tensor_filter_subplugin.c b/ext/nnstreamer/tensor_filter/vivante/tensor_filter_subplugin.c
index 220c3df..d0a677a 100644 (file)
--- a/ext/nnstreamer/tensor_filter/vivante/tensor_filter_subplugin.c
+++ b/ext/nnstreamer/tensor_filter/vivante/tensor_filter_subplugin.c
@@ -245,11 +245,15 @@ convert_tensortype (unsigned tensor_type)
       return _NNS_INT16;
     case VSI_NN_TYPE_UINT16:
       return _NNS_UINT16;
-  /** Note that the current nnstreamer (tensor_typedef.h) does not support FLOAT16.
-   *  Let's use UINT16 as a workaround.
-   */
     case VSI_NN_TYPE_FLOAT16:
+#ifdef FLOAT16_SUPPORT
+      return _NNS_FLOAT16;
+#else
+      /**
+       *  Let's use UINT16 as a workaround if FLOAT16 is not supported.
+       */
       return _NNS_UINT16;
+#endif
     case VSI_NN_TYPE_FLOAT32:
       return _NNS_FLOAT32;
     default:
@@ -364,7 +368,8 @@ vivante_open (const GstTensorFilterProperties * prop, void **private_data)
     /** Get an input data type: VSI_NN_TYPE_UINT8 (u8) in case of inceptionv3 */
     pdata->input_tensor.info[i].type =
         convert_tensortype (i_tensor->attr.dtype.vx_type);
-    asprintf (&pdata->input_tensor.info[i].name, "%i", pdata->graph->input.tensors[i]);
+    asprintf (&pdata->input_tensor.info[i].name, "%i",
+        pdata->graph->input.tensors[i]);
                                          /** dummy name */
     pdata->input_tensor.num_tensors = pdata->graph->input.num; /** number of tensors */
   }
@@ -390,7 +395,8 @@ vivante_open (const GstTensorFilterProperties * prop, void **private_data)
     /** Get an output data type: VSI_NN_TYPE_FLOAT16 (f16) in case of inceptionv3 */
     pdata->output_tensor.info[i].type =
         convert_tensortype (o_tensor->attr.dtype.vx_type);
-    asprintf (&pdata->output_tensor.info[i].name, "%i", pdata->graph->output.tensors[i]);
+    asprintf (&pdata->output_tensor.info[i].name, "%i",
+        pdata->graph->output.tensors[i]);
                                           /** dummy name */
     pdata->output_tensor.num_tensors = pdata->graph->output.num; /** number of tensors */
   }

diff --git a/tests/nnstreamer_filter_lua/unittest_filter_lua.cc b/tests/nnstreamer_filter_lua/unittest_filter_lua.cc
index e8e5f31..d180ac2 100644 (file)
--- a/tests/nnstreamer_filter_lua/unittest_filter_lua.cc
+++ b/tests/nnstreamer_filter_lua/unittest_filter_lua.cc
@@ -954,6 +954,10 @@ TEST (nnstreamerFilterLua, dataType01)
 
   for (int i = 0; i < _NNS_END; ++i) {
     tensor_type ttype = (tensor_type) i;
+#ifndef FLOAT16_SUPPORT
+    if (i == _NNS_FLOAT16)
+      continue;
+#endif
     model = g_strdup_printf ("\
     inputTensorsInfo={num=1,dim={{1,2,2,1},},type={'%s',}} \
     outputTensorsInfo={num=1,dim={{1,2,2,1},},type={'%s',}} \