Introduce FLOAT16 stream type

Add FLOAT16 type.
Fortunately, gcc for x64, aarch64, and arm has extension for
float16.

@todo: update meson script so that arch-specific compiler
options for float16 can be added if float16 is enabled.

Addressed #3784

Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>

diff --git a/gst/nnstreamer/include/tensor_typedef.h b/gst/nnstreamer/include/tensor_typedef.h
index c3e3db7..2371ad3 100644 (file)
--- a/gst/nnstreamer/include/tensor_typedef.h
+++ b/gst/nnstreamer/include/tensor_typedef.h
@@ -59,7 +59,7 @@
 /**
  * @brief Possible tensor element types
  */
-#define GST_TENSOR_TYPE_ALL "{ float32, float64, int64, uint64, int32, uint32, int16, uint16, int8, uint8 }"
+#define GST_TENSOR_TYPE_ALL "{ float16, float32, float64, int64, uint64, int32, uint32, int16, uint16, int8, uint8 }"
 
 /**
  * @brief Possible tensor formats
@@ -162,11 +162,27 @@ typedef enum _nns_tensor_type
   _NNS_FLOAT32,
   _NNS_INT64,
   _NNS_UINT64,
+  _NNS_FLOAT16, /**< added with nnstreamer 2.1.1-devel. If you add any operators (e.g., tensor_transform) to float16, it will either be not supported or be too inefficient. */
 
   _NNS_END,
 } tensor_type;
 
 /**
+ * @brief Float16 compiler extension support
+ */
+#if defined(FLOAT16_SUPPORT)
+#if defined(__aarch64__) || defined(__arm__)
+/** arm (32b) requires "-mfp16-format=ieee */
+typedef __fp16 float16;
+#elif defined(__x86_64) || defined(__i686__)
+/** recommends "-mavx512fp16" for hardware acceleration (gcc>=12 x64)*/
+typedef _Float16 float16;
+#else
+#error "Float 16 supported only with aarch64, arm, x86/64. In arm, you need -mfp16-format=ieee"
+#endif /* x86/64, arm64/arm32 */
+#endif /* FLOAT16_SUPPORT */
+
+/**
  * @brief Possible input stream types for other/tensor.
  *
  * This is related with media input stream to other/tensor.
@@ -212,6 +228,9 @@ typedef union {
   float _float;
   int64_t _int64_t;
   uint64_t _uint64_t;
+#ifdef FLOAT16_SUPPORT
+  float16 _float16;
+#endif
 } tensor_element;
 
 typedef uint32_t tensor_dim[NNS_TENSOR_RANK_LIMIT];

diff --git a/gst/nnstreamer/nnstreamer_plugin_api_util_impl.c b/gst/nnstreamer/nnstreamer_plugin_api_util_impl.c
index 78fdddb..8d9fba6 100644 (file)
--- a/gst/nnstreamer/nnstreamer_plugin_api_util_impl.c
+++ b/gst/nnstreamer/nnstreamer_plugin_api_util_impl.c
@@ -28,6 +28,7 @@ static const gchar *tensor_element_typename[] = {
   [_NNS_FLOAT32] = "float32",
   [_NNS_INT64] = "int64",
   [_NNS_UINT64] = "uint64",
+  [_NNS_FLOAT16] = "float16",
   [_NNS_END] = NULL,
 };
 
@@ -45,7 +46,7 @@ static const guint tensor_element_size[] = {
   [_NNS_FLOAT32] = 4,
   [_NNS_INT64] = 8,
   [_NNS_UINT64] = 8,
-
+  [_NNS_FLOAT16] = 2,
   [_NNS_END] = 0,
 };
 
@@ -1063,11 +1064,14 @@ gst_tensor_get_type (const gchar * typestr)
       case 64:
         type = _NNS_INT64;
     }
-  } else if (g_regex_match_simple ("^float(32|64)$",
+  } else if (g_regex_match_simple ("^float(16|32|64)$",
           type_string, G_REGEX_CASELESS, 0)) {
     size = (gsize) g_ascii_strtoull (&type_string[5], NULL, 10);
 
     switch (size) {
+      case 16:
+        type = _NNS_FLOAT16;
+        break;
       case 32:
         type = _NNS_FLOAT32;
         break;

diff --git a/gst/nnstreamer/tensor_converter/tensor_converter.c b/gst/nnstreamer/tensor_converter/tensor_converter.c
index cc7bd0b..6f277a6 100644 (file)
--- a/gst/nnstreamer/tensor_converter/tensor_converter.c
+++ b/gst/nnstreamer/tensor_converter/tensor_converter.c
@@ -2001,6 +2001,11 @@ gst_tensor_converter_get_possible_media_caps (GstTensorConverter * self)
               case _NNS_UINT32:
                 aformat = GST_AUDIO_FORMAT_U32;
                 break;
+              case _NNS_FLOAT16:
+                aformat = GST_AUDIO_FORMAT_UNKNOWN;
+                ml_loge
+                    ("tensor_converter: Audio stream cannot be converted to float16 stream directly because GStreamer's standard audio streams do not support float16. Try Float32 or Float64 instead and 'transform' it to Float16 later.\n");
+                break;
               case _NNS_FLOAT32:
                 aformat = GST_AUDIO_FORMAT_F32;
                 break;

diff --git a/gst/nnstreamer/tensor_data.c b/gst/nnstreamer/tensor_data.c
index 8ca623e..980cc8c 100644 (file)
--- a/gst/nnstreamer/tensor_data.c
+++ b/gst/nnstreamer/tensor_data.c
@@ -38,6 +38,19 @@
     (td)->data._##otype = out_val; \
   } while (0)
 
+#ifdef FLOAT16_SUPPORT
+#define td_typecast_to_fromf16(td,otype) do { \
+    float16 in_val = (td)->data._float16; \
+    otype out_val = (otype) in_val; \
+    (td)->data._##otype = out_val; \
+  } while (0)
+#else /* FLOAT16_SUPPORT */
+#define td_typecast_to_fromf16(td,itype) do { \
+    nns_loge ("Your nnstreamer binary is built without -DFLOAT16_SUPPORT option; thus float16 is not supported.\n"); \
+    g_assert (0); \
+  } while (0)
+#endif
+
 #define td_typecast(td,otype) do { \
     switch ((td)->type) { \
       case _NNS_INT32: td_typecast_to (td, int32_t, otype); break; \
@@ -48,6 +61,7 @@
       case _NNS_UINT8: td_typecast_to (td, uint8_t, otype); break; \
       case _NNS_FLOAT64: td_typecast_to (td, double, otype); break; \
       case _NNS_FLOAT32: td_typecast_to (td, float, otype); break; \
+      case _NNS_FLOAT16: td_typecast_to_fromf16 (td, otype); break; \
       case _NNS_INT64: td_typecast_to (td, int64_t, otype); break; \
       case _NNS_UINT64: td_typecast_to (td, uint64_t, otype); break; \
       default: g_assert (0); break; \
@@ -95,6 +109,15 @@ gst_tensor_data_set (tensor_data_s * td, tensor_type type, gpointer value)
     case _NNS_FLOAT32:
       td_set_data (td, value, float);
       break;
+    case _NNS_FLOAT16:
+#ifdef FLOAT16_SUPPORT
+      td_set_data (td, value, float16);
+      break;
+#else
+      nns_loge
+          ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+      return FALSE;
+#endif
     case _NNS_INT64:
       td_set_data (td, value, int64_t);
       break;
@@ -147,6 +170,15 @@ gst_tensor_data_get (tensor_data_s * td, gpointer value)
     case _NNS_FLOAT32:
       td_get_data (td, value, float);
       break;
+    case _NNS_FLOAT16:
+#ifdef FLOAT16_SUPPORT
+      td_get_data (td, value, float16);
+      break;
+#else
+      nns_loge
+          ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+      return FALSE;
+#endif
     case _NNS_INT64:
       td_get_data (td, value, int64_t);
       break;
@@ -176,7 +208,7 @@ gst_tensor_data_typecast (tensor_data_s * td, tensor_type type)
 
   /* do nothing when transform to same type */
   if (td->type != type) {
-    is_float = (td->type == _NNS_FLOAT32 || td->type == _NNS_FLOAT64);
+    is_float = (td->type == _NNS_FLOAT32 || td->type == _NNS_FLOAT64 || td->type == _NNS_FLOAT16);
 
     switch (type) {
       case _NNS_INT32:
@@ -215,6 +247,14 @@ gst_tensor_data_typecast (tensor_data_s * td, tensor_type type)
       case _NNS_FLOAT32:
         td_typecast (td, float);
         break;
+      case _NNS_FLOAT16:
+#ifdef FLOAT16_SUPPORT
+        td_typecast (td, float16);
+#else
+        nns_loge ("NNStreamer requires -DFLOAT16_SUPPORT as a build option to enable float16 type. This binary does not have float16 feature enabled; thus, float16 type is not supported in this instance.\n");
+        return FALSE;
+#endif
+        break;
       case _NNS_INT64:
         td_typecast (td, int64_t);
         break;

diff --git a/meson.build b/meson.build
index 5576502..9ee4444 100644 (file)
--- a/meson.build
+++ b/meson.build
@@ -615,6 +615,32 @@ if check_mosquitto != ''
   add_project_arguments('-D__MQTT_BROKER_ENABLED__=1', language: ['c', 'cpp'])
 endif
 
+# Float16 Support
+if get_option('enable-float16')
+  arch = target_machine.cpu_family()
+  add_project_arguments('-DFLOAT16_SUPPORT', language: ['c', 'cpp'])
+
+  if arch == 'aarch64'
+    message ('Float16 for aarch64 enabled. Modern gcc-aarch64 generally supports float16 with __fp16. It uses IEEE 754-2008 format.')
+  elif arch == 'arm'
+    add_project_arguments('-mfp16-format=ieee', language: ['c', 'cpp'])
+    message ('Float16 for arm (32b) enabled. Modern gcc-arm generally supports float16 with __fp16 by -mfp16-format=ieee for IEEE 754-2008 format.')
+  elif arch == 'x86_64' or arch == 'x86'
+    # GCC 12+ has fp16 avx512 support.
+    has_avx512fp16 = cc.has_argument('-mavx512fp16')
+
+    if (has_avx512fp16)
+      add_project_arguments(['-mavx512fp16'], language: ['c', 'cpp'])
+      message ('Float16 for x86_64 enabled. Modern gcc-x64 genrally supports float16 with _Float16. -mavx512fp16 added for hardware acceleration')
+    else
+      warning ('Float16 for x86_64 enabled. However, software emulation is applied for fp16, making it slower and inconsistent. Use GCC 12+ for AVX512 FP16 support. This build will probably fail unless you bring a compiler that supports fp16 for x64.')
+    endif
+  else
+    error ('Float16 is supported only in aarch64, arm, and x86_64. If you believe the compiler for this architecture supports float16, you will need to update the build script (meson.build).')
+  endif
+
+endif
+
 # Build nnstreamer (common, plugins)
 subdir('gst')
 

diff --git a/meson_options.txt b/meson_options.txt
index ec1356e..04f1034 100644 (file)
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -49,6 +49,7 @@ option('framework-priority-nb', type: 'string', value: '', description: 'A comma
 option('framework-priority-bin', type: 'string', value: '', description: 'A comma separated prioritized list of neural network frameworks to open a .bin file')
 option('skip-tflite-flatbuf-check', type: 'boolean', value: false, description: 'Do not check the availability of flatbuf for tensorflow-lite build. In some systems, flatbuffers\' dependency cannot be found with meson.')
 option('trix-engine-alias', type: 'string', value: 'srnpu', description: 'The alias name list of trix-engine sub-plugin. This option provides backward compatibility of the previous framework name.')
+option('enable-float16', type: 'boolean', value: false, description: 'Support float16 streams with GCC extensions')
 
 # Utilities
 option('enable-nnstreamer-check', type: 'boolean', value: true)