From 75d32d20bc5d10e3ac3436d55740d44425ae8540 Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Thu, 7 Nov 2019 16:00:03 +0900 Subject: [PATCH] [tensor_filter/plugin] Update for supporting various accelerators Added update to support various accelerators for tensor filters Updated tensorflow-lite nnapi setting based on these accelerators Added testcases for tensorflow-lite Signed-off-by: Parichay Kapoor --- .../tensor_filter/tensor_filter_tensorflow_lite.c | 47 +------- .../tensor_filter_tensorflow_lite_core.cc | 84 ++++++++++++-- .../tensor_filter_tensorflow_lite_core.h | 9 +- gst/nnstreamer/nnstreamer_plugin_api_filter.h | 74 ++++++++++-- .../tensor_filter/tensor_filter_common.c | 125 ++++++++++++++++++--- tests/nnstreamer_filter_tensorflow_lite/runTest.sh | 84 ++++++++++++-- 6 files changed, 333 insertions(+), 90 deletions(-) diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c index a34fb42..669954e 100644 --- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c +++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite.c @@ -30,7 +30,6 @@ #include #include "tensor_filter_tensorflow_lite_core.h" -#include "tensor_common.h" void init_filter_tflite (void) __attribute__ ((constructor)); void fini_filter_tflite (void) __attribute__ ((destructor)); @@ -45,28 +44,6 @@ struct _Tflite_data typedef struct _Tflite_data tflite_data; /** - * @brief nnapi hw type string - */ -static const char *nnapi_hw_string[] = { - [NNAPI_CPU] = "cpu", - [NNAPI_GPU] = "gpu", - [NNAPI_NPU] = "npu", - [NNAPI_UNKNOWN] = "unknown", - NULL -}; - -/** - * @brief return nnapi_hw type - */ -static nnapi_hw -get_nnapi_hw_type (const gchar * str) -{ - gint index = 0; - index = find_key_strv (nnapi_hw_string, str); - return (index < 0) ? NNAPI_UNKNOWN : index; -} - -/** * @brief Free privateData and move on. */ static void @@ -92,28 +69,6 @@ tflite_loadModelFile (const GstTensorFilterProperties * prop, void **private_data) { tflite_data *tf; - nnapi_hw hw = NNAPI_UNKNOWN; - - if (prop->nnapi) { - gchar **strv = NULL; - guint len; - - strv = g_strsplit (prop->nnapi, ":", 2); - len = g_strv_length (strv); - - if (g_ascii_strcasecmp (strv[0], "true") == 0) { - if (len >= 2) { - hw = get_nnapi_hw_type (strv[1]); - } else { - /** defalut hw for nnapi is CPU */ - hw = NNAPI_CPU; - } - - g_info ("NNAPI HW type: %s", nnapi_hw_string[hw]); - } - - g_strfreev (strv); - } if (*private_data != NULL) { /** @todo : Check the integrity of filter->data and filter->model_file, nnfw */ @@ -131,7 +86,7 @@ tflite_loadModelFile (const GstTensorFilterProperties * prop, return -1; } - tf->tflite_private_data = tflite_core_new (prop->model_file, hw); + tf->tflite_private_data = tflite_core_new (prop->model_file, prop->accl_str); if (tf->tflite_private_data) { if (tflite_core_init (tf->tflite_private_data)) { g_printerr ("failed to initialize the object: Tensorflow-lite"); diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.cc b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.cc index e0e0fa5..6c0041b 100644 --- a/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.cc +++ b/ext/nnstreamer/tensor_filter/tensor_filter_tensorflow_lite_core.cc @@ -36,26 +36,42 @@ #define DBG FALSE #endif +/** Match all accelerators for nnapi at once */ +#define REGEX_ACCL_NNAPI \ + "(^(true)[:]?([(]?(" \ + REGEX_ACCL_AUTO "|" \ + REGEX_ACCL_DEF "|" \ + REGEX_ACCL_CPU "|" \ + REGEX_ACCL_GPU "|" \ + REGEX_ACCL_NPU "|" \ + REGEX_ACCL_NEON ")*[)]?))" + +/** Match accelerator for nnapi one by one */ +#define REGEX_ACCL_NNAPI_ELEM \ + "(" \ + "(? &tensor_idx_list, GstTensorsInfo * tensorMeta); + + void setAccelerator (const char * accelerators); }; /** @@ -86,7 +87,7 @@ extern "C" { #endif - void *tflite_core_new (const char *_model_path, nnapi_hw hw); + void *tflite_core_new (const char *_model_path, const char *accelerators); void tflite_core_delete (void * tflite); int tflite_core_init (void * tflite); const char *tflite_core_getModelPath (void * tflite); diff --git a/gst/nnstreamer/nnstreamer_plugin_api_filter.h b/gst/nnstreamer/nnstreamer_plugin_api_filter.h index 60a4e74..912b8fb 100644 --- a/gst/nnstreamer/nnstreamer_plugin_api_filter.h +++ b/gst/nnstreamer/nnstreamer_plugin_api_filter.h @@ -26,21 +26,69 @@ #include "tensor_typedef.h" +/** Macros for accelerator types */ +#define ACCL_NONE_STR "none" +#define ACCL_AUTO_STR "auto" +#define ACCL_CPU_STR "cpu" +#define ACCL_GPU_STR "gpu" +#define ACCL_NPU_STR "npu" +#define ACCL_NEON_STR "neon" +#define ACCL_SRCN_STR "srcn" +#define ACCL_DEF_STR "default" + +#define REGEX_ACCL_EACH_ELEM(ELEM) "[^!]" ELEM "[,]?" + +#define REGEX_ACCL_AUTO REGEX_ACCL_EACH_ELEM(ACCL_AUTO_STR) +#define REGEX_ACCL_CPU REGEX_ACCL_EACH_ELEM(ACCL_CPU_STR) +#define REGEX_ACCL_GPU REGEX_ACCL_EACH_ELEM(ACCL_GPU_STR) +#define REGEX_ACCL_NPU REGEX_ACCL_EACH_ELEM(ACCL_NPU_STR) +#define REGEX_ACCL_NEON REGEX_ACCL_EACH_ELEM(ACCL_NEON_STR) +#define REGEX_ACCL_SRCN REGEX_ACCL_EACH_ELEM(ACCL_SRCN_STR) +#define REGEX_ACCL_DEF REGEX_ACCL_EACH_ELEM(ACCL_DEF_STR) + #ifdef __cplusplus extern "C" { #endif /** - * @brief nnapi hw properties. + * @brief acceleration hw properties. + * @details Enabling acceleration (choosing any accelerator hardware other + * ACCL_NONE) will enable acceleration for the framework dependent on + * the acceleration supported by the framework. + * + * For example, enabling acceleration with tflite will enable NNAPI. + * However, with NNFW will enable GPU/NEON etc. + * + * Appropriate acceleration should be used with each framework. For + * example, ACCL_NEON is only supported with NNFW tensor filter. Using + * ACCL_NEON with pytorch would result in a warning message, and + * the accelerator would fallback on ACCL_AUTO. + * + * ACCL_AUTO automatically chooses the accelerator based on the ones + * supported by the subplugin framework. However, ACCL_DEFAULT would + * use the accelerator set by the subplugin framework, if any. + * + * @note Acceleration for a framework can be enabled/disabled in the build. If + * the acceleration for a framework was disabled in the build, setting the + * accelerator while use a tensor filter with that framework will have no + * effect. */ typedef enum { - NNAPI_CPU = 0, - NNAPI_GPU = 1, - NNAPI_NPU = 2, + /**< no explicit acceleration */ + ACCL_NONE = 0, /**< no acceleration (defaults to CPU) */ + + /** Enables acceleration, 0xn000 any version of that device, 0xnxxx: device # xxx-1 */ + ACCL_AUTO = 0x1, /**< choose optimized device automatically */ + ACCL_CPU = 0x1000, /**< specify device as CPU, if possible */ + ACCL_GPU = 0x2000, /**< specify device as GPU, if possible */ + ACCL_NPU = 0x4000, /**< specify device as NPU, if possible */ + ACCL_NEON = 0x8000, /**< specify device as NEON, if possible */ + ACCL_SRCN = 0x10000, /**< specify device as SRCN, if possible */ - NNAPI_UNKNOWN -} nnapi_hw; + /** If there is no default config, and device needs to be specified, fallback to ACCL_AUTO */ + ACCL_DEFAULT = 0x20000 /**< use default device configuration by the framework*/ +} accl_hw; /** * @brief GstTensorFilter's properties for NN framework (internal data structure) @@ -62,7 +110,7 @@ typedef struct _GstTensorFilterProperties GstTensorsInfo output_meta; /**< configured output tensor info */ const char *custom_properties; /**< sub-plugin specific custom property values in string */ - const char *nnapi; + const char *accl_str; /**< accelerator configuration passed in as parameter */ } GstTensorFilterProperties; @@ -180,6 +228,18 @@ nnstreamer_filter_exit (const char *name); extern const GstTensorFilterFramework * nnstreamer_filter_find (const char *name); +/** + * @brief return accl_hw type from string + */ +extern accl_hw +get_accl_hw_type (const char * str); + +/** + * @brief return string based on accl_hw type + */ +extern const char * +get_accl_hw_str (const accl_hw key); + #ifdef __cplusplus } #endif diff --git a/gst/nnstreamer/tensor_filter/tensor_filter_common.c b/gst/nnstreamer/tensor_filter/tensor_filter_common.c index 90148de..e11ae1d 100644 --- a/gst/nnstreamer/tensor_filter/tensor_filter_common.c +++ b/gst/nnstreamer/tensor_filter/tensor_filter_common.c @@ -29,7 +29,19 @@ #include "tensor_filter_common.h" -#define REGEX_NNAPI_OPTION "(^(true|false)$|^(true|false)([:]?(cpu|gpu|npu)))" +/** + * @brief Accelerator regex verification + * @note more details in nnstreamer_plugin_api_filter.h with accl_hw + */ +#define REGEX_ACCL_ALL \ + "(^(true|false)[:]?([(]?(" \ + REGEX_ACCL_AUTO "|" \ + REGEX_ACCL_DEF "|" \ + REGEX_ACCL_CPU "|" \ + REGEX_ACCL_GPU "|" \ + REGEX_ACCL_NPU "|" \ + REGEX_ACCL_NEON "|" \ + REGEX_ACCL_SRCN ")*[)]?))" /** * @brief Free memory @@ -53,7 +65,7 @@ enum PROP_OUTPUTNAME, PROP_CUSTOM, PROP_SUBPLUGINS, - PROP_NNAPI + PROP_ACCELERATOR }; /** @@ -301,9 +313,14 @@ gst_tensor_filter_install_properties (GObjectClass * gobject_class) g_param_spec_string ("sub-plugins", "Sub-plugins", "Registrable sub-plugins list", "", G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_NNAPI, - g_param_spec_string ("nnapi", "NNAPI", - "Enable Neural Newtorks API ?", "", G_PARAM_READWRITE)); + g_object_class_install_property (gobject_class, PROP_ACCELERATOR, + g_param_spec_string ("accelerator", "ACCELERATOR", + "Set accelerator for the subplugin with format " + "(true/false):(comma separated ACCELERATOR(s)). " + "true/false determines if accelerator is to be used. " + "list of accelerators determines the backend (ignored with false). " + "Example, if GPU, NPU can be used but not CPU - true:(GPU,NPU,!CPU).", + "", G_PARAM_READWRITE)); } /** @@ -322,7 +339,7 @@ gst_tensor_filter_common_init_property (GstTensorFilterPrivate * priv) prop->input_configured = FALSE; prop->output_configured = FALSE; prop->model_file = NULL; - prop->nnapi = NULL; + prop->accl_str = NULL; prop->custom_properties = NULL; gst_tensors_info_init (&prop->input_meta); gst_tensors_info_init (&prop->output_meta); @@ -348,6 +365,7 @@ gst_tensor_filter_common_free_property (GstTensorFilterPrivate * priv) g_free_const (prop->fwname); g_free_const (prop->model_file); + g_free_const (prop->accl_str); g_free_const (prop->custom_properties); gst_tensors_info_free (&prop->input_meta); @@ -560,15 +578,34 @@ gst_tensor_filter_common_set_property (GstTensorFilterPrivate * priv, prop->custom_properties = g_value_dup_string (value); g_debug ("Custom Option = %s\n", prop->custom_properties); break; - case PROP_NNAPI: - prop->nnapi = g_value_dup_string (value); - if (!g_regex_match_simple (REGEX_NNAPI_OPTION, prop->nnapi, 0, 0)) { - g_critical - ("nnapi: \'%s\' is not valid string: it should be in the form of BOOL:ACCELLERATOR or BOOL with a regex, " - REGEX_NNAPI_OPTION "\n", prop->nnapi); + case PROP_ACCELERATOR: + { + /** + * TODO: allow updating the subplugin accelerator after it has been init + * by reopening + */ + if (priv->prop.fw_opened == TRUE) { break; } + + prop->accl_str = g_value_dup_string (value); + /** + * Perform error check on the string passed, + * parsing the string and finalizing the hw is done in each subplugin + */ + if (!g_regex_match_simple (REGEX_ACCL_ALL, prop->accl_str, + G_REGEX_CASELESS, 0)) { + g_critical + ("accelerator: \'%s\' is not valid string. " + "It should be in the form of BOOL:comma separated ACCELERATOR(s). " + "Example, if GPU, NPU can be used but not CPU - true:(GPU,NPU,!CPU)." + REGEX_ACCL_ALL "\n", prop->accl_str); + g_free_const (prop->accl_str); + prop->accl_str = NULL; + } + break; + } default: return FALSE; } @@ -711,8 +748,12 @@ gst_tensor_filter_common_get_property (GstTensorFilterPrivate * priv, g_value_take_string (value, g_string_free (subplugins, FALSE)); break; } - case PROP_NNAPI: - g_value_set_string (value, prop->nnapi); + case PROP_ACCELERATOR: + if (prop->accl_str != NULL) { + g_value_set_string (value, prop->accl_str); + } else { + g_value_set_string (value, ""); + } break; default: /* unknown property */ @@ -756,3 +797,59 @@ gst_tensor_filter_common_close_fw (GstTensorFilterPrivate * priv) priv->privateData = NULL; } } + +/** + * @brief return accl_hw type from string + * @param key The key string value + * @return Corresponding index. Returns ACCL_NONE if not found. + */ +accl_hw +get_accl_hw_type (const gchar * key) +{ + if (!g_ascii_strncasecmp (key, ACCL_AUTO_STR, strlen (ACCL_AUTO_STR))) + return ACCL_AUTO; + else if (!g_ascii_strncasecmp (key, ACCL_CPU_STR, strlen (ACCL_CPU_STR))) + return ACCL_CPU; + else if (!g_ascii_strncasecmp (key, ACCL_GPU_STR, strlen (ACCL_GPU_STR))) + return ACCL_GPU; + else if (!g_ascii_strncasecmp (key, ACCL_NPU_STR, strlen (ACCL_NPU_STR))) + return ACCL_NPU; + else if (!g_ascii_strncasecmp (key, ACCL_NEON_STR, strlen (ACCL_NEON_STR))) + return ACCL_NEON; + else if (!g_ascii_strncasecmp (key, ACCL_SRCN_STR, strlen (ACCL_SRCN_STR))) + return ACCL_SRCN; + else if (!g_ascii_strncasecmp (key, ACCL_DEF_STR, strlen (ACCL_DEF_STR))) + return ACCL_DEFAULT; + + return ACCL_NONE; /* key == "none" or Not Found */ +} + +/** + * @brief return string based on accl_hw type + * @param key The key enum value + * @return Corresponding string. Returns ACCL_NONE_STR if not found. + */ +const gchar * +get_accl_hw_str (const accl_hw key) +{ + switch (key) { + case ACCL_NONE: + return ACCL_NONE_STR; + case ACCL_AUTO: + return ACCL_AUTO_STR; + case ACCL_CPU: + return ACCL_CPU_STR; + case ACCL_GPU: + return ACCL_GPU_STR; + case ACCL_NPU: + return ACCL_NPU_STR; + case ACCL_NEON: + return ACCL_NEON_STR; + case ACCL_SRCN: + return ACCL_SRCN_STR; + case ACCL_DEFAULT: + return ACCL_DEF_STR; + default: /* Not found */ + return ACCL_NONE_STR; + } +} diff --git a/tests/nnstreamer_filter_tensorflow_lite/runTest.sh b/tests/nnstreamer_filter_tensorflow_lite/runTest.sh index 5af2c3b..5a2639f 100644 --- a/tests/nnstreamer_filter_tensorflow_lite/runTest.sh +++ b/tests/nnstreamer_filter_tensorflow_lite/runTest.sh @@ -79,23 +79,89 @@ gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! gstTest "--gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} output=1:7 outputtype=int8 ! filesink location=tensorfilter.out.log" 3F_n 0 1 $PERFORMANCE if [ -f /etc/tizen-platform.conf ] || [[ ! -z $(cat /etc/motd | grep Tizen) ]]; then + function run_pipeline() { + gst-launch-1.0 --gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} accelerator=$1 ! filesink location=tensorfilter.out.log 2>info + } + # Property reading test for nnapi - gst-launch-1.0 --gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} nnapi=true:cpu ! filesink location=tensorfilter.out.log 2> info + run_pipeline true:cpu,npu,gpu + cat info | grep "nnapi = 1, accl = cpu" + testResult $? 2-1 "NNAPI activation test" 1 1 - cat info | grep "true : cpu" - testResult $? 2-1 "NNAPI activaion test" 1 1 + # Property reading test for nnapi + run_pipeline true:!cpu + cat info | grep "nnapi = 1, accl = auto" + testResult $? 2-2 "NNAPI activation test" 1 1 # Property reading test for nnapi - gst-launch-1.0 --gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} nnapi=true ! filesink location=tensorfilter.out.log 2> info + run_pipeline true:!npu,gpu + cat info | grep "nnapi = 1, accl = gpu" + testResult $? 2-3 "NNAPI activation test" 1 1 - cat info | grep "true : cpu" - testResult $? 2-2 "NNAPI activation test" 1 1 + # Property reading test for nnapi + run_pipeline true:!npu,gpu,abcd + cat info | grep "nnapi = 1, accl = gpu" + testResult $? 2-4 "NNAPI activation test" 1 1 # Property reading test for nnapi - gst-launch-1.0 --gst-plugin-path=${PATH_TO_PLUGIN} filesrc location=${PATH_TO_IMAGE} ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,framerate=0/1 ! tensor_converter ! tensor_filter framework=tensorflow-lite model=${PATH_TO_MODEL} nnapi=true:gpu ! filesink location=tensorfilter.out.log 2> info + run_pipeline true:!npu,!abcd,gpu + cat info | grep "nnapi = 1, accl = gpu" + testResult $? 2-5 "NNAPI activation test" 1 1 - cat info | grep "true : gpu" - testResult $? 2-3 "NNAPI activation test" 1 1 + # Property reading test for nnapi + run_pipeline true:auto + cat info | grep "nnapi = 1, accl = auto" + testResult $? 2-6 "NNAPI activation test" 1 1 + + # Property reading test for nnapi + run_pipeline true:default,cpu + cat info | grep "nnapi = 1, accl = default" + testResult $? 2-7 "NNAPI activation test" 1 1 + + # Property reading test for nnapi + run_pipeline true:!cpu,default + cat info | grep "nnapi = 1, accl = default" + testResult $? 2-8 "NNAPI activation test" 1 1 + + # Property reading test for nnapi + run_pipeline true:!default + cat info | grep "nnapi = 1, accl = auto" + testResult $? 2-9 "NNAPI activation test" 1 1 + + # Property reading test for nnapi - based on ini + run_pipeline true:srcn + cat info | grep "nnapi = 1, accl = auto" + testResult $? 2-10 "NNAPI activation test" 1 1 + + # Property reading test for nnapi + run_pipeline false:abcd + cat info | grep "nnapi = 0, accl = none" + testResult $? 2-11 "NNAPI activation test" 1 1 + + # Property reading test for nnapi + run_pipeline false + cat info | grep "nnapi = 0, accl = none" + testResult $? 2-12 "NNAPI activation test" 1 1 + + # Property reading test for nnapi + run_pipeline true: + cat info | grep "nnapi = 1, accl = auto" + testResult $? 2-13 "NNAPI activation test" 1 1 + + # Property reading test for nnapi + run_pipeline true + cat info | grep "nnapi = 1, accl = auto" + testResult $? 2-14 "NNAPI activation test" 1 1 + + # Property reading test for nnapi + run_pipeline auto + cat info | grep "nnapi = 0, accl = none" + testResult $? 2-15 "NNAPI activation test" 1 1 + + # Property reading test for nnapi + run_pipeline true:!npu,abcd,gpu + cat info | grep "nnapi = 1, accl = gpu" + testResult $? 2-16 "NNAPI activation test" 1 1 fi report -- 2.7.4