From 79c1809285eebca8aa422292898fa21914021e84 Mon Sep 17 00:00:00 2001 From: Yongjoo Ahn Date: Fri, 15 May 2020 10:15:48 +0900 Subject: [PATCH] [Android] prepare dsp/npu runtime for snpe filter - Add required snpe-sdk libs - Set `jniLibs.srcDirs` to this project explictly contains snpe-sdk libs for dsp/npu - Make function `nns_set_env` to set env "ADSP_LIBRARY_PATH" (https://developer.qualcomm.com/docs/snpe/dsp_runtime.html) - Log SNPE version - Prevent build SNAP and SNPE together in android build script and Android.mk Signed-off-by: Yongjoo Ahn --- api/android/api/build.gradle | 12 +++ .../api/src/main/jni/Android-snpe-prebuilt.mk | 24 +++++ api/android/api/src/main/jni/Android-snpe.mk | 3 +- api/android/api/src/main/jni/Android.mk | 6 ++ .../api/src/main/jni/nnstreamer-native-api.c | 117 +++++++++++++++++++++ api/android/build-android-lib.sh | 7 +- ext/nnstreamer/tensor_filter/tensor_filter_snpe.cc | 7 ++ 7 files changed, 172 insertions(+), 4 deletions(-) diff --git a/api/android/api/build.gradle b/api/android/api/build.gradle index 0436257..80b8fc1 100644 --- a/api/android/api/build.gradle +++ b/api/android/api/build.gradle @@ -60,6 +60,18 @@ android { } productFlavors { } + sourceSets { + main { + if (project.hasProperty('SNPE_DSP_LIBRARY_PATH')) { + jniLibs.srcDirs += project.properties['SNPE_DSP_LIBRARY_PATH'] + println 'Set jniLibs.srcDirs includes libraries for SNPE DSP runtime' + } + } + } + packagingOptions { + // To do not show build warning messages + doNotStrip "*/arm64-v8a/*_skel.so" + } } dependencies { diff --git a/api/android/api/src/main/jni/Android-snpe-prebuilt.mk b/api/android/api/src/main/jni/Android-snpe-prebuilt.mk index 6d450f8..9f5132e 100644 --- a/api/android/api/src/main/jni/Android-snpe-prebuilt.mk +++ b/api/android/api/src/main/jni/Android-snpe-prebuilt.mk @@ -20,3 +20,27 @@ LOCAL_MODULE := libSNPE LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libSNPE.so include $(PREBUILT_SHARED_LIBRARY) SNPE_PREBUILT_LIBS += libSNPE + +include $(CLEAR_VARS) +LOCAL_MODULE := libsnpe_adsp +LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libsnpe_adsp.so +include $(PREBUILT_SHARED_LIBRARY) +SNPE_PREBUILT_LIBS += libsnpe_adsp + +include $(CLEAR_VARS) +LOCAL_MODULE := libsnpe_dsp_domains +LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libsnpe_dsp_domains.so +include $(PREBUILT_SHARED_LIBRARY) +SNPE_PREBUILT_LIBS += libsnpe_dsp_domains + +include $(CLEAR_VARS) +LOCAL_MODULE := libsnpe_dsp_domains_v2 +LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libsnpe_dsp_domains_v2.so +include $(PREBUILT_SHARED_LIBRARY) +SNPE_PREBUILT_LIBS += libsnpe_dsp_domains_v2 + +include $(CLEAR_VARS) +LOCAL_MODULE := libhta +LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libhta.so +include $(PREBUILT_SHARED_LIBRARY) +SNPE_PREBUILT_LIBS += libhta diff --git a/api/android/api/src/main/jni/Android-snpe.mk b/api/android/api/src/main/jni/Android-snpe.mk index abc66d7..e43f7b7 100644 --- a/api/android/api/src/main/jni/Android-snpe.mk +++ b/api/android/api/src/main/jni/Android-snpe.mk @@ -18,7 +18,6 @@ SNPE_INCLUDES := $(SNPE_DIR)/include/zdl/ ifeq ($(TARGET_ARCH_ABI),arm64-v8a) SNPE_LIB_PATH := $(SNPE_DIR)/lib/aarch64-android-clang6.0 -SNPE_DSP_LIB_PATH := $(SNPE_DIR)/lib/dsp else $(error Target arch ABI not supported: $(TARGET_ARCH_ABI)) endif @@ -26,7 +25,7 @@ endif #------------------------------------------------------ # snpe-sdk (prebuilt shared library) #------------------------------------------------------ -include $(LOCAL_PATH)/Android-snap-prebuilt.mk +include $(LOCAL_PATH)/Android-snpe-prebuilt.mk #------------------------------------------------------ # tensor-filter sub-plugin for snpe diff --git a/api/android/api/src/main/jni/Android.mk b/api/android/api/src/main/jni/Android.mk index 8345068..cdf4078 100644 --- a/api/android/api/src/main/jni/Android.mk +++ b/api/android/api/src/main/jni/Android.mk @@ -37,6 +37,12 @@ ENABLE_NNFW := false # SNPE (Snapdragon Neural Processing Engine) ENABLE_SNPE := false +ifeq ($(ENABLE_SNAP),true) + ifeq ($(ENABLE_SNPE),true) + $(error DO NOT enable SNAP and SNPE both. The app would fail to use DSP or NPU runtime.) + endif +endif + # Common libraries for sub-plugins NNS_API_LIBS := nnstreamer gstreamer_android NNS_API_FLAGS := diff --git a/api/android/api/src/main/jni/nnstreamer-native-api.c b/api/android/api/src/main/jni/nnstreamer-native-api.c index 53db076..1b1a6f2 100644 --- a/api/android/api/src/main/jni/nnstreamer-native-api.c +++ b/api/android/api/src/main/jni/nnstreamer-native-api.c @@ -637,6 +637,118 @@ nns_get_nnfw_type (jint fw_type, ml_nnfw_type_e * nnfw) } /** + * @brief Set additional environment + */ +static gboolean +nns_set_env (JNIEnv * env, jobject context) +{ + gboolean ret = TRUE; + +#if defined (ENABLE_SNPE) + gboolean snpe_failed = TRUE; + + jclass context_class = NULL; + jmethodID get_application_info_method_id = NULL; + jobject application_info_object = NULL; + jclass application_info_object_class = NULL; + jfieldID native_library_dir_field_id = NULL; + jstring native_library_dir_path = NULL; + + const gchar *native_library_dir_path_str; + gchar *new_path; + + context_class = (*env)->GetObjectClass (env, context); + if (!context_class) { + nns_loge ("Failed to get context class."); + goto snpe_done; + } + + get_application_info_method_id = (*env)->GetMethodID (env, context_class, + "getApplicationInfo", "()Landroid/content/pm/ApplicationInfo;"); + if (!get_application_info_method_id) { + nns_loge ("Failed to get method ID for `ApplicationInfo()`."); + goto snpe_done; + } + + application_info_object = (*env)->CallObjectMethod (env, context, + get_application_info_method_id); + if ((*env)->ExceptionCheck (env)) { + (*env)->ExceptionDescribe (env); + (*env)->ExceptionClear (env); + nns_loge ("Failed to call method `ApplicationInfo()`."); + goto snpe_done; + } + + application_info_object_class = (*env)->GetObjectClass (env, + application_info_object); + if (!application_info_object_class) { + nns_loge ("Failed to get `ApplicationInfo` object class"); + goto snpe_done; + } + + native_library_dir_field_id = (*env)->GetFieldID (env, + application_info_object_class, + "nativeLibraryDir", "Ljava/lang/String;"); + if (!native_library_dir_field_id) { + nns_loge ("Failed to get field ID for `nativeLibraryDir`."); + goto snpe_done; + } + + native_library_dir_path = (*env)->GetObjectField (env, + application_info_object, native_library_dir_field_id); + if (!native_library_dir_path) { + nns_loge ("Failed to get field `nativeLibraryDir`."); + goto snpe_done; + } + + native_library_dir_path_str = (*env)->GetStringUTFChars (env, + native_library_dir_path, NULL); + if ((*env)->ExceptionCheck (env)) { + (*env)->ExceptionDescribe (env); + (*env)->ExceptionClear (env); + nns_loge ("Failed to get string `nativeLibraryDir`"); + goto snpe_done; + } + + new_path = g_strconcat (native_library_dir_path_str, + ";/system/lib/rfsa/adsp;/system/vendor/lib/rfsa/adsp;/dsp", NULL); + + /* See https://developer.qualcomm.com/docs/snpe/dsp_runtime.html for details */ + nns_logi ("Set env ADSP_LIBRARY_PATH for snpe DSP/AIP runtime: %s", + new_path); + g_setenv ("ADSP_LIBRARY_PATH", new_path, TRUE); + + g_free (new_path); + (*env)->ReleaseStringUTFChars (env, native_library_dir_path, + native_library_dir_path_str); + + snpe_failed = FALSE; + +snpe_done: + ret = ret && !(snpe_failed); + + if (native_library_dir_path) { + (*env)->DeleteLocalRef (env, native_library_dir_path); + } + + if (application_info_object_class) { + (*env)->DeleteLocalRef (env, application_info_object_class); + } + + if (application_info_object) { + (*env)->DeleteLocalRef (env, application_info_object); + } + + if (context_class) { + (*env)->DeleteLocalRef (env, context_class); + } + +#endif + + return ret; +} + +/** * @brief Initialize NNStreamer, register required plugins. */ jboolean @@ -657,6 +769,11 @@ nnstreamer_native_initialize (JNIEnv * env, jobject context) goto done; } + if (!nns_set_env (env, context)) { + nns_loge ("Failed to set extra env"); + goto done; + } + if (nns_is_initilaized == FALSE) { /* register nnstreamer plugins */ #if !defined (NNS_SINGLE_ONLY) diff --git a/api/android/build-android-lib.sh b/api/android/build-android-lib.sh index 1ea2e8a..f486e07 100644 --- a/api/android/build-android-lib.sh +++ b/api/android/build-android-lib.sh @@ -153,6 +153,7 @@ if [[ $enable_nnfw == "yes" ]]; then fi if [[ $enable_snpe == "yes" ]]; then + [ $enable_snap == "yes" ] && echo "DO NOT enable SNAP and SNPE both. The app would fail to use DSP or NPU runtime." && exit 1 [ -z "$SNPE_DIRECTORY" ] && echo "Need to set SNPE_DIRECTORY, to build sub-plugin for SNPE." && exit 1 [ $target_abi != "arm64-v8a" ] && echo "Set target ABI arm64-v8a to build sub-plugin for SNPE." && exit 1 @@ -267,9 +268,11 @@ fi if [[ $enable_snpe == "yes" ]]; then sed -i "s|ENABLE_SNPE := false|ENABLE_SNPE := true|" external/Android-nnstreamer-prebuilt.mk sed -i "s|ENABLE_SNPE := false|ENABLE_SNPE := true|" api/src/main/jni/Android.mk - mkdir api/src/main/jni/snpe + sed -i "$ a SNPE_DSP_LIBRARY_PATH=src/main/jni/snpe/lib/dsp" gradle.properties + mkdir -p api/src/main/jni/snpe/lib/dsp/arm64-v8a cp -r $SNPE_DIRECTORY/include api/src/main/jni/snpe - cp -r $SNPE_DIRECTORY/lib api/src/main/jni/snpe + cp -r $SNPE_DIRECTORY/lib/aarch64-android-clang6.0 api/src/main/jni/snpe/lib/aarch64-android-clang6.0 + cp $SNPE_DIRECTORY/lib/dsp/libsnpe*.so api/src/main/jni/snpe/lib/dsp/arm64-v8a fi # Update tf-lite option diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_snpe.cc b/ext/nnstreamer/tensor_filter/tensor_filter_snpe.cc index fb5bc0d..5c98893 100644 --- a/ext/nnstreamer/tensor_filter/tensor_filter_snpe.cc +++ b/ext/nnstreamer/tensor_filter/tensor_filter_snpe.cc @@ -136,6 +136,9 @@ tensor_filter_subplugin & snpe_subplugin::getEmptyInstance () void snpe_subplugin::configure_instance (const GstTensorFilterProperties *prop) { + nns_logi ("SNPE Version: %s", + zdl::SNPE::SNPEFactory::getLibraryVersion ().asString ().c_str ()); + if (!empty_model) { /* Already opend */ @@ -282,6 +285,10 @@ void snpe_subplugin::fini_filter_snpe (void) void _init_filter_snpe () { + if (nnstreamer_filter_find ("snap")) { + nns_loge ("Cannot use SNPE and SNAP both. Won't register this SNPE subplugin."); + return; + } snpe_subplugin::init_filter_snpe (); } -- 2.7.4