[Android] prepare dsp/npu runtime for snpe filter

- Add required snpe-sdk libs
- Set `jniLibs.srcDirs` to this project explictly contains snpe-sdk libs for dsp/npu
- Make function `nns_set_env` to set env "ADSP_LIBRARY_PATH" (https://developer.qualcomm.com/docs/snpe/dsp_runtime.html)
- Log SNPE version
- Prevent build SNAP and SNPE together in android build script and
  Android.mk

Signed-off-by: Yongjoo Ahn <yongjoo1.ahn@samsung.com>

diff --git a/api/android/api/build.gradle b/api/android/api/build.gradle
index 0436257..80b8fc1 100644 (file)
--- a/api/android/api/build.gradle
+++ b/api/android/api/build.gradle
@@ -60,6 +60,18 @@ android {
     }
     productFlavors {
     }
+    sourceSets {
+        main {
+            if (project.hasProperty('SNPE_DSP_LIBRARY_PATH')) {
+                jniLibs.srcDirs += project.properties['SNPE_DSP_LIBRARY_PATH']
+                println 'Set jniLibs.srcDirs includes libraries for SNPE DSP runtime'
+            }
+        }
+    }
+    packagingOptions {
+        // To do not show build warning messages
+        doNotStrip "*/arm64-v8a/*_skel.so"
+    }
 }
 
 dependencies {

diff --git a/api/android/api/src/main/jni/Android-snpe-prebuilt.mk b/api/android/api/src/main/jni/Android-snpe-prebuilt.mk
index 6d450f8..9f5132e 100644 (file)
--- a/api/android/api/src/main/jni/Android-snpe-prebuilt.mk
+++ b/api/android/api/src/main/jni/Android-snpe-prebuilt.mk
@@ -20,3 +20,27 @@ LOCAL_MODULE := libSNPE
 LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libSNPE.so
 include $(PREBUILT_SHARED_LIBRARY)
 SNPE_PREBUILT_LIBS += libSNPE
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := libsnpe_adsp
+LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libsnpe_adsp.so
+include $(PREBUILT_SHARED_LIBRARY)
+SNPE_PREBUILT_LIBS += libsnpe_adsp
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := libsnpe_dsp_domains
+LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libsnpe_dsp_domains.so
+include $(PREBUILT_SHARED_LIBRARY)
+SNPE_PREBUILT_LIBS += libsnpe_dsp_domains
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := libsnpe_dsp_domains_v2
+LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libsnpe_dsp_domains_v2.so
+include $(PREBUILT_SHARED_LIBRARY)
+SNPE_PREBUILT_LIBS += libsnpe_dsp_domains_v2
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := libhta
+LOCAL_SRC_FILES := $(SNPE_LIB_PATH)/libhta.so
+include $(PREBUILT_SHARED_LIBRARY)
+SNPE_PREBUILT_LIBS += libhta

diff --git a/api/android/api/src/main/jni/Android-snpe.mk b/api/android/api/src/main/jni/Android-snpe.mk
index abc66d7..e43f7b7 100644 (file)
--- a/api/android/api/src/main/jni/Android-snpe.mk
+++ b/api/android/api/src/main/jni/Android-snpe.mk
@@ -18,7 +18,6 @@ SNPE_INCLUDES := $(SNPE_DIR)/include/zdl/
 
 ifeq ($(TARGET_ARCH_ABI),arm64-v8a)
 SNPE_LIB_PATH := $(SNPE_DIR)/lib/aarch64-android-clang6.0
-SNPE_DSP_LIB_PATH := $(SNPE_DIR)/lib/dsp
 else
 $(error Target arch ABI not supported: $(TARGET_ARCH_ABI))
 endif
@@ -26,7 +25,7 @@ endif
 #------------------------------------------------------
 # snpe-sdk (prebuilt shared library)
 #------------------------------------------------------
-include $(LOCAL_PATH)/Android-snap-prebuilt.mk
+include $(LOCAL_PATH)/Android-snpe-prebuilt.mk
 
 #------------------------------------------------------
 # tensor-filter sub-plugin for snpe

diff --git a/api/android/api/src/main/jni/Android.mk b/api/android/api/src/main/jni/Android.mk
index 8345068..cdf4078 100644 (file)
--- a/api/android/api/src/main/jni/Android.mk
+++ b/api/android/api/src/main/jni/Android.mk
@@ -37,6 +37,12 @@ ENABLE_NNFW := false
 # SNPE (Snapdragon Neural Processing Engine)
 ENABLE_SNPE := false
 
+ifeq ($(ENABLE_SNAP),true)
+  ifeq ($(ENABLE_SNPE),true)
+   $(error DO NOT enable SNAP and SNPE both. The app would fail to use DSP or NPU runtime.)
+  endif
+endif
+
 # Common libraries for sub-plugins
 NNS_API_LIBS := nnstreamer gstreamer_android
 NNS_API_FLAGS :=

diff --git a/api/android/api/src/main/jni/nnstreamer-native-api.c b/api/android/api/src/main/jni/nnstreamer-native-api.c
index 53db076..1b1a6f2 100644 (file)
--- a/api/android/api/src/main/jni/nnstreamer-native-api.c
+++ b/api/android/api/src/main/jni/nnstreamer-native-api.c
@@ -637,6 +637,118 @@ nns_get_nnfw_type (jint fw_type, ml_nnfw_type_e * nnfw)
 }
 
 /**
+ * @brief Set additional environment
+ */
+static gboolean
+nns_set_env (JNIEnv * env, jobject context)
+{
+  gboolean ret = TRUE;
+
+#if defined (ENABLE_SNPE)
+  gboolean snpe_failed = TRUE;
+
+  jclass context_class = NULL;
+  jmethodID get_application_info_method_id = NULL;
+  jobject application_info_object = NULL;
+  jclass application_info_object_class = NULL;
+  jfieldID native_library_dir_field_id = NULL;
+  jstring native_library_dir_path = NULL;
+
+  const gchar *native_library_dir_path_str;
+  gchar *new_path;
+
+  context_class = (*env)->GetObjectClass (env, context);
+  if (!context_class) {
+    nns_loge ("Failed to get context class.");
+    goto snpe_done;
+  }
+
+  get_application_info_method_id = (*env)->GetMethodID (env, context_class,
+      "getApplicationInfo", "()Landroid/content/pm/ApplicationInfo;");
+  if (!get_application_info_method_id) {
+    nns_loge ("Failed to get method ID for `ApplicationInfo()`.");
+    goto snpe_done;
+  }
+
+  application_info_object = (*env)->CallObjectMethod (env, context,
+      get_application_info_method_id);
+  if ((*env)->ExceptionCheck (env)) {
+    (*env)->ExceptionDescribe (env);
+    (*env)->ExceptionClear (env);
+    nns_loge ("Failed to call method `ApplicationInfo()`.");
+    goto snpe_done;
+  }
+
+  application_info_object_class = (*env)->GetObjectClass (env,
+      application_info_object);
+  if (!application_info_object_class) {
+    nns_loge ("Failed to get `ApplicationInfo` object class");
+    goto snpe_done;
+  }
+
+  native_library_dir_field_id = (*env)->GetFieldID (env,
+      application_info_object_class,
+      "nativeLibraryDir", "Ljava/lang/String;");
+  if (!native_library_dir_field_id) {
+    nns_loge ("Failed to get field ID for `nativeLibraryDir`.");
+    goto snpe_done;
+  }
+
+  native_library_dir_path = (*env)->GetObjectField (env,
+      application_info_object, native_library_dir_field_id);
+  if (!native_library_dir_path) {
+    nns_loge ("Failed to get field `nativeLibraryDir`.");
+    goto snpe_done;
+  }
+
+  native_library_dir_path_str = (*env)->GetStringUTFChars (env,
+      native_library_dir_path, NULL);
+  if ((*env)->ExceptionCheck (env)) {
+    (*env)->ExceptionDescribe (env);
+    (*env)->ExceptionClear (env);
+    nns_loge ("Failed to get string `nativeLibraryDir`");
+    goto snpe_done;
+  }
+
+  new_path = g_strconcat (native_library_dir_path_str,
+      ";/system/lib/rfsa/adsp;/system/vendor/lib/rfsa/adsp;/dsp", NULL);
+
+  /* See https://developer.qualcomm.com/docs/snpe/dsp_runtime.html for details */
+  nns_logi ("Set env ADSP_LIBRARY_PATH for snpe DSP/AIP runtime: %s",
+      new_path);
+  g_setenv ("ADSP_LIBRARY_PATH", new_path, TRUE);
+
+  g_free (new_path);
+  (*env)->ReleaseStringUTFChars (env, native_library_dir_path,
+      native_library_dir_path_str);
+
+  snpe_failed = FALSE;
+  
+snpe_done:
+  ret = ret && !(snpe_failed);
+
+  if (native_library_dir_path) {
+    (*env)->DeleteLocalRef (env, native_library_dir_path);
+  }
+
+  if (application_info_object_class) {
+    (*env)->DeleteLocalRef (env, application_info_object_class);
+  }
+
+  if (application_info_object) {
+    (*env)->DeleteLocalRef (env, application_info_object);
+  }
+
+  if (context_class) {
+    (*env)->DeleteLocalRef (env, context_class);
+  }
+
+#endif
+
+  return ret;
+}
+
+/**
  * @brief Initialize NNStreamer, register required plugins.
  */
 jboolean
@@ -657,6 +769,11 @@ nnstreamer_native_initialize (JNIEnv * env, jobject context)
     goto done;
   }
 
+  if (!nns_set_env (env, context)) {
+    nns_loge ("Failed to set extra env");
+    goto done;
+  }
+
   if (nns_is_initilaized == FALSE) {
     /* register nnstreamer plugins */
 #if !defined (NNS_SINGLE_ONLY)

diff --git a/api/android/build-android-lib.sh b/api/android/build-android-lib.sh
index 1ea2e8a..f486e07 100644 (file)
--- a/api/android/build-android-lib.sh
+++ b/api/android/build-android-lib.sh
@@ -153,6 +153,7 @@ if [[ $enable_nnfw == "yes" ]]; then
 fi
 
 if [[ $enable_snpe == "yes" ]]; then
+    [ $enable_snap == "yes" ] && echo "DO NOT enable SNAP and SNPE both. The app would fail to use DSP or NPU runtime." && exit 1
     [ -z "$SNPE_DIRECTORY" ] && echo "Need to set SNPE_DIRECTORY, to build sub-plugin for SNPE." && exit 1
     [ $target_abi != "arm64-v8a" ] && echo "Set target ABI arm64-v8a to build sub-plugin for SNPE." && exit 1
 
@@ -267,9 +268,11 @@ fi
 if [[ $enable_snpe == "yes" ]]; then
     sed -i "s|ENABLE_SNPE := false|ENABLE_SNPE := true|" external/Android-nnstreamer-prebuilt.mk
     sed -i "s|ENABLE_SNPE := false|ENABLE_SNPE := true|" api/src/main/jni/Android.mk
-    mkdir api/src/main/jni/snpe
+    sed -i "$ a SNPE_DSP_LIBRARY_PATH=src/main/jni/snpe/lib/dsp" gradle.properties
+    mkdir -p api/src/main/jni/snpe/lib/dsp/arm64-v8a
     cp -r $SNPE_DIRECTORY/include api/src/main/jni/snpe
-    cp -r $SNPE_DIRECTORY/lib api/src/main/jni/snpe
+    cp -r $SNPE_DIRECTORY/lib/aarch64-android-clang6.0 api/src/main/jni/snpe/lib/aarch64-android-clang6.0
+    cp $SNPE_DIRECTORY/lib/dsp/libsnpe*.so api/src/main/jni/snpe/lib/dsp/arm64-v8a
 fi
 
 # Update tf-lite option

diff --git a/ext/nnstreamer/tensor_filter/tensor_filter_snpe.cc b/ext/nnstreamer/tensor_filter/tensor_filter_snpe.cc
index fb5bc0d..5c98893 100644 (file)
--- a/ext/nnstreamer/tensor_filter/tensor_filter_snpe.cc
+++ b/ext/nnstreamer/tensor_filter/tensor_filter_snpe.cc
@@ -136,6 +136,9 @@ tensor_filter_subplugin & snpe_subplugin::getEmptyInstance ()
 
 void snpe_subplugin::configure_instance (const GstTensorFilterProperties *prop)
 {
+  nns_logi ("SNPE Version: %s",
+    zdl::SNPE::SNPEFactory::getLibraryVersion ().asString ().c_str ());
+
   if (!empty_model) {
     /* Already opend */
 
@@ -282,6 +285,10 @@ void snpe_subplugin::fini_filter_snpe (void)
 
 void _init_filter_snpe ()
 {
+  if (nnstreamer_filter_find ("snap")) {
+    nns_loge ("Cannot use SNPE and SNAP both. Won't register this SNPE subplugin.");
+    return;
+  }
   snpe_subplugin::init_filter_snpe ();
 }