From 88a184261e1acb5be16386de2c392757eed8905f Mon Sep 17 00:00:00 2001 From: InHong Han Date: Wed, 20 Jul 2022 18:58:46 +0900 Subject: [PATCH] Modified not to load wakeup engine with wake_word_detection turned off Change-Id: If55e1690071f43540e008bead090a6ecb3dc206b --- CMakeLists.txt | 1 + packaging/org.tizen.multi-assistant-service.spec | 1 + plugins/wakeup-manager/CMakeLists.txt | 1 + .../dependency-default/inc/dependency_default.h | 1 + .../dependency-default/src/dependency_default.cpp | 5 ++ plugins/wakeup-manager/inc/dependency_resolver.h | 4 ++ plugins/wakeup-manager/inc/wakeup_manager.h | 5 ++ plugins/wakeup-manager/inc/wakeup_settings.h | 8 ++- plugins/wakeup-manager/src/dependency_resolver.cpp | 31 ++++++++++ plugins/wakeup-manager/src/wakeup_manager.cpp | 69 ++++++++++++++++++---- plugins/wakeup-manager/src/wakeup_settings.cpp | 67 ++++++++++++++++++--- 11 files changed, 173 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 243649b..816b812 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ pkg_check_modules(pkgs REQUIRED multi-assistant vconf pkgmgr-info + jsoncpp ) # SET FLAG --------------------------------------------------------------------------------------- diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 9681fc8..0302c13 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -23,6 +23,7 @@ BuildRequires: pkgconfig(libxml-2.0) BuildRequires: pkgconfig(multi-assistant) BuildRequires: pkgconfig(vconf) BuildRequires: pkgconfig(pkgmgr-info) +BuildRequires: pkgconfig(jsoncpp) BuildRequires: boost-devel %if %{defined _test_type} BuildRequires: gtest-devel diff --git a/plugins/wakeup-manager/CMakeLists.txt b/plugins/wakeup-manager/CMakeLists.txt index 63db74a..170bc35 100644 --- a/plugins/wakeup-manager/CMakeLists.txt +++ b/plugins/wakeup-manager/CMakeLists.txt @@ -21,6 +21,7 @@ SET(WMPKG_CHECK_MODULES vconf pkgmgr-info multi-assistant + jsoncpp ) INCLUDE(FindPkgConfig) diff --git a/plugins/wakeup-manager/dependency-default/inc/dependency_default.h b/plugins/wakeup-manager/dependency-default/inc/dependency_default.h index d179718..e053dd4 100644 --- a/plugins/wakeup-manager/dependency-default/inc/dependency_default.h +++ b/plugins/wakeup-manager/dependency-default/inc/dependency_default.h @@ -43,6 +43,7 @@ EXPORT_API int mas_dependency_process_wakeup_engine_command(const char* engine_n EXPORT_API int mas_dependency_set_voice_key_tap_duration(float duration); EXPORT_API int mas_dependency_unset_voice_key_tap_duration(void); EXPORT_API int mas_dependency_set_voice_key_support_mode(const char* support_mode); +EXPORT_API int mas_dependency_get_custom_vconf_key(const char* default_key, char** custom_key); #ifdef __cplusplus } diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp index a742035..b5496f9 100644 --- a/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp @@ -111,4 +111,9 @@ int mas_dependency_set_voice_key_support_mode(const char* support_mode) { LOGD("support_mode : %s", support_mode); return 0; +} + +int mas_dependency_get_custom_vconf_key(const char* default_key, char** custom_key) +{ + return 0; } \ No newline at end of file diff --git a/plugins/wakeup-manager/inc/dependency_resolver.h b/plugins/wakeup-manager/inc/dependency_resolver.h index 931d323..0f4dbf6 100644 --- a/plugins/wakeup-manager/inc/dependency_resolver.h +++ b/plugins/wakeup-manager/inc/dependency_resolver.h @@ -60,6 +60,8 @@ typedef int (*mas_dependency_set_voice_key_tap_duration)(float duration); typedef int (*mas_dependency_unset_voice_key_tap_duration)(void); #define MAS_DEPENDENCY_FUNC_SET_VOICE_KEY_SUPPORT_MODE "mas_dependency_set_voice_key_support_mode" typedef int (*mas_dependency_set_voice_key_support_mode)(const char* support_mode); +#define MAS_DEPENDENCY_FUNC_GET_CUSTOM_VCONF_KEY "mas_dependency_get_custom_vconf_key" +typedef int (*mas_dependency_get_custom_vconf_key)(const char* default_key, char** custom_key); typedef struct { @@ -77,6 +79,7 @@ typedef struct { mas_dependency_set_voice_key_tap_duration set_voice_key_tap_duration; mas_dependency_unset_voice_key_tap_duration unset_voice_key_tap_duration; mas_dependency_set_voice_key_support_mode set_voice_key_support_mode; + mas_dependency_get_custom_vconf_key get_custom_vconf_key; } mas_dependency_module_interface; int dependency_resolver_initialize(mas_dependency_plugin_proxy_interface interface); @@ -93,6 +96,7 @@ int dependency_resolver_process_wakeup_candidate(mas_wakeup_event_info* info); int dependency_resolver_set_voice_key_tap_duration(float duration); int dependency_resolver_unset_voice_key_tap_duration(); int dependency_resolver_set_voice_key_support_mode(const char* support_mode); +int dependency_resolver_get_custom_vconf_key(const char* default_key, char** custom_key); mas_dependency_module_interface* dependency_resolver_get_interface(); diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index eb8f158..fda6b2a 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -37,6 +37,10 @@ namespace wakeup using namespace std; +#define WAKEUP_VCONF_KEY_DEFAULT_ASSISTANT_APPID "db/multi-assistant/default_assistant_appid" +#define WAKEUP_VCONF_KEY_ENABLED_ASSISTANTS "db/multi-assistant/enabled_assistants" +#define WAKEUP_VCONF_KEY_WAKE_WORD_DETECTION_ENABLED "db/multi-assistant/wake_word_detection_enabled" + enum class STREAMING_MODE { NONE, UTTERANCE, @@ -202,6 +206,7 @@ private: bool on_assistant_enabled_info_changed(const char* appid, bool enabled) override; bool on_default_assistant_appid_changed(const char* appid) override; bool on_multiple_mode_changed() override; + bool on_wake_word_detection_enabled_info_changed() override; void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; } private: diff --git a/plugins/wakeup-manager/inc/wakeup_settings.h b/plugins/wakeup-manager/inc/wakeup_settings.h index cb24f0d..d73437b 100644 --- a/plugins/wakeup-manager/inc/wakeup_settings.h +++ b/plugins/wakeup-manager/inc/wakeup_settings.h @@ -20,6 +20,7 @@ #include #include #include +#include typedef void (*input_language_changed_cb)(void* data); @@ -39,6 +40,7 @@ using namespace std; #define WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_PRIORITY "db/multi-assistant/wakeup_policy_priority" #define WAKEUP_SETTINGS_KEY_STREAMING_DURATION_MAX "db/multi-assistant/streaming_duration_max" #define WAKEUP_SETTINGS_KEY_VOICE_INPUT_LANGUAGE "db/multi-assistant/voice_input_language" +#define WAKEUP_SETTINGS_KEY_WAKE_WORD_DETECTION_ENABLED "db/multi-assistant/wake_word_detection_enabled" class ISettingsEventObserver { @@ -48,6 +50,7 @@ public: virtual bool on_assistant_enabled_info_changed(const char* appid, bool enabled) = 0; virtual bool on_default_assistant_appid_changed(const char* appid) = 0; virtual bool on_multiple_mode_changed() = 0; + virtual bool on_wake_word_detection_enabled_info_changed() = 0; }; class CWakeupSettings @@ -59,7 +62,7 @@ public: CWakeupSettings(const CWakeupSettings&) = delete; CWakeupSettings& operator=(const CWakeupSettings&) = delete; - void initialize(); + void initialize(map custom_keys); void deinitialize(); void subscribe(ISettingsEventObserver *observer); @@ -77,6 +80,7 @@ public: vector get_wakeup_policy_priority(); float get_streaming_duration_max(); string get_current_language(void); + vector get_wake_word_disabled_assistants(); private: vector mObservers; @@ -89,6 +93,8 @@ private: vector mWakeupPolicyPriority; // No priority by default float mStreamingDurationMax{15.0}; string mVoiceInputLanguage; + map mCustomVconfKeys; + vector mWakeWordDisabledAssistants; }; } // wakeup diff --git a/plugins/wakeup-manager/src/dependency_resolver.cpp b/plugins/wakeup-manager/src/dependency_resolver.cpp index 8569844..c8892cc 100644 --- a/plugins/wakeup-manager/src/dependency_resolver.cpp +++ b/plugins/wakeup-manager/src/dependency_resolver.cpp @@ -103,6 +103,12 @@ int dependency_resolver_initialize(mas_dependency_plugin_proxy_interface interfa g_mas_dependency.unset_voice_key_tap_duration = (mas_dependency_unset_voice_key_tap_duration)dlsym(g_handle, MAS_DEPENDENCY_FUNC_UNSET_VOICE_KEY_TAP_DURATION); + g_mas_dependency.set_voice_key_support_mode = + (mas_dependency_set_voice_key_support_mode)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_SET_VOICE_KEY_SUPPORT_MODE); + g_mas_dependency.get_custom_vconf_key = + (mas_dependency_get_custom_vconf_key)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_GET_CUSTOM_VCONF_KEY); int ret = -1; int dependency_version = 0; @@ -457,4 +463,29 @@ int dependency_resolver_set_voice_key_support_mode(const char* support_mode) } return ret; +} + +int dependency_resolver_get_custom_vconf_key(const char* default_key, char** custom_key) +{ + int ret = -1; + if (NULL != g_handle) { + mas_dependency_get_custom_vconf_key func = g_mas_dependency.get_custom_vconf_key; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_GET_CUSTOM_VCONF_KEY); + } else { + try { + ret = func(default_key, custom_key); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_GET_CUSTOM_VCONF_KEY, e.what()); + } + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to get custom vconf key, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + + return ret; } \ No newline at end of file diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 59934bd..60073e3 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -22,6 +22,7 @@ #include #include +#include #include @@ -63,6 +64,39 @@ static bool initialize_wakeup_event_info(mas_wakeup_event_info* wakeup_info) return ret; } +static map get_custom_vconf_keys() +{ + map vconf_keys; + char* custom_key = NULL; + + vconf_keys.clear(); + dependency_resolver_get_custom_vconf_key(WAKEUP_VCONF_KEY_DEFAULT_ASSISTANT_APPID, &custom_key); + vconf_keys.insert({WAKEUP_VCONF_KEY_DEFAULT_ASSISTANT_APPID, custom_key ? custom_key : WAKEUP_VCONF_KEY_DEFAULT_ASSISTANT_APPID}); + + if (custom_key) { + free(custom_key); + custom_key = nullptr; + } + + dependency_resolver_get_custom_vconf_key(WAKEUP_VCONF_KEY_ENABLED_ASSISTANTS, &custom_key); + vconf_keys.insert({WAKEUP_VCONF_KEY_ENABLED_ASSISTANTS, custom_key ? custom_key : WAKEUP_VCONF_KEY_ENABLED_ASSISTANTS}); + + if (custom_key) { + free(custom_key); + custom_key = nullptr; + } + + dependency_resolver_get_custom_vconf_key(WAKEUP_VCONF_KEY_WAKE_WORD_DETECTION_ENABLED, &custom_key); + vconf_keys.insert({WAKEUP_VCONF_KEY_WAKE_WORD_DETECTION_ENABLED, custom_key ? custom_key : WAKEUP_VCONF_KEY_WAKE_WORD_DETECTION_ENABLED}); + + if (custom_key) { + free(custom_key); + custom_key = nullptr; + } + + return vconf_keys; +} + CWakeupManager::CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingValueObserver* setting_observer) { initialize_wakeup_event_info(&mLastWakeupEventInfo); @@ -112,8 +146,16 @@ bool CWakeupManager::initialize() mAudioEventObserver.set_wakeup_manager(this); mSettingsEventObserver.set_wakeup_manager(this); + mas_dependency_plugin_proxy_interface interface; + interface.process_event = wakeup_manager_process_plugin_event; + interface.feed_audio_data = wakeup_manager_feed_audio_data; + interface.send_command = wakeup_manager_set_dependency_module_command; + interface.wakeup_assistant = wakeup_manager_wakeup_assistant; + + dependency_resolver_initialize(interface); + mWakeupSettings.subscribe(&mSettingsEventObserver); - mWakeupSettings.initialize(); + mWakeupSettings.initialize(get_custom_vconf_keys()); mAudioEventObserver.set_wakeup_engine_manager(&mWakeupEngineManager); mAudioManager.subscribe(&mAudioEventObserver); @@ -124,14 +166,6 @@ bool CWakeupManager::initialize() mAssistantConfigManager.initialize(); - mas_dependency_plugin_proxy_interface interface; - interface.process_event = wakeup_manager_process_plugin_event; - interface.feed_audio_data = wakeup_manager_feed_audio_data; - interface.send_command = wakeup_manager_set_dependency_module_command; - interface.wakeup_assistant = wakeup_manager_wakeup_assistant; - - dependency_resolver_initialize(interface); - initialize_wakeup_policy(); start_periodic_monitor_timer(); @@ -279,10 +313,15 @@ bool CWakeupManager::add_assistant_wakeup_engine(string appid, string engine) { MWR_LOGI("[ENTER] %s %s", appid.c_str(), engine.c_str()); + vector disabled_assistants = mWakeupSettings.get_wake_word_disabled_assistants(); + auto iter = find(disabled_assistants.begin(), disabled_assistants.end(), appid); + if (iter != disabled_assistants.end()) + return true; + bool found = false; if (mWakeupSettings.get_multiple_mode()) { vector assistants = mWakeupSettings.get_enabled_assistants(); - auto iter = find(assistants.begin(), assistants.end(), appid); + iter = find(assistants.begin(), assistants.end(), appid); if (iter != assistants.end()) found = true; } else { @@ -1284,5 +1323,15 @@ bool CWakeupManager::CSettingsEventObserver::on_multiple_mode_changed() return true; } +bool CWakeupManager::CSettingsEventObserver::on_wake_word_detection_enabled_info_changed() +{ + if (nullptr == mWakeupManager) return false; + vector observers = mWakeupManager->get_setting_observers(); + for (const auto& observer : observers) { + observer->on_loaded_wakeup_engine_changed(); + } + return true; +} + } // wakeup } // multiassistant diff --git a/plugins/wakeup-manager/src/wakeup_settings.cpp b/plugins/wakeup-manager/src/wakeup_settings.cpp index bb90051..be28afe 100644 --- a/plugins/wakeup-manager/src/wakeup_settings.cpp +++ b/plugins/wakeup-manager/src/wakeup_settings.cpp @@ -3,6 +3,7 @@ #include #include +#include namespace multiassistant { @@ -142,14 +143,34 @@ static void wakeup_setting_multiple_mode_changed_cb(keynode_t* node, void* data) } } -void CWakeupSettings::initialize() +static void wakeup_setting_enabled_wake_word_detection_changed_cb(keynode_t* node, void* data) +{ + MWR_LOGD("[ENTER]"); + + CWakeupSettings* settings = static_cast(data); + if (nullptr == settings) return; + + const auto& observers = settings->get_observers(); + for (const auto& observer : observers) { + if (observer) { + if (!observer->on_wake_word_detection_enabled_info_changed()) { + LOGW("[Settings WARNING] One of the observer returned false"); + } + } + } +} + +void CWakeupSettings::initialize(map custom_keys) { int vconf_ret; char *vconf_str; int vconf_bool; double vconf_double; - vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID); + mCustomVconfKeys.clear(); + mCustomVconfKeys = custom_keys; + + vconf_str = vconf_get_str(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID].c_str()); if (vconf_str) { mDefaultAssistantAppid = vconf_str; MWR_LOGD("default_assistant_appid : %s", mDefaultAssistantAppid.c_str()); @@ -179,7 +200,7 @@ void CWakeupSettings::initialize() MWR_LOGD("multiple_mode : %s", (mMultipleMode ? "true" : "false")); } if (true == mMultipleMode) { - vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS); + vconf_str = vconf_get_str(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS].c_str()); if (vconf_str) { string token; istringstream iss(vconf_str); @@ -199,6 +220,25 @@ void CWakeupSettings::initialize() vconf_str = nullptr; } } + vconf_str = vconf_get_str(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_WAKE_WORD_DETECTION_ENABLED].c_str()); + if (vconf_str) { + Json::Reader reader; + Json::Value root; + mWakeWordDisabledAssistants.clear(); + if (!reader.parse(vconf_str, root)) { + LOGW("[Settings WARNING] Failed to parse Json : %s", reader.getFormattedErrorMessages().c_str()); + } else { + auto member = root.getMemberNames(); + for (string m : member) { + if (0 == root[m].asString().compare("off")) { + mWakeWordDisabledAssistants.push_back(m); + MWR_LOGD("wake_word_detection_disabled assistant : %s", m.c_str()); + } + } + } + free(vconf_str); + vconf_str = nullptr; + } vconf_ret = vconf_get_dbl(WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_DELAY, &vconf_double); if (0 == vconf_ret) { mWakeupPolicyDelay = vconf_double; @@ -238,28 +278,32 @@ void CWakeupSettings::initialize() vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_VOICE_INPUT_LANGUAGE, wakeup_setting_input_language_changed_cb, this); - vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID, + vconf_notify_key_changed(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID].c_str(), wakeup_setting_default_assistant_appid_changed_cb, this); if (true == mMultipleMode) { - vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS, + vconf_notify_key_changed(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS].c_str(), wakeup_setting_enabled_assistants_changed_cb, this); } vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_MULTIPLE_MODE, wakeup_setting_multiple_mode_changed_cb, this); + vconf_notify_key_changed(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_WAKE_WORD_DETECTION_ENABLED].c_str(), + wakeup_setting_enabled_wake_word_detection_changed_cb, this); } void CWakeupSettings::deinitialize() { vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_VOICE_INPUT_LANGUAGE, wakeup_setting_input_language_changed_cb); - vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID, + vconf_ignore_key_changed(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID].c_str(), wakeup_setting_default_assistant_appid_changed_cb); if (true == mMultipleMode) { - vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS, + vconf_ignore_key_changed(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS].c_str(), wakeup_setting_enabled_assistants_changed_cb); } vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_MULTIPLE_MODE, wakeup_setting_multiple_mode_changed_cb); + vconf_ignore_key_changed(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_WAKE_WORD_DETECTION_ENABLED].c_str(), + wakeup_setting_enabled_wake_word_detection_changed_cb); } void CWakeupSettings::subscribe(ISettingsEventObserver *observer) @@ -283,7 +327,7 @@ const vector& CWakeupSettings::get_observers() string CWakeupSettings::get_default_assistant_appid() { char *vconf_str; - vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID); + vconf_str = vconf_get_str(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID].c_str()); MWR_LOGD("vconf default_assistant_appid : %s", vconf_str); if (vconf_str) { mDefaultAssistantAppid = vconf_str; @@ -301,7 +345,7 @@ void CWakeupSettings::set_default_assistant_appid(std::string appid) MWR_LOGE("Default assistant appid not changed, ignoring..."); return; } - int ret = vconf_set_str(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID, appid.c_str()); + int ret = vconf_set_str(mCustomVconfKeys[WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID].c_str(), appid.c_str()); MWR_LOGD("default_assistant_appid : %s, %d", appid.c_str(), ret); mDefaultAssistantAppid = appid; } @@ -352,5 +396,10 @@ std::string CWakeupSettings::get_current_language(void) return result; } +vector CWakeupSettings::get_wake_word_disabled_assistants() +{ + return mWakeWordDisabledAssistants; +} + } // wakeup } // multiassistant -- 2.7.4