From 871c00fcc97b27edfb84f723407a84286fae69a8 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 29 Oct 2019 16:03:40 +0900 Subject: [PATCH 01/16] Prelaunch default assistant on setting change event Change-Id: I1a8f96a408bff8668d81020876cdebad0623e969 --- inc/multi_assistant_service.h | 2 + inc/multi_assistant_service_plugin.h | 3 + inc/multi_wakeup_recognizer.h | 2 + plugins/wakeup-manager/inc/wakeup_manager.h | 22 +++++-- .../wakeup-manager/inc/wakeup_manager_wrapper.h | 4 ++ plugins/wakeup-manager/src/wakeup_manager.cpp | 67 ++++++++++++++++------ .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 37 +++++++++++- src/multi_assistant_service.c | 28 +++++---- src/multi_assistant_service_plugin.c | 32 +++++++++++ 9 files changed, 164 insertions(+), 33 deletions(-) diff --git a/inc/multi_assistant_service.h b/inc/multi_assistant_service.h index 0bbc37a..775245d 100644 --- a/inc/multi_assistant_service.h +++ b/inc/multi_assistant_service.h @@ -81,6 +81,8 @@ int mas_set_current_client_by_appid(const char *appid); int mas_launch_client_by_wakeup_word(const char *wakeup_word); +int mas_prelaunch_default_assistant(); + typedef enum { CLIENT_LAUNCH_MODE_ACTIVATION, CLIENT_LAUNCH_MODE_PRELAUNCH, diff --git a/inc/multi_assistant_service_plugin.h b/inc/multi_assistant_service_plugin.h index 7bfd761..dbfd673 100644 --- a/inc/multi_assistant_service_plugin.h +++ b/inc/multi_assistant_service_plugin.h @@ -169,6 +169,8 @@ typedef int (*wakeup_manager_set_previous_utterance_streaming_callback)(wakeup_s typedef int (*wakeup_manager_set_follow_up_streaming_callback)(wakeup_service_speech_streaming_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_SPEECH_STATUS_CALLBACK "wakeup_manager_set_speech_status_callback" typedef int (*wakeup_manager_set_speech_status_callback)(wakeup_service_speech_status_cb callback, void* user_data); +#define MA_WAKEUP_MANAGER_FUNC_SET_SETTING_CHANGED_CALLBACK "wakeup_manager_set_setting_changed_callback" +typedef int (*wakeup_manager_set_setting_changed_callback)(wakeup_service_setting_changed_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_ERROR_CALLBACK "wakeup_manager_set_error_callback" typedef int (*wakeup_manager_set_error_callback)(wakeup_service_error_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK "wakeup_manager_set_streaming_section_changed_callback" @@ -205,6 +207,7 @@ typedef struct { wakeup_manager_set_previous_utterance_streaming_callback set_previous_utterance_streaming_callback; wakeup_manager_set_follow_up_streaming_callback set_follow_up_streaming_callback; wakeup_manager_set_speech_status_callback set_speech_status_callback; + wakeup_manager_set_setting_changed_callback set_setting_changed_callback; wakeup_manager_set_error_callback set_error_callback; wakeup_manager_set_streaming_section_changed_callback set_streaming_section_changed_callback; } wakeup_manager_interface; diff --git a/inc/multi_wakeup_recognizer.h b/inc/multi_wakeup_recognizer.h index 6c3eb6f..b0732f9 100644 --- a/inc/multi_wakeup_recognizer.h +++ b/inc/multi_wakeup_recognizer.h @@ -99,6 +99,8 @@ typedef void (*wakeup_service_speech_streaming_cb)(wakeup_speech_streaming_event typedef void (*wakeup_service_speech_status_cb)(wakeup_speech_status_e status, void *user_data); +typedef void (*wakeup_service_setting_changed_cb)(void *user_data); + typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* user_data); typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index e6f5251..d574646 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -50,6 +50,12 @@ public: virtual void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0; }; +class ISettingValueObserver { +public: + virtual ~ISettingValueObserver() = default; + virtual void on_value_changed() = 0; +}; + /* If a wakeup event is raised by pressing a voice key, the wakeup_engine attribute of wakeup_event_info will have the following value */ #define WAKEUP_ENGINE_VOICE_KEY "voice_key" @@ -57,7 +63,7 @@ public: class CWakeupManager { public: - CWakeupManager(IWakeupEventObserver *observer); + CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingValueObserver* setting_observer); virtual ~CWakeupManager(); CWakeupManager(const CWakeupManager&) = delete; @@ -66,8 +72,11 @@ public: bool initialize(); bool deinitialize(); - void subscribe(IWakeupEventObserver *observer); - void unsubscribe(IWakeupEventObserver *observer); + void subscribe_wakeup_observer(IWakeupEventObserver* observer); + void unsubscribe_wakeup_observer(IWakeupEventObserver* observer); + + void subscribe_setting_observer(ISettingValueObserver* observer); + void unsubscribe_setting_observer(ISettingValueObserver* observer); bool activate(); bool deactivate(); @@ -107,9 +116,11 @@ public: CAudioManager* get_audio_manager(); CWakeupSettings* get_wakeup_settings(); - vector get_observers(); + vector get_wakeup_observers(); void set_last_wakeup_event_info(wakeup_event_info wakeup_info); + vector get_setting_observers(); + bool change_manager_state(wakeup_manager_state_e state); wakeup_manager_state_e get_manager_state(); @@ -178,7 +189,8 @@ private: map mAssistantSupportsCurrentLanguage; map mAssistantEnabled; - vector mObservers; + vector mWakeupObservers; + vector mSettingObservers; unique_ptr mWakeupPolicy; diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index 7f408dc..2df503e 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -63,6 +63,8 @@ typedef void (*wakeup_service_speech_streaming_cb)(wakeup_speech_streaming_event typedef void (*wakeup_service_speech_status_cb)(wakeup_service_speech_status_e status, void *user_data); +typedef void (*wakeup_service_setting_changed_cb)( void *user_data); + typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* user_data); typedef void (*wakeup_service_audio_data_require_status_cb)(bool require, void* user_data); @@ -146,6 +148,8 @@ EXPORT_API int wakeup_manager_set_follow_up_streaming_callback(wakeup_service_sp EXPORT_API int wakeup_manager_set_speech_status_callback(wakeup_service_speech_status_cb callback, void* user_data); +EXPORT_API int wakeup_manager_set_setting_changed_callback(wakeup_service_setting_changed_cb callback, void* user_data); + EXPORT_API int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* user_data); EXPORT_API int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streaming_section_changed_cb callback, void* user_data); diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 81ed6eb..9e16d36 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -54,12 +54,15 @@ static bool initialize_wakeup_event_info(wakeup_event_info* wakeup_info) return ret; } -CWakeupManager::CWakeupManager(IWakeupEventObserver *observer) +CWakeupManager::CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingValueObserver* setting_observer) { initialize_wakeup_event_info(&mLastWakeupEventInfo); - if (observer) { - subscribe(observer); + if (wakeup_observer) { + subscribe_wakeup_observer(wakeup_observer); + } + if (setting_observer) { + subscribe_setting_observer(setting_observer); } } @@ -136,16 +139,29 @@ bool CWakeupManager::deinitialize() return true; } -void CWakeupManager::subscribe(IWakeupEventObserver *observer) +void CWakeupManager::subscribe_wakeup_observer(IWakeupEventObserver *observer) { - mObservers.push_back(observer); + mWakeupObservers.push_back(observer); } -void CWakeupManager::unsubscribe(IWakeupEventObserver *observer) +void CWakeupManager::unsubscribe_wakeup_observer(IWakeupEventObserver *observer) { - auto iter = find(mObservers.begin(), mObservers.end(), observer); - if (iter != mObservers.end()) { - mObservers.erase(iter); + auto iter = find(mWakeupObservers.begin(), mWakeupObservers.end(), observer); + if (iter != mWakeupObservers.end()) { + mWakeupObservers.erase(iter); + } +} + +void CWakeupManager::subscribe_setting_observer(ISettingValueObserver* observer) +{ + mSettingObservers.push_back(observer); +} + +void CWakeupManager::unsubscribe_setting_observer(ISettingValueObserver* observer) +{ + auto iter = find(mSettingObservers.begin(), mSettingObservers.end(), observer); + if (iter != mSettingObservers.end()) { + mSettingObservers.erase(iter); } } @@ -469,7 +485,7 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) set_last_wakeup_event_info(wakeup_info); mWakeupEngineManager.set_selected_wakeup_info(wakeup_info); - for (const auto& observer : mObservers) { + for (const auto& observer : mWakeupObservers) { observer->on_wakeup(wakeup_info); } } @@ -498,9 +514,9 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) return true; } -vector CWakeupManager::get_observers() +vector CWakeupManager::get_wakeup_observers() { - return mObservers; + return mWakeupObservers; } void CWakeupManager::set_last_wakeup_event_info(wakeup_event_info wakeup_info) @@ -508,6 +524,11 @@ void CWakeupManager::set_last_wakeup_event_info(wakeup_event_info wakeup_info) mLastWakeupEventInfo = wakeup_info; } +vector CWakeupManager::get_setting_observers() +{ + return mSettingObservers; +} + static Eina_Bool streaming_duration_expired(void *data) { MWR_LOGD("[ENTER]"); @@ -534,7 +555,7 @@ static Eina_Bool streaming_duration_expired(void *data) } unsigned char final_buffer[2] = {'\0', }; - vector observers = wakeup_manager->get_observers(); + vector observers = wakeup_manager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_streaming_audio_data( WAKEUP_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer)); @@ -841,7 +862,7 @@ bool CWakeupManager::CEngineEventObserver::on_streaming_audio_data( { if (nullptr == mWakeupManager) return false; - vector observers = mWakeupManager->get_observers(); + vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_streaming_audio_data(event, buffer, len); } @@ -857,7 +878,7 @@ bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section( { if (nullptr == mWakeupManager) return false; - vector observers = mWakeupManager->get_observers(); + vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_audio_streaming_data_section(section); } @@ -878,7 +899,7 @@ void CWakeupManager::CPolicyEventObserver::on_wakeup(wakeup_event_info wakeup_in mWakeupManager->set_last_wakeup_event_info(wakeup_info); engine_manager->set_selected_wakeup_info(wakeup_info); - vector observers = mWakeupManager->get_observers(); + vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_wakeup(wakeup_info); } @@ -906,7 +927,7 @@ bool CWakeupManager::CAudioEventObserver::on_streaming_audio_data( { if (nullptr == mWakeupManager) return false; - vector observers = mWakeupManager->get_observers(); + vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_streaming_audio_data(event, buffer, len); } @@ -922,6 +943,10 @@ bool CWakeupManager::CSettingsEventObserver::on_voice_input_language_changed( { if (nullptr == mWakeupManager || nullptr == language) return false; mWakeupManager->set_language(std::string(language)); + vector observers = mWakeupManager->get_setting_observers(); + for (const auto& observer : observers) { + observer->on_value_changed(); + } return true; } @@ -930,6 +955,10 @@ bool CWakeupManager::CSettingsEventObserver::on_assistant_enabled_info_changed( { if (nullptr == mWakeupManager || nullptr == appid) return false; mWakeupManager->set_assistant_enabled(std::string(appid), enabled); + vector observers = mWakeupManager->get_setting_observers(); + for (const auto& observer : observers) { + observer->on_value_changed(); + } return true; } @@ -938,6 +967,10 @@ bool CWakeupManager::CSettingsEventObserver::on_default_assistant_appid_changed( { if (nullptr == mWakeupManager || nullptr == appid) return false; mWakeupManager->set_default_assistant(std::string(appid)); + vector observers = mWakeupManager->get_setting_observers(); + for (const auto& observer : observers) { + observer->on_value_changed(); + } return true; } diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 9853c34..1c72602 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -34,6 +34,9 @@ static void* g_follow_up_streaming_user_data; static wakeup_service_speech_status_cb g_speech_status_cb; static void* g_speech_status_user_data; +static wakeup_service_setting_changed_cb g_setting_changed_cb; +static void* g_setting_changed_user_data; + static wakeup_service_error_cb g_error_cb; static void* g_error_user_data; @@ -48,8 +51,14 @@ class CWakeupEventObserver : public IWakeupEventObserver void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override; }; +class CSettingValueObserver : public ISettingValueObserver +{ + void on_value_changed() override; +}; + static CWakeupEventObserver g_wakeup_event_observer; -static CWakeupManager g_wakeup_manager(&g_wakeup_event_observer); +static CSettingValueObserver g_setting_value_observer; +static CWakeupManager g_wakeup_manager(&g_wakeup_event_observer, &g_setting_value_observer); int wakeup_manager_initialize(void) { @@ -67,6 +76,9 @@ int wakeup_manager_initialize(void) g_speech_status_cb = NULL; g_speech_status_user_data = NULL; + g_setting_changed_cb = NULL; + g_setting_changed_user_data = NULL; + g_error_cb = NULL; g_error_user_data = NULL; @@ -481,6 +493,22 @@ int wakeup_manager_set_speech_status_callback(wakeup_service_speech_status_cb ca return 0; } +int wakeup_manager_set_setting_changed_callback(wakeup_service_setting_changed_cb callback, void* user_data) +{ + MWR_LOGD("[ENTER]"); + + if (NULL == callback) { + MWR_LOGE("[ERROR] Input parameter is NULL"); + return -1; + } + + g_setting_changed_cb = callback; + g_setting_changed_user_data = user_data; + + MWR_LOGD("[END]"); + return 0; +} + int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* user_data) { MWR_LOGD("[ENTER]"); @@ -549,3 +577,10 @@ void CWakeupEventObserver::on_audio_streaming_data_section( g_streaming_section_changed_cb(section, g_streaming_section_changed_user_data); } } + +void CSettingValueObserver::on_value_changed() +{ + if (g_setting_changed_cb) { + g_setting_changed_cb(g_setting_changed_user_data); + } +} \ No newline at end of file diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index b2da67b..35cc1ed 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -762,16 +762,7 @@ static int init_wakeup(void) #endif } - /* CHECK NEEDED : should the code segment below and activation logic above be moved to wakeup manger? */ - int prelaunch_mode; - int res = vconf_get_bool(WAKEUP_SETTINGS_KEY_PRELAUNCH_MODE, &prelaunch_mode); - if (0 == res && 0 != prelaunch_mode) { - const char *default_assistant = NULL; - if (0 == multi_assistant_service_plugin_get_default_assistant(&default_assistant)) { - MAS_LOGD("prelaunching default_assistant_appid : %s", default_assistant); - mas_launch_client_by_appid(default_assistant, CLIENT_LAUNCH_MODE_PRELAUNCH); - } - } + mas_prelaunch_default_assistant(); /* For the case of preprocessing assistant, it always have to be launched beforehand */ char *vconf_str; @@ -1080,6 +1071,23 @@ int mas_launch_client_by_wakeup_word(const char *wakeup_word) return mas_launch_client_by_appid(appid, CLIENT_LAUNCH_MODE_ACTIVATION); } +int mas_prelaunch_default_assistant() +{ + /* CHECK NEEDED : should the code segment below and activation logic above be moved to wakeup manger? */ + int prelaunch_mode; + int res = vconf_get_bool(WAKEUP_SETTINGS_KEY_PRELAUNCH_MODE, &prelaunch_mode); + if (0 == res && 0 != prelaunch_mode) { + const char *default_assistant = NULL; + if (0 == multi_assistant_service_plugin_get_default_assistant(&default_assistant)) { + if (0 == aul_app_is_running(default_assistant)) { + MAS_LOGD("prelaunching default_assistant_appid : %s", default_assistant); + mas_launch_client_by_appid(default_assistant, CLIENT_LAUNCH_MODE_PRELAUNCH); + } + } + } + return 0; +} + int mas_process_voice_key_event(bool pressed) { diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 5e9e063..8307104 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -337,6 +337,12 @@ static void __error_cb(int error, const char* err_msg, void* user_data) } } +static void __setting_changed_cb(void *user_data) +{ + mas_prelaunch_default_assistant(); + MAS_LOGD( "[SUCCESS] __setting_changed_cb is called"); +} + static void __streaming_section_changed_cb(ma_audio_streaming_data_section_e section, void* user_data) { MAS_LOGD( "[SUCCESS] __streaming_section_changed_cb is called, section(%d)", section); @@ -454,6 +460,9 @@ int multi_assistant_service_plugin_initialize(void) _wakeup_manager_interface.set_speech_status_callback = (wakeup_manager_set_speech_status_callback)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_SPEECH_STATUS_CALLBACK); + _wakeup_manager_interface.set_setting_changed_callback = + (wakeup_manager_set_setting_changed_callback)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_SET_SETTING_CHANGED_CALLBACK); _wakeup_manager_interface.set_error_callback = (wakeup_manager_set_error_callback)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_ERROR_CALLBACK); @@ -996,6 +1005,12 @@ int multi_assistant_service_plugin_set_callbacks(void) return ret; } + ret = multi_assistant_service_plugin_set_setting_changed_callback(__setting_changed_cb, NULL); + if (0 != ret) { + MAS_LOGE("Fail to set setting changed cb"); + return ret; + } + ret = multi_assistant_service_plugin_set_error_callback(__error_cb, NULL); if (0 != ret) { MAS_LOGE("Fail to set error cb"); @@ -1104,6 +1119,23 @@ int multi_assistant_service_plugin_set_speech_status_callback(wakeup_service_spe return ret; } +int multi_assistant_service_plugin_set_setting_changed_callback(wakeup_service_setting_changed_cb callback, void* user_data) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_set_setting_changed_callback func = _wakeup_manager_interface.set_setting_changed_callback; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_SETTING_CHANGED_CALLBACK); + } else { + ret = func(callback, user_data); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set setting_changed callback, ret(%d)", ret); + } + } + } + return ret; +} + int multi_assistant_service_plugin_set_error_callback(wakeup_service_error_cb callback, void* user_data) { int ret = -1; -- 2.7.4 From 02784ac4a8b72a15134b394c2bbee278eb27333c Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 29 Oct 2019 16:04:40 +0900 Subject: [PATCH 02/16] Bump version to 0.2.6 Change-Id: I0d42ae4371b25e7e24e8cd4329251ba39f672fe9 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 483d407..15f683c 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 9f783fc..127eeb5 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.5 +Version: 0.2.6 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 11a6ddfc6830fce8f1850c0dbab91a0dc614286e Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 31 Oct 2019 20:16:18 +0900 Subject: [PATCH 03/16] Finalize audio data when voice key is in pressed state When audio is finalized after voice key variable is set to false, the FINISH audio event will be fed to wakeup engines where the event itself is not useful for wakeup engines. Change-Id: Ifdd41be5e76a84b779d697edefd57203ac2bfe5a --- plugins/wakeup-manager/src/wakeup_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 9e16d36..1ac4694 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -491,8 +491,8 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) } } else if (MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_PUSH == event) { if (mVoiceKeyPressed != false) { - mVoiceKeyPressed = false; mAudioManager.finalize_audio_data(); + mVoiceKeyPressed = false; if (mWakeupEngineManager.get_audio_data_required()) { /* Restart recorder thread using appropriate recording device */ -- 2.7.4 From 199d497dec43a6f3b1a20844c64bf9feccd4bd55 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 31 Oct 2019 20:20:28 +0900 Subject: [PATCH 04/16] Change manager state right away on voice key release Change-Id: I97f074bfca65f0af8e770eae675dbf46fbb8713d --- plugins/wakeup-manager/src/wakeup_manager.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 1ac4694..12b82a8 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -494,6 +494,12 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) mAudioManager.finalize_audio_data(); mVoiceKeyPressed = false; + if (STREAMING_MODE::UTTERANCE == mStreamingMode) { + change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING); + } else { + change_manager_state(WAKEUP_MANAGER_STATE_LISTENING); + } + if (mWakeupEngineManager.get_audio_data_required()) { /* Restart recorder thread using appropriate recording device */ mAudioManager.stop_recording(true); -- 2.7.4 From d9c3427e05d7a12872ee2161071aa60494493bea Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 31 Oct 2019 20:22:42 +0900 Subject: [PATCH 05/16] Stop streaming only when necessary Streaming stop function was being called inside wakeup_manager state change function, which results invoking unnecessary streaming stop functions in many cases. Change-Id: I723b187375ce3a8d6c4b4933d6577c7a9be758d2 --- plugins/wakeup-manager/src/wakeup_manager.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 12b82a8..ece33b9 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -195,6 +195,10 @@ bool CWakeupManager::deactivate(void) mAudioManager.stop_recording(true); change_manager_state(WAKEUP_MANAGER_STATE_INACTIVE); + stop_streaming_utterance_data(); + stop_streaming_previous_utterance_data(); + stop_streaming_follow_up_data(); + MWR_LOGD("[END]"); return true; } @@ -356,15 +360,6 @@ bool CWakeupManager::set_streaming_mode(STREAMING_MODE mode) bool CWakeupManager::change_manager_state(wakeup_manager_state_e state) { - if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState || - WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState) { - if (WAKEUP_MANAGER_STATE_VOICE_FEEDBACK == state || - WAKEUP_MANAGER_STATE_LISTENING == state) { - stop_streaming_utterance_data(); - stop_streaming_previous_utterance_data(); - stop_streaming_follow_up_data(); - } - } mWakeupManagerState = state; mWakeupEngineManager.update_manager_state(state); return true; @@ -431,6 +426,9 @@ bool CWakeupManager::update_recognition_result(string appid, int result) if (WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState || WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { change_manager_state(WAKEUP_MANAGER_STATE_LISTENING); + stop_streaming_utterance_data(); + stop_streaming_previous_utterance_data(); + stop_streaming_follow_up_data(); } MWR_LOGD("[END]"); return true; -- 2.7.4 From f3721c04a7476621d70fc65f4ae35447a8fde0d8 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 31 Oct 2019 20:25:41 +0900 Subject: [PATCH 06/16] Bump version to 0.2.7 Change-Id: If861f52ac4920f04326c8a399adffa7c4f58c9bb --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 15f683c..d18e9c7 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 127eeb5..1dca5f5 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.6 +Version: 0.2.7 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From a2aa3a6561b314fbd9bda2d2996deebb9b59d62a Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 16:00:41 +0900 Subject: [PATCH 07/16] Apply secure options to compile flags Change-Id: Ia582efe7e1b7d7beb6b1fe989ada3f4f47ef6559 --- CMakeLists.txt | 4 ++-- plugins/wakeup-manager/CMakeLists.txt | 2 +- plugins/wakeup-manager/dependency-default/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59845fa..1cf3c04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,8 +36,8 @@ FOREACH(flag ${pkgs_CFLAGS}) ENDFOREACH(flag) SET(CMAKE_EXE_LINKER_FLAGS "-Wl,--as-needed -pie") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CFLAGS} -fPIE -std=c++11 -fvisibility=hidden -fvisibility-inlines-hidden") -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_CFLAGS} -fPIE -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIE -pie -std=c++11 -fvisibility=hidden -fvisibility-inlines-hidden") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_CFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIE -pie -fvisibility=hidden") # SET WARNNING OPTION ---------------------------------------------------------------------------- SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") diff --git a/plugins/wakeup-manager/CMakeLists.txt b/plugins/wakeup-manager/CMakeLists.txt index 752f085..7f07524 100644 --- a/plugins/wakeup-manager/CMakeLists.txt +++ b/plugins/wakeup-manager/CMakeLists.txt @@ -58,7 +58,7 @@ SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -fPIC -Wall" ) IF("${_TV_PRODUCT}" STREQUAL "TRUE") SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -DTV_PRODUCT") ENDIF() -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -std=c++11 -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIC -std=c++11 -fvisibility=hidden") SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") ADD_LIBRARY(${PROJECT_NAME} SHARED ${SRCS} ) diff --git a/plugins/wakeup-manager/dependency-default/CMakeLists.txt b/plugins/wakeup-manager/dependency-default/CMakeLists.txt index 163ede8..a08730d 100644 --- a/plugins/wakeup-manager/dependency-default/CMakeLists.txt +++ b/plugins/wakeup-manager/dependency-default/CMakeLists.txt @@ -49,7 +49,7 @@ ENDFOREACH(flag) SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -fPIC -Wall" ) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -std=c++11 -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIC -std=c++11 -fvisibility=hidden") SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") ADD_LIBRARY(${PROJECT_NAME} SHARED ${SRCS} ) -- 2.7.4 From dc1e7118f4ca47e7772e334d24de5c84c16a4654 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 16:01:59 +0900 Subject: [PATCH 08/16] Bump version to 0.2.8 Change-Id: I14024040e9cc66a2687f7bba6099f39c2f0dcc8f --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index d18e9c7..3e31401 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 1dca5f5..486ab55 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.7 +Version: 0.2.8 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 905b2b79394c84db6ac5f2051a6702f1e253e97f Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 4 Nov 2019 11:32:56 +0900 Subject: [PATCH 09/16] Add reload feature for updating pluings on-the-fly Change-Id: Ibf873b8861b22807f9b51f5f8e92efb82f8520ba --- .../src/dependency_default_audio.cpp | 36 ++++++- .../wakeup-manager/src/wakeup_audio_manager.cpp | 12 +++ .../wakeup-manager/src/wakeup_engine_manager.cpp | 7 ++ .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 117 +++++++++++++++------ src/multi_assistant_service.c | 54 +++++++--- src/multi_assistant_service_plugin.c | 2 + 6 files changed, 184 insertions(+), 44 deletions(-) diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp index e803739..d1662c8 100644 --- a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp @@ -17,6 +17,7 @@ static mas_proxy_interface g_proxy_interface; using namespace std; static audio_in_h g_audio_in = NULL; +static audio_io_state_e g_audio_io_state = AUDIO_IO_STATE_IDLE; static sound_stream_info_h g_volume_stream = NULL; static virtual_sound_stream_h g_virtual_sound_stream = NULL; @@ -111,6 +112,13 @@ static void _device_connection_changed_cb(sound_device_h device, bool isConnecte return; } +static void _audio_in_state_changed_cb(audio_in_h handle, audio_io_state_e previous, audio_io_state_e current, bool by_policy, void *user_data) +{ + if (handle == g_audio_in) { + g_audio_io_state = current; + } +} + void dependency_default_audio_initialize(mas_proxy_interface interfaces) { const int rate = 16000; @@ -155,12 +163,24 @@ void dependency_default_audio_initialize(mas_proxy_interface interfaces) return; } + ret = audio_in_set_state_changed_cb(g_audio_in, _audio_in_state_changed_cb, NULL); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGE("[Recorder ERROR] Fail to set state changed callback : %d", ret); + sound_manager_remove_device_connection_changed_cb(g_device_id); + sound_manager_destroy_stream_information(g_stream_info); + g_stream_info = NULL; + audio_in_destroy(g_audio_in); + g_audio_in = NULL; + return; + } + ret = audio_in_set_sound_stream_info(g_audio_in, g_stream_info); if (AUDIO_IO_ERROR_NONE != ret) { LOGE("[Recorder ERROR] Fail to set stream info : %d", ret); sound_manager_remove_device_connection_changed_cb(g_device_id); sound_manager_destroy_stream_information(g_stream_info); g_stream_info = NULL; + audio_in_unset_state_changed_cb(g_audio_in); audio_in_destroy(g_audio_in); g_audio_in = NULL; return; @@ -177,6 +197,7 @@ void dependency_default_audio_initialize(mas_proxy_interface interfaces) sound_manager_remove_device_connection_changed_cb(g_device_id); sound_manager_destroy_stream_information(g_stream_info); g_stream_info = NULL; + audio_in_unset_state_changed_cb(g_audio_in); audio_in_destroy(g_audio_in); g_audio_in = NULL; return; @@ -231,6 +252,11 @@ void dependency_default_audio_deinitialize() } if (g_audio_in) { + ret = audio_in_unset_state_changed_cb(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGD("[Recorder ERROR] Fail to unset state changed callback : %d", ret); + } + ret = audio_in_destroy(g_audio_in); if (AUDIO_IO_ERROR_NONE != ret) { LOGD("[Recorder ERROR] Fail to destroy audio : %d", ret); @@ -271,7 +297,10 @@ static void recorder_thread_func() void dependency_default_audio_start_recording() { if (g_audio_in) { - int ret = audio_in_resume(g_audio_in); + int ret = AUDIO_IO_ERROR_NONE; + if (AUDIO_IO_STATE_RUNNING != g_audio_io_state) { + ret = audio_in_resume(g_audio_in); + } if (AUDIO_IO_ERROR_NONE != ret) { LOGD("[Recorder ERROR] Fail to resume audio in : %d", ret); } @@ -289,7 +318,10 @@ void dependency_default_audio_stop_recording() g_recorder_thread.join(); } if (g_audio_in) { - int ret = audio_in_pause(g_audio_in); + int ret = AUDIO_IO_ERROR_NONE; + if (AUDIO_IO_STATE_RUNNING == g_audio_io_state) { + ret = audio_in_pause(g_audio_in); + } if (AUDIO_IO_ERROR_NONE != ret) { LOGD("[Recorder ERROR] Fail to pause audio in : %d", ret); } diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp index 8b68362..94ce56e 100644 --- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp @@ -68,6 +68,18 @@ int CAudioManager::initialize(void) int CAudioManager::deinitialize(void) { + clear_audio_data(); + if (mStreamingThread.joinable()) { + MWR_LOGD("mStreamingThread is joinable, trying join()"); + mStopStreamingThread.store(true); + try { + mStreamingThread.join(); + } catch (std::exception &e) { + MWR_LOGE("Exception thrown : %s", e.what()); + } + } + mStopStreamingThread.store(false); + sound_manager_remove_focus_state_watch_cb(mSoundFocusWatchId); return 0; diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index 5e1163c..ebc50a9 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -76,6 +76,13 @@ void CWakeupEngineManager::initialize() void CWakeupEngineManager::deinitialize() { + if (mStreamingThread.joinable()) { + MWR_LOGD("mStreamingThread is joinable, trying join()"); + mStopStreamingThread.store(true); + mStreamingThread.join(); + } + mStopStreamingThread.store(false); + for (auto& info : mEngineInfo) { try { if (info.interface.set_wakeup_event_callback) { diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 1c72602..77549bb 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -17,6 +17,8 @@ #include "wakeup_manager_main.h" #include "wakeup_manager.h" +#include + using namespace multiassistant::wakeup; static wakeup_service_wakeup_event_cb g_wakeup_event_cb; @@ -56,14 +58,29 @@ class CSettingValueObserver : public ISettingValueObserver void on_value_changed() override; }; -static CWakeupEventObserver g_wakeup_event_observer; -static CSettingValueObserver g_setting_value_observer; -static CWakeupManager g_wakeup_manager(&g_wakeup_event_observer, &g_setting_value_observer); +static std::unique_ptr g_wakeup_event_observer; +static std::unique_ptr g_setting_value_observer; +static std::unique_ptr g_wakeup_manager; int wakeup_manager_initialize(void) { MWR_LOGD("[ENTER]"); + try { + g_wakeup_event_observer.reset(new CWakeupEventObserver); + g_setting_value_observer.reset(new CSettingValueObserver); + if (NULL == g_wakeup_event_observer || NULL == g_setting_value_observer) { + return -1; + } + g_wakeup_manager.reset(new CWakeupManager(g_wakeup_event_observer.get(), g_setting_value_observer.get())); + if (NULL == g_wakeup_manager) { + return -1; + } + } catch (const std::exception& e) { + MWR_LOGE("Exception caught : %s", e.what()); + return -1; + } + g_wakeup_event_cb = NULL; g_wakeup_event_user_data = NULL; @@ -82,7 +99,7 @@ int wakeup_manager_initialize(void) g_error_cb = NULL; g_error_user_data = NULL; - g_wakeup_manager.initialize(); + g_wakeup_manager->initialize(); MWR_LOGD("[END]"); return 0; @@ -92,7 +109,18 @@ int wakeup_manager_deinitialize(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.deinitialize(); + if (nullptr == g_wakeup_manager) return -1; + + try { + g_wakeup_manager->deinitialize(); + g_wakeup_manager.reset(); + + g_wakeup_event_observer.reset(); + g_setting_value_observer.reset(); + } catch (const std::exception& e) { + MWR_LOGE("Exception caught : %s", e.what()); + return -1; + } MWR_LOGD("[END]"); return 0; @@ -104,7 +132,9 @@ int wakeup_manager_get_settings(ma_plugin_settings **settings, size_t *struct_si MWR_LOGD("[ERROR] Parameter is invalid, settings(%p), struct_size(%p)", settings, struct_size); return -1; } - CWakeupSettings* wakeup_settings = g_wakeup_manager.get_wakeup_settings(); + + if (nullptr == g_wakeup_manager) return -1; + CWakeupSettings* wakeup_settings = g_wakeup_manager->get_wakeup_settings(); if (wakeup_settings) { const int PLUGIN_VERSION = 1; static ma_plugin_settings current_settings; @@ -125,7 +155,9 @@ int wakeup_manager_add_assistant_wakeup_word(const char* appid, const char* wake MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), wakeup_word(%s), language(%s)", appid, wakeup_word, language); return -1; } - g_wakeup_manager.add_assistant_wakeup_word( + + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->add_assistant_wakeup_word( string{appid}, string{wakeup_word}, (language ? string{language} : string{})); MWR_LOGD("[END]"); @@ -141,7 +173,8 @@ int wakeup_manager_add_assistant_language(const char* appid, const char* languag return -1; } - g_wakeup_manager.add_assistant_language(string{appid}, string{language}); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->add_assistant_language(string{appid}, string{language}); MWR_LOGD("[DEBUG] language(%s)", language); @@ -158,8 +191,9 @@ int wakeup_manager_set_assistant_wakeup_engine(const char* appid, const char* en return -1; } + if (nullptr == g_wakeup_manager) return -1; MWR_LOGD("[DEBUG] appid(%s), wakeup engine(%s)", appid, engine); - g_wakeup_manager.set_assistant_wakeup_engine(string{appid}, string{engine}); + g_wakeup_manager->set_assistant_wakeup_engine(string{appid}, string{engine}); MWR_LOGD("[END]"); return 0; @@ -174,8 +208,9 @@ int wakeup_manager_set_default_assistant(const char* appid) return -1; } + if (nullptr == g_wakeup_manager) return -1; MWR_LOGD("[DEBUG] default_assistant appid(%s)", appid); - g_wakeup_manager.set_default_assistant(string{appid}); + g_wakeup_manager->set_default_assistant(string{appid}); MWR_LOGD("[END]"); return 0; @@ -184,7 +219,9 @@ int wakeup_manager_set_default_assistant(const char* appid) int wakeup_manager_get_default_assistant(const char** appid) { static string default_assistant; - default_assistant = g_wakeup_manager.get_default_assistant(); + + if (nullptr == g_wakeup_manager) return -1; + default_assistant = g_wakeup_manager->get_default_assistant(); if (default_assistant.empty()) { *appid = nullptr; @@ -203,7 +240,8 @@ int wakeup_manager_set_language(const char* language) return -1; } - g_wakeup_manager.set_language(string{language}); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_language(string{language}); MWR_LOGD("[END]"); return 0; @@ -211,7 +249,8 @@ int wakeup_manager_set_language(const char* language) int wakeup_manager_change_state(wakeup_manager_state_e state) { - g_wakeup_manager.change_manager_state(state); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->change_manager_state(state); return 0; } @@ -219,7 +258,8 @@ int wakeup_manager_activate(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.activate(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->activate(); MWR_LOGD("[END]"); return 0; @@ -229,7 +269,8 @@ int wakeup_manager_deactivate(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.deactivate(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->deactivate(); MWR_LOGD("[END]"); return 0; @@ -246,7 +287,8 @@ int wakeup_manager_update_voice_feedback_state(const char* appid, int state) appid_string = appid; } - g_wakeup_manager.update_voice_feedback_state(appid_string, state); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->update_voice_feedback_state(appid_string, state); MWR_LOGD("[END]"); return 0; @@ -264,7 +306,8 @@ int wakeup_manager_send_assistant_specific_command(const char* appid, const char if (appid) appid_string = appid; if (command) command_string = command; - g_wakeup_manager.send_assistant_specific_command(appid_string, command_string); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->send_assistant_specific_command(appid_string, command_string); MWR_LOGD("[END]"); return 0; @@ -281,7 +324,8 @@ int wakeup_manager_set_background_volume(const char* appid, double ratio) appid_string = appid; } - g_wakeup_manager.set_background_volume(appid_string, ratio); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_background_volume(appid_string, ratio); MWR_LOGD("[END]"); return 0; @@ -297,7 +341,9 @@ int wakeup_manager_update_recognition_result(const char* appid, int result) } else { appid_string = appid; } - g_wakeup_manager.update_recognition_result(appid_string, result); + + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->update_recognition_result(appid_string, result); MWR_LOGD("[END]"); return 0; @@ -307,7 +353,8 @@ int wakeup_manager_process_event(int event, void* data, int len) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.process_event(static_cast(event), data, len); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->process_event(static_cast(event), data, len); MWR_LOGD("[END]"); return 0; @@ -317,7 +364,8 @@ int wakeup_manager_start_streaming_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.start_streaming_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->start_streaming_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -327,7 +375,8 @@ int wakeup_manager_stop_streaming_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.stop_streaming_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->stop_streaming_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -337,7 +386,8 @@ int wakeup_manager_start_streaming_follow_up_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.start_streaming_follow_up_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->start_streaming_follow_up_data(); MWR_LOGD("[END]"); return 0; @@ -347,7 +397,8 @@ int wakeup_manager_stop_streaming_follow_up_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.stop_streaming_follow_up_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->stop_streaming_follow_up_data(); MWR_LOGD("[END]"); return 0; @@ -357,7 +408,8 @@ int wakeup_manager_start_streaming_previous_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.start_streaming_previous_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->start_streaming_previous_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -367,7 +419,8 @@ int wakeup_manager_stop_streaming_previous_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.stop_streaming_previous_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->stop_streaming_previous_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -382,7 +435,8 @@ int wakeup_manager_get_audio_format(int* rate, int* channel, int* audio_type) return -1; } - g_wakeup_manager.get_audio_format(rate, channel, audio_type); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->get_audio_format(rate, channel, audio_type); MWR_LOGD("[END] rate(%d), channel(%d), audio_type(%d)", *rate, *channel, *audio_type); return 0; @@ -397,7 +451,8 @@ int wakeup_manager_get_audio_source_type(char** type) return -1; } - g_wakeup_manager.get_audio_source_type(type); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->get_audio_source_type(type); MWR_LOGD("[END] type(%s)", *type); return 0; @@ -407,7 +462,8 @@ int wakeup_manager_set_wake_word_audio_require_flag(bool require) { MWR_LOGD("[ENTER] : %d", require); - g_wakeup_manager.set_wake_word_audio_require_flag(require); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_wake_word_audio_require_flag(require); MWR_LOGD("[END]"); return 0; @@ -543,7 +599,8 @@ int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streami int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len) { - g_wakeup_manager.feed_audio_data(event, buffer, len); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->feed_audio_data(event, buffer, len); return 0; } diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 35cc1ed..137c7de 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -677,15 +677,8 @@ static void mas_active_state_changed_cb(keynode_t* key, void* data) } } -static int init_wakeup(void) +static int init_plugin(void) { - MAS_LOGD("[Enter] init_wakeup"); - - int ret = mas_dbus_open_connection(); - if (0 != ret) { - MAS_LOGE("[ERROR] Fail to open connection"); - } - if (0 != multi_assistant_service_plugin_initialize()) { MAS_LOGE("Fail to ws intialize"); return -1; @@ -740,6 +733,18 @@ static int init_wakeup(void) return -1; } + return 0; +} + +static int deinit_plugin(void) +{ + if (0 != multi_assistant_service_plugin_deinitialize()) { + MAS_LOGE("Fail to deinitialize"); + } +} + +static int process_multi_assistant_activated_setting() +{ if (0 == vconf_notify_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb, NULL)) { /* Activate / deactivate according to the vconf key setting */ mas_active_state_changed_cb(NULL, NULL); @@ -761,6 +766,21 @@ static int init_wakeup(void) } #endif } + return 0; +} + +static int init_wakeup(void) +{ + MAS_LOGD("[Enter] init_wakeup"); + + int ret = mas_dbus_open_connection(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to open connection"); + } + + init_plugin(); + + process_multi_assistant_activated_setting(); mas_prelaunch_default_assistant(); @@ -786,6 +806,7 @@ static void deinit_wakeup(void) g_current_lang = NULL; } */ + deinit_plugin(); vconf_ignore_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb); @@ -793,10 +814,6 @@ static void deinit_wakeup(void) if (0 != ret) { MAS_LOGE("[ERROR] Fail to close connection"); } - - if (0 != multi_assistant_service_plugin_deinitialize()) { - MAS_LOGE("Fail to ws deinitialize"); - } } int mas_get_current_client_pid() @@ -1246,6 +1263,19 @@ void service_app_terminate(void *data) void service_app_control(app_control_h app_control, void *data) { // Todo: add your code here. + char *operation = NULL; + int ret = app_control_get_operation(app_control, &operation); + if (ret == APP_CONTROL_ERROR_NONE && operation) { + //strncmp(operation, APP_CONTROL_OPERATION_LAUNCH_ON_EVENT, strlen(operation)) == 0){ + char* value = NULL; + app_control_get_extra_data(app_control, "event", &value); + LOGD("operation : %s , %s", operation, value); + if (value && 0 == strcmp(value, "reload")) { + deinit_plugin(); + init_plugin(); + process_multi_assistant_activated_setting(); + } + } return; } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 8307104..b9f5f7a 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -511,6 +511,8 @@ int multi_assistant_service_plugin_deinitialize(void) MAS_LOGD("[Recorder ERROR] File not found!"); } #endif + multi_assistant_service_plugin_deactivate(); + int ret = -1; if (NULL != g_handle) { wakeup_manager_deinitialize func = _wakeup_manager_interface.deinitialize; -- 2.7.4 From 1393f5ccc126eee00404c22dbacadff58b93f9c4 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 19:07:56 +0900 Subject: [PATCH 10/16] Reload plugin modules depending on pkgmgr event Change-Id: I5c12c6600b80f2657e371b15c4eeac13c0044722 --- CMakeLists.txt | 1 + src/multi_assistant_service.c | 151 +++++++++++++++++++++++++++++++---- src/multi_assistant_service_plugin.c | 1 - 3 files changed, 137 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1cf3c04..1717bcf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ pkg_check_modules(pkgs REQUIRED libxml-2.0 multi-assistant vconf + pkgmgr-info ) # SET FLAG --------------------------------------------------------------------------------------- diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 137c7de..fc2f3cb 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -70,6 +72,7 @@ typedef struct { static int g_current_maclient_info = 0; static int g_current_preprocessing_maclient_info = -1; static const char *g_wakeup_maclient_appid = NULL; +static package_manager_h g_pkgmgr = NULL; static PREPROCESSING_STATE g_current_preprocessing_state = PREPROCESSING_STATE_NONE; @@ -738,12 +741,15 @@ static int init_plugin(void) static int deinit_plugin(void) { + if (0 != multi_assistant_service_plugin_deactivate()) { + MAS_LOGE("Fail to deactivate"); + } if (0 != multi_assistant_service_plugin_deinitialize()) { MAS_LOGE("Fail to deinitialize"); } } -static int process_multi_assistant_activated_setting() +static int process_activated_setting() { if (0 == vconf_notify_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb, NULL)) { /* Activate / deactivate according to the vconf key setting */ @@ -780,7 +786,7 @@ static int init_wakeup(void) init_plugin(); - process_multi_assistant_activated_setting(); + process_activated_setting(); mas_prelaunch_default_assistant(); @@ -1239,6 +1245,112 @@ int mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT event) return 0; } +static int pkg_app_list_cb(const pkgmgrinfo_appinfo_h handle, void *user_data) +{ + char *appid = NULL; + + int ret = pkgmgrinfo_appinfo_get_appid (handle, &appid); + if (PMINFO_R_OK == ret && NULL != appid) { + int *result = (int*)user_data; + if (result) { + for (int loop = 0; loop < MAX_MACLIENT_INFO_NUM;loop++) { + if (g_maclient_info[loop].used) { + LOGD("comparing appid : %s %s", g_maclient_info[loop].wakeup_engine, appid); + if (0 == strncmp(g_maclient_info[loop].wakeup_engine, appid, MAX_APPID_LEN)) { + *result = 1; + } + } + } + } + } else { + LOGE("pkgmgrinfo_appinfo_get_appid failed! error code=%d", ret); + return 0; + } + + return 0; +} + +/* +INFO: Package install/update/uninstall scenario +Install and Uninstall are obviously simple. + Install: just INSTALL + Uninstall: just UNINSTALL +Update package (change the source codes and Run As again), there are four scenarios: +1. UPDATE + Source code change +2. UNINSTALL -> INSTALL + This happens when Tizen IDE Property > Tizen SDK > Rapid Development Support > Check "Enable Project specific settings" + and change Application ID in tizen-manifest.xml file and Run As. +3. UPDATE -> INSTALL + This happens when Tizen IDE Property > Tizen SDK > Rapid Development Support > Uncheck "Enable Project specific settings" + and change Application ID in tizen-manifest.xml file and Run As. + At UPDATE event, pkgid (package parameter) is invalid... +4. UPDATE + Exceptionally, only UPDATE can be called when Application ID in tizen-manifest.xml file is changed. + At UPDATE event, pkgid (package parameter) is valid, and only appid is changed; the previous appid is invalid. +*/ +static void _package_manager_event_cb(const char *type, const char *package, package_manager_event_type_e event_type, package_manager_event_state_e event_state, int progress, package_manager_error_e error, void *user_data) +{ + int ret = 0; + uid_t uid = getuid (); + pkgmgrinfo_pkginfo_h handle = NULL; + static bool in_progress = false; + + if (!package || !type) + return; + + if (PACKAGE_MANAGER_EVENT_TYPE_UPDATE != event_type && + PACKAGE_MANAGER_EVENT_TYPE_INSTALL != event_type && + PACKAGE_MANAGER_EVENT_TYPE_UNINSTALL != event_type) + return; + + if (PACKAGE_MANAGER_EVENT_STATE_STARTED != event_state && + PACKAGE_MANAGER_EVENT_STATE_COMPLETED != event_state) + return; + + bool user = false; + LOGD("type=%s package=%s event_type=%d event_state=%d progress=%d error=%d", + type, package, event_type, event_state, progress, error); + ret = pkgmgrinfo_pkginfo_get_pkginfo(package, &handle); + if (ret != PMINFO_R_OK || NULL == handle) { + LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); + /* Try to get in user packages */ + ret = pkgmgrinfo_pkginfo_get_usr_pkginfo (package, uid, &handle); + if (ret != PMINFO_R_OK || NULL == handle) { + LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); + return; + } + } + + if (user) { + /* Try to get in user packages */ + pkgmgrinfo_appinfo_get_usr_list(handle, PMINFO_ALL_APP, pkg_app_list_cb, (void *)&ret, uid); + } else { + /* Try to get in global packages */ + pkgmgrinfo_appinfo_get_list(handle, PMINFO_ALL_APP, pkg_app_list_cb, (void *)&ret); + } + if (1 == ret) { + if (PACKAGE_MANAGER_EVENT_STATE_STARTED == event_state) { + LOGD("processing PACKAGE_MANAGER_EVENT_STATE_STARTED event"); + if (false == in_progress) { + in_progress = true; + deinit_plugin(); + } + } else if (PACKAGE_MANAGER_EVENT_STATE_COMPLETED == event_state) { + LOGD("processing PACKAGE_MANAGER_EVENT_STATE_COMPLETED event"); + if (true == in_progress) { + init_plugin(); + process_activated_setting(); + in_progress = false; + } + } + } + + pkgmgrinfo_pkginfo_destroy_pkginfo(handle); + + return; +} + bool service_app_create(void *data) { // Todo: add your code here. @@ -1250,11 +1362,33 @@ bool service_app_create(void *data) return false; } + if (!g_pkgmgr) { + int ret = package_manager_create(&g_pkgmgr); + if (ret == PACKAGE_MANAGER_ERROR_NONE) { + ret = package_manager_set_event_cb(g_pkgmgr, _package_manager_event_cb, NULL); + if (ret == PACKAGE_MANAGER_ERROR_NONE) { + LOGD("package_manager_set_event_cb succeeded."); + } + else { + LOGE("package_manager_set_event_cb failed(%d)", ret); + } + } + else { + LOGE("package_manager_create failed(%d)", ret); + } + } + return true; } void service_app_terminate(void *data) { + if (g_pkgmgr) { + package_manager_unset_event_cb(g_pkgmgr); + package_manager_destroy(g_pkgmgr); + g_pkgmgr = NULL; + } + // Todo: add your code here. deinit_wakeup(); return; @@ -1263,19 +1397,6 @@ void service_app_terminate(void *data) void service_app_control(app_control_h app_control, void *data) { // Todo: add your code here. - char *operation = NULL; - int ret = app_control_get_operation(app_control, &operation); - if (ret == APP_CONTROL_ERROR_NONE && operation) { - //strncmp(operation, APP_CONTROL_OPERATION_LAUNCH_ON_EVENT, strlen(operation)) == 0){ - char* value = NULL; - app_control_get_extra_data(app_control, "event", &value); - LOGD("operation : %s , %s", operation, value); - if (value && 0 == strcmp(value, "reload")) { - deinit_plugin(); - init_plugin(); - process_multi_assistant_activated_setting(); - } - } return; } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index b9f5f7a..e62a25f 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -511,7 +511,6 @@ int multi_assistant_service_plugin_deinitialize(void) MAS_LOGD("[Recorder ERROR] File not found!"); } #endif - multi_assistant_service_plugin_deactivate(); int ret = -1; if (NULL != g_handle) { -- 2.7.4 From eb1c183dc66625bcb50ef700045b3044ca54cdf0 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 8 Nov 2019 15:02:09 +0900 Subject: [PATCH 11/16] Fix bug recording_session is not changed properly Change-Id: Ia2ad3c215cb6bb2b2f65872122d3c4b8aff33115 --- plugins/wakeup-manager/src/wakeup_manager.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index ece33b9..1fa4387 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -422,6 +422,9 @@ bool CWakeupManager::set_background_volume(string appid, double ratio) bool CWakeupManager::update_recognition_result(string appid, int result) { MWR_LOGD("[ENTER]"); + if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { + mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD); + } mWakeupEngineManager.update_recognition_result(appid, result); if (WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState || WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { @@ -893,20 +896,24 @@ bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section( void CWakeupManager::CPolicyEventObserver::on_wakeup(wakeup_event_info wakeup_info) { if (nullptr == mWakeupManager) return; + + CAudioManager *audio_manager = mWakeupManager->get_audio_manager(); + CWakeupEngineManager *engine_manager = mWakeupManager->get_engine_manager(); + if (nullptr == audio_manager || nullptr == engine_manager) return; + mWakeupManager->stop_streaming_utterance_data(); mWakeupManager->stop_streaming_previous_utterance_data(); mWakeupManager->stop_streaming_follow_up_data(); mWakeupManager->change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE); - CWakeupEngineManager *engine_manager = mWakeupManager->get_engine_manager(); - if (nullptr == engine_manager) return; - mWakeupManager->set_last_wakeup_event_info(wakeup_info); engine_manager->set_selected_wakeup_info(wakeup_info); vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_wakeup(wakeup_info); } + + audio_manager->set_recording_session(RECORDING_SESSION_UTTERANCE); } bool CWakeupManager::CAudioEventObserver::on_recording_audio_data(long time, void* data, int len) -- 2.7.4 From dca681cbde530a626a227afe1a004ce75f3a0b43 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 19:16:30 +0900 Subject: [PATCH 12/16] Bump version to 0.2.9 Change-Id: I1febcfbce71db18e29d69652af547d2910518ec8 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 3e31401..4dc2a2c 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 486ab55..039a724 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.8 +Version: 0.2.9 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From d2da0780b6b6ab297f28db11816f24ae3e29301d Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 13 Nov 2019 19:56:26 +0900 Subject: [PATCH 13/16] Add support for setting assistant language Change-Id: Id7d6e6c48541088ec7004fcc2db27c3d473afdb3 --- inc/multi_assistant_main.h | 1 + inc/multi_assistant_service_plugin.h | 3 +++ plugins/wakeup-manager/inc/wakeup_engine_manager.h | 1 + plugins/wakeup-manager/inc/wakeup_manager.h | 1 + .../wakeup-manager/inc/wakeup_manager_wrapper.h | 2 ++ .../wakeup-manager/src/wakeup_engine_manager.cpp | 23 ++++++++++++++++ plugins/wakeup-manager/src/wakeup_manager.cpp | 10 +++++++ .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 20 ++++++++++++++ src/multi_assistant_dbus.c | 3 +++ src/multi_assistant_dbus_server.c | 31 ++++++++++++++++++++++ src/multi_assistant_service.c | 14 ++++++++++ src/multi_assistant_service_plugin.c | 22 +++++++++++++++ 12 files changed, 131 insertions(+) diff --git a/inc/multi_assistant_main.h b/inc/multi_assistant_main.h index 6c04aa6..8acd240 100644 --- a/inc/multi_assistant_main.h +++ b/inc/multi_assistant_main.h @@ -66,6 +66,7 @@ #define MA_METHOD_SET_PREPROCESSING_ALLOW_MODE "ma_method_set_preprocessing_allow_mode" #define MA_METHOD_SEND_PREPROCESSING_RESULT "ma_method_send_preprocessing_result" #define MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "ma_method_set_wake_word_audio_require_flag" +#define MA_METHOD_SET_ASSISTANT_LANGUAGE "ma_method_set_assistant_language" #define MA_METHOD_ERROR "ma_method_error" #define MA_UI_METHOD_INITIALIZE "ma_ui_method_initialize" diff --git a/inc/multi_assistant_service_plugin.h b/inc/multi_assistant_service_plugin.h index dbfd673..48a9233 100644 --- a/inc/multi_assistant_service_plugin.h +++ b/inc/multi_assistant_service_plugin.h @@ -159,6 +159,8 @@ typedef int (*wakeup_manager_get_audio_format)(int* rate, int* channel, int* aud typedef int (*wakeup_manager_get_audio_source_type)(char** type); #define MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "wakeup_manager_set_wake_word_audio_require_flag" typedef int (*wakeup_manager_set_wake_word_audio_require_flag)(bool require); +#define MA_WAKEUP_MANAGER_FUNC_SET_ASSISTANT_LANGUAGE "wakeup_manager_set_assistant_language" +typedef int (*wakeup_manager_set_assistant_language)(const char* appid, const char* language); #define MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_manager_set_wakeup_event_callback" typedef int (*wakeup_manager_set_wakeup_event_callback)(wakeup_service_wakeup_event_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_UTTERANCE_STREAMING_CALLBACK "wakeup_manager_set_utterance_streaming_callback" @@ -202,6 +204,7 @@ typedef struct { wakeup_manager_get_audio_format get_audio_format; wakeup_manager_get_audio_source_type get_audio_source_type; wakeup_manager_set_wake_word_audio_require_flag set_wake_word_audio_require_flag; + wakeup_manager_set_assistant_language set_assistant_language; wakeup_manager_set_wakeup_event_callback set_wakeup_event_callback; wakeup_manager_set_utterance_streaming_callback set_utterance_streaming_callback; wakeup_manager_set_previous_utterance_streaming_callback set_previous_utterance_streaming_callback; diff --git a/plugins/wakeup-manager/inc/wakeup_engine_manager.h b/plugins/wakeup-manager/inc/wakeup_engine_manager.h index c768f9b..d9d478c 100644 --- a/plugins/wakeup-manager/inc/wakeup_engine_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_engine_manager.h @@ -149,6 +149,7 @@ public: void set_selected_wakeup_info(wakeup_event_info wakeup_info); bool set_language(string language); + bool set_assistant_language(string appid, string language); void set_assistant_activated(string appid, bool activated); bool get_assistant_activated(string appid); void set_wake_word_audio_require_flag(bool require); diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index d574646..c6623a2 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -84,6 +84,7 @@ public: bool add_assistant_language(string appid, string language); bool add_assistant_wakeup_word(string appid, string wakeup_word, string language); bool set_assistant_wakeup_engine(string appid, string engine); + bool set_assistant_language(string appid, string language); bool set_assistant_enabled(string appid, bool enabled); bool get_assistant_enabled(string appid); diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index 2df503e..d6d20a5 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -138,6 +138,8 @@ EXPORT_API int wakeup_manager_get_audio_source_type(char** type); EXPORT_API int wakeup_manager_set_wake_word_audio_require_flag(bool require); +EXPORT_API int wakeup_manager_set_assistant_language(const char* appid, const char *language); + EXPORT_API int wakeup_manager_set_wakeup_event_callback(wakeup_service_wakeup_event_cb callback, void* user_data); EXPORT_API int wakeup_manager_set_utterance_streaming_callback(wakeup_service_speech_streaming_cb callback, void* user_data); diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index ebc50a9..cef7b6d 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -161,6 +161,29 @@ bool CWakeupEngineManager::set_language(string language) return true; } +bool CWakeupEngineManager::set_assistant_language(string appid, string language) +{ + for (auto& info : mEngineInfo) { + const auto& iter = find_if(info.assistant_list.begin(), info.assistant_list.end(), + [appid](const string& assistant) { + return (0 == assistant.compare(appid)); + }); + + /* If the appid is in the assistant list */ + if (info.assistant_list.end() != iter) { + try { + int ret = info.interface.set_language(language.c_str()); + MWR_LOGD("set_language returned %d : %s %s %s", + ret, appid.c_str(), info.engine_name.c_str(), language.c_str()); + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); + } + } + } + return true; +} + void CWakeupEngineManager::set_assistant_activated(string appid, bool activated) { MWR_LOGD("[ENTER] : %s %d", appid.c_str(), activated); diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 1fa4387..a6a6281 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -249,6 +249,16 @@ bool CWakeupManager::set_assistant_wakeup_engine(string appid, string engine) return true; } +bool CWakeupManager::set_assistant_language(string appid, string language) +{ + MWR_LOGD("[ENTER] : %s, %s", appid.c_str(), language.c_str()); + + mWakeupEngineManager.set_assistant_language(appid, language); + + MWR_LOGD("[END]"); + return true; +} + bool CWakeupManager::set_assistant_enabled(string appid, bool enabled) { MWR_LOGD("[ENTER]"); diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 77549bb..49b24eb 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -469,6 +469,26 @@ int wakeup_manager_set_wake_word_audio_require_flag(bool require) return 0; } +int wakeup_manager_set_assistant_language(const char* appid, const char* language) +{ + MWR_LOGD("[ENTER]"); + + string appid_string; + string language_string; + if (NULL == appid || NULL == language) { + MWR_LOGE("[ERROR] Parameter is invalid, appid(%s), language(%s)", appid, language); + return -1; + } + appid_string = appid; + language_string = language; + + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_assistant_language(appid_string, language_string); + + MWR_LOGD("[END]"); + return 0; +} + int wakeup_manager_set_wakeup_event_callback(wakeup_service_wakeup_event_cb callback, void* user_data) { MWR_LOGD("[ENTER]"); diff --git a/src/multi_assistant_dbus.c b/src/multi_assistant_dbus.c index ac92dba..53130d8 100644 --- a/src/multi_assistant_dbus.c +++ b/src/multi_assistant_dbus.c @@ -875,6 +875,9 @@ static Eina_Bool listener_event_callback(void* data, Ecore_Fd_Handler *fd_handle } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG)) { ma_service_dbus_set_wake_word_audio_require_flag(g_conn_listener, msg); + } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_SET_ASSISTANT_LANGUAGE)) { + ma_service_dbus_set_assistant_language(g_conn_listener, msg); + } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_UI_METHOD_INITIALIZE)) { ma_service_ui_dbus_initialize(g_conn_listener, msg); diff --git a/src/multi_assistant_dbus_server.c b/src/multi_assistant_dbus_server.c index 0d00f3e..40c9e13 100644 --- a/src/multi_assistant_dbus_server.c +++ b/src/multi_assistant_dbus_server.c @@ -663,6 +663,37 @@ int ma_service_dbus_set_wake_word_audio_require_flag(DBusConnection* conn, DBusM return 0; } +int ma_service_dbus_set_assistant_language(DBusConnection* conn, DBusMessage* msg) +{ + DBusError err; + dbus_error_init(&err); + + int pid; + int ret = 0; + char* language; + + dbus_message_get_args(msg, &err, + DBUS_TYPE_INT32, &pid, + DBUS_TYPE_STRING, &language, + DBUS_TYPE_INVALID); + + MAS_LOGD("[DEBUG] MAS SET ASSISTANT LANGUAGE"); + + if (dbus_error_is_set(&err)) { + MAS_LOGE("[IN ERROR] mas set assistant language : Get arguments error (%s)", err.message); + dbus_error_free(&err); + ret = -1; //MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD("[IN] mas set assistant language : pid(%d), language(%s)", pid, language); + ret = mas_client_set_assistant_language(pid, language); + } + + MAS_LOGD("<<<<<"); + MAS_LOGD(" "); + + return 0; +} + int ma_service_ui_dbus_initialize(DBusConnection* conn, DBusMessage* msg) { DBusError err; diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index fc2f3cb..94b2fb4 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -533,6 +533,20 @@ int mas_client_set_wake_word_audio_require_flag(int pid, bool require) return 0; } +int mas_client_set_assistant_language(int pid, const char* language) +{ + const char* pid_appid = NULL; + char buf[MAX_APPID_LEN] = {'\0',}; + int ret = aul_app_get_appid_bypid(pid, buf, sizeof(buf)); + if (AUL_R_OK == ret) { + buf[MAX_APPID_LEN - 1] = '\0'; + pid_appid = buf; + } + + multi_assistant_service_plugin_assistant_language(pid_appid, language); + return 0; +} + int mas_ui_client_initialize(int pid) { MAS_LOGD("[Enter] pid(%d)", pid); diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index e62a25f..e7640f6 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -445,6 +445,9 @@ int multi_assistant_service_plugin_initialize(void) _wakeup_manager_interface.set_wake_word_audio_require_flag = (wakeup_manager_set_wake_word_audio_require_flag)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG); + _wakeup_manager_interface.set_assistant_language = + (wakeup_manager_set_assistant_language)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_SET_ASSISTANT_LANGUAGE); _wakeup_manager_interface.set_wakeup_event_callback = (wakeup_manager_set_wakeup_event_callback)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_EVENT_CALLBACK); @@ -974,6 +977,25 @@ int multi_assistant_service_plugin_set_wake_word_audio_require_flag(const char* return ret; } +int multi_assistant_service_plugin_assistant_language(const char* appid, const char* language) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_set_assistant_language func = _wakeup_manager_interface.set_assistant_language; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_ASSISTANT_LANGUAGE); + } else { + ret = func(appid, language); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set assistant language, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + return ret; +} + int multi_assistant_service_plugin_set_callbacks(void) { int ret = multi_assistant_service_plugin_set_wakeup_event_callback(__wakeup_event_cb, NULL); -- 2.7.4 From a329e2482b363b5986482a876cddbcf83ab8f16d Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 18 Nov 2019 11:19:36 +0900 Subject: [PATCH 14/16] Fix defects detected by static analysis tool Change-Id: Ic5c1e6a0068ad243476f62b016131a9e0c5b9e1f --- src/multi_assistant_service.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 94b2fb4..96ff563 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -761,6 +761,7 @@ static int deinit_plugin(void) if (0 != multi_assistant_service_plugin_deinitialize()) { MAS_LOGE("Fail to deinitialize"); } + return 0; } static int process_activated_setting() @@ -1329,6 +1330,7 @@ static void _package_manager_event_cb(const char *type, const char *package, pac if (ret != PMINFO_R_OK || NULL == handle) { LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); /* Try to get in user packages */ + user = true; ret = pkgmgrinfo_pkginfo_get_usr_pkginfo (package, uid, &handle); if (ret != PMINFO_R_OK || NULL == handle) { LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); -- 2.7.4 From 301719685888589864b56ba4e52ba0eda987d7ee Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 19 Nov 2019 11:41:42 +0900 Subject: [PATCH 15/16] Add logs for tracking proper destruction Change-Id: If35c356ee2c1c5849d0b82d1a470076fc45cb370 --- plugins/wakeup-manager/src/wakeup_engine_manager.cpp | 11 +++++++++-- plugins/wakeup-manager/src/wakeup_manager.cpp | 5 +++-- plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp | 4 ++-- src/multi_assistant_service.c | 11 +++++++++-- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index cef7b6d..896057f 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -27,6 +27,7 @@ CWakeupEngineManager::CWakeupEngineManager() CWakeupEngineManager::~CWakeupEngineManager() { + MWR_LOGI("Wakeup Engine Manager is now being destroyed"); } CWakeupEngineManager::CWakeupEngineManager(IEngineEventObserver *observer) : CWakeupEngineManager() @@ -76,6 +77,7 @@ void CWakeupEngineManager::initialize() void CWakeupEngineManager::deinitialize() { + MWR_LOGI("[START]"); if (mStreamingThread.joinable()) { MWR_LOGD("mStreamingThread is joinable, trying join()"); mStopStreamingThread.store(true); @@ -97,20 +99,25 @@ void CWakeupEngineManager::deinitialize() if (info.interface.set_audio_data_require_status_callback) { info.interface.set_audio_data_require_status_callback(nullptr, nullptr); } + MWR_LOGI("Trying to deinitialize engine : %s", info.engine_name.c_str()); if (info.interface.deinitialize) { - info.interface.deinitialize(); + int ret = info.interface.deinitialize(); + MWR_LOGI("Deinitialization [%s] returned %d", info.engine_name.c_str(), ret); } } catch (const std::exception& e) { MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", info.engine_name.c_str(), e.what()); } if (info.engine_handle) { - dlclose(info.engine_handle); + int ret = dlclose(info.engine_handle); + MWR_LOGI("Closing [%s] returned %d, [%s]", info.engine_name.c_str(), ret, + ((0 == ret) ? "" : dlerror())); info.engine_handle = nullptr; } } mSelectedEngine = nullptr; mEngineInfo.clear(); + MWR_LOGI("[END]"); } void CWakeupEngineManager::subscribe(IEngineEventObserver *observer) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index a6a6281..75fa067 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -68,6 +68,7 @@ CWakeupManager::CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingVa CWakeupManager::~CWakeupManager() { + MWR_LOGI("Wakeup Manager is now being destroyed"); } void CWakeupManager::initialize_wakeup_policy() @@ -122,7 +123,7 @@ bool CWakeupManager::initialize() bool CWakeupManager::deinitialize() { - MWR_LOGD("[ENTER]"); + MWR_LOGI("[ENTER]"); dependency_resolver_deinitialize(); @@ -135,7 +136,7 @@ bool CWakeupManager::deinitialize() mWakeupSettings.deinitialize(); mAssistantLanguageInfo.clear(); - MWR_LOGD("[END]"); + MWR_LOGI("[END]"); return true; } diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 49b24eb..0d30121 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -107,7 +107,7 @@ int wakeup_manager_initialize(void) int wakeup_manager_deinitialize(void) { - MWR_LOGD("[ENTER]"); + MWR_LOGI("[ENTER]"); if (nullptr == g_wakeup_manager) return -1; @@ -122,7 +122,7 @@ int wakeup_manager_deinitialize(void) return -1; } - MWR_LOGD("[END]"); + MWR_LOGI("[END]"); return 0; } diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 96ff563..8c1101f 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -755,12 +755,14 @@ static int init_plugin(void) static int deinit_plugin(void) { + MAS_LOGI("[ENTER]"); if (0 != multi_assistant_service_plugin_deactivate()) { MAS_LOGE("Fail to deactivate"); } if (0 != multi_assistant_service_plugin_deinitialize()) { MAS_LOGE("Fail to deinitialize"); } + MAS_LOGI("[END]"); return 0; } @@ -820,7 +822,7 @@ static int init_wakeup(void) static void deinit_wakeup(void) { - MAS_LOGD("[Enter] deinit_wakeup "); + MAS_LOGI("[Enter] deinit_wakeup"); /* if (NULL != g_current_lang) { free(g_current_lang); @@ -835,6 +837,7 @@ static void deinit_wakeup(void) if (0 != ret) { MAS_LOGE("[ERROR] Fail to close connection"); } + MAS_LOGI("[END]"); } int mas_get_current_client_pid() @@ -1399,6 +1402,7 @@ bool service_app_create(void *data) void service_app_terminate(void *data) { + MAS_LOGI("[ENTER]"); if (g_pkgmgr) { package_manager_unset_event_cb(g_pkgmgr); package_manager_destroy(g_pkgmgr); @@ -1407,6 +1411,7 @@ void service_app_terminate(void *data) // Todo: add your code here. deinit_wakeup(); + MAS_LOGI("[END]"); return; } @@ -1456,5 +1461,7 @@ int main(int argc, char* argv[]) service_app_add_event_handler(&handlers[APP_EVENT_LANGUAGE_CHANGED], APP_EVENT_LANGUAGE_CHANGED, service_app_lang_changed, &ad); service_app_add_event_handler(&handlers[APP_EVENT_REGION_FORMAT_CHANGED], APP_EVENT_REGION_FORMAT_CHANGED, service_app_region_changed, &ad); - return service_app_main(argc, argv, &event_callback, ad); + int ret = service_app_main(argc, argv, &event_callback, ad); + LOGI("Main function exits with : %d", ret); + return ret; } -- 2.7.4 From 30582d1b78a1021aab437eef6a2636524fe21c85 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 19 Nov 2019 15:08:06 +0900 Subject: [PATCH 16/16] Bump version to 0.2.10 Change-Id: Icd0479f15c231519edb6fcf0aa2bbf2d0630a821 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 4dc2a2c..adb0d85 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 039a724..b7c668a 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.9 +Version: 0.2.10 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4