From 54f7b4364bcdfc7d925fb4f261ce1b3345af847b Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 24 Oct 2019 21:33:18 +0900 Subject: [PATCH 01/16] Remove multi-assistant header inclusion Change-Id: I13635bc2bbd080e6dc2706a057f807a2f29f51d7 --- inc/multi_wakeup_recognizer.h | 33 +++++++++++++++++++++- plugins/wakeup-manager/inc/wakeup_engine_manager.h | 2 -- plugins/wakeup-manager/inc/wakeup_manager.h | 2 -- .../wakeup-manager/inc/wakeup_manager_wrapper.h | 15 +++++++++- src/multi_assistant_service.c | 1 - src/multi_assistant_service_plugin.c | 1 - 6 files changed, 46 insertions(+), 8 deletions(-) diff --git a/inc/multi_wakeup_recognizer.h b/inc/multi_wakeup_recognizer.h index c818e1a..6c3eb6f 100644 --- a/inc/multi_wakeup_recognizer.h +++ b/inc/multi_wakeup_recognizer.h @@ -19,7 +19,6 @@ #define _MULTI_WAKEUP_RECOGNIZER_H_ #include -#include #ifdef __cplusplus extern "C" { @@ -61,6 +60,38 @@ typedef enum { WAKEUP_ASR_RESULT_EVENT_ERROR /**< Event when the recognition has failed */ } wakeup_asr_result_event_e; +typedef enum { + MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE = 0, /**< Current utterance */ + MA_AUDIO_STREAMING_DATA_TYPE_PREVIOUS_UTTERANCE, /**< Previous utterance */ + MA_AUDIO_STREAMING_DATA_TYPE_FOLLOW_UP_SPEECH /**< Follow-up speech */ +} ma_audio_streaming_data_type_e; + +typedef enum { + MA_ACTIVE_STATE_INACTIVE = 0, /**< 'Inactive' state */ + MA_ACTIVE_STATE_ACTIVE, /**< 'Active' state */ + MA_ACTIVE_STATE_PREPROCESSING, /**< 'Preprocessing' state */ +} ma_active_state_e; + +typedef enum { + MA_RECOGNITION_RESULT_EVENT_SUCCESS = 0, /**< Recognition succeeded */ + MA_RECOGNITION_RESULT_EVENT_EMPTY_TEXT, /**< No text result recognized */ + MA_RECOGNITION_RESULT_EVENT_ERROR, /**< Unknown error occurred */ + MA_RECOGNITION_RESULT_EVENT_FALSE_TRIGGER /**< Turned out to be a false trigger */ +} ma_recognition_result_event_e; + +typedef enum { + MA_PREPROCESSING_ALLOW_NONE = 0, /**< No preprocessing allowed */ + MA_PREPROCESSING_ALLOW_UTTERANCE, /**< Preprocessing allowed for utterance audio only */ + MA_PREPROCESSING_ALLOW_FOLLOW_UP, /**< Preprocessing allowed for follow-up audio only */ + MA_PREPROCESSING_ALLOW_ALL, /**< Preprocessing allowed for all audio */ +} ma_preprocessing_allow_mode_e; + +typedef enum { + MA_AUDIO_STREAMING_DATA_SECTION_UTTERANCE = 0, /**< Utterance section started */ + MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD, /**< Wake word section started */ +} ma_audio_streaming_data_section_e; + +typedef void (*ma_audio_streaming_data_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); typedef void (*wakeup_service_wakeup_event_cb)(wakeup_event_info wakeup_info, const char* wakeup_word, void* user_data); diff --git a/plugins/wakeup-manager/inc/wakeup_engine_manager.h b/plugins/wakeup-manager/inc/wakeup_engine_manager.h index 49d8e22..c768f9b 100644 --- a/plugins/wakeup-manager/inc/wakeup_engine_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_engine_manager.h @@ -19,8 +19,6 @@ #include "wakeup_manager_wrapper.h" -#include - #include #include #include diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index d682387..e6f5251 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -24,8 +24,6 @@ #include "wakeup_audio_manager.h" #include "wakeup_policy_default.h" -#include "multi_assistant_common.h" - #include #include diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index c566d83..7f408dc 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -22,7 +22,6 @@ #include #include #include -#include #include "wakeup_interfaces.h" @@ -44,6 +43,20 @@ typedef enum { WAKEUP_MANAGER_STATE_VOICE_FEEDBACK = 4 } wakeup_manager_state_e; +typedef enum { + MA_PREPROCESSING_ALLOW_NONE = 0, /**< No preprocessing allowed */ + MA_PREPROCESSING_ALLOW_UTTERANCE, /**< Preprocessing allowed for utterance audio only */ + MA_PREPROCESSING_ALLOW_FOLLOW_UP, /**< Preprocessing allowed for follow-up audio only */ + MA_PREPROCESSING_ALLOW_ALL, /**< Preprocessing allowed for all audio */ +} ma_preprocessing_allow_mode_e; + +typedef enum { + MA_AUDIO_STREAMING_DATA_SECTION_UTTERANCE = 0, /**< Utterance section started */ + MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD, /**< Wake word section started */ +} ma_audio_streaming_data_section_e; + +typedef void (*ma_audio_streaming_data_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); + typedef void (*wakeup_service_wakeup_event_cb)(wakeup_event_info wakeup_info, void* user_data); typedef void (*wakeup_service_speech_streaming_cb)(wakeup_speech_streaming_event_e event, void* buffer, int len, void *user_data); diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 5b2b220..0baf44e 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -33,7 +33,6 @@ #include "multi_assistant_service_plugin.h" #include "multi_assistant_dbus.h" #include "multi_assistant_config.h" -#include "multi_assistant_common.h" static const char *g_current_lang = "en_US"; diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 518e5e5..a04366f 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -32,7 +32,6 @@ #include "multi_assistant_service.h" #include "multi_assistant_service_plugin.h" #include "multi_assistant_dbus.h" -#include "multi_assistant_common.h" /* Sound buf save for test */ #if 0 -- 2.7.4 From 497c80a7dca588a5472d7f85f2bebae427cfe346 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 24 Oct 2019 21:37:00 +0900 Subject: [PATCH 02/16] Bump version to 0.2.4 Change-Id: I3923e7e0c848fcbf751523dc43ee676fd9169b1b --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 4ae038a..0fe54b1 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 19ee155..09ea4a2 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.3 +Version: 0.2.4 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 1f68c5e3f0416f7ced134cf1fb992b81466ca1c7 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 24 Oct 2019 22:18:58 +0900 Subject: [PATCH 03/16] Prevent sending preprocessing appid as NULL Change-Id: Id3744e4a00bfb4f1a4e66996b8952027a285f74c --- src/multi_assistant_service.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 0baf44e..278c60c 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -305,9 +305,11 @@ int mas_client_send_preprocessing_information(int pid) char* vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_PREPROCESSING_ASSISTANT_APPID); MAS_LOGD("preprocessing_assistant_appid : %s", vconf_str); - ret = masc_dbus_send_preprocessing_information(pid, vconf_str); - free(vconf_str); - vconf_str = NULL; + if (vconf_str) { + ret = masc_dbus_send_preprocessing_information(pid, vconf_str); + free(vconf_str); + vconf_str = NULL; + } return ret; } -- 2.7.4 From 5ffe702859843fb6574805759f6bd0fdce178cb8 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 24 Oct 2019 22:21:03 +0900 Subject: [PATCH 04/16] Bump version to 0.2.5 Change-Id: Ie3e3f4936ba5eedadeadba5a8fdcd2f98de08c1f --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 0fe54b1..483d407 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 09ea4a2..9f783fc 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.4 +Version: 0.2.5 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 4b026e616c97126331848a3f57534bd07abcd7bd Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 28 Oct 2019 14:45:41 +0900 Subject: [PATCH 05/16] Fix incorrect log messages Change-Id: I5722be377ae9e07f7ab2fa1cba7770091613d141 --- src/multi_assistant_dbus_server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/multi_assistant_dbus_server.c b/src/multi_assistant_dbus_server.c index 16ec6c8..0d00f3e 100644 --- a/src/multi_assistant_dbus_server.c +++ b/src/multi_assistant_dbus_server.c @@ -428,7 +428,7 @@ int ma_service_dbus_start_streaming_audio_data(DBusConnection* conn, DBusMessage dbus_error_free(&err); ret = -1; //MAS_ERROR_OPERATION_FAILED; } else { - MAS_LOGD("[IN] mas send start streaming : pid(%d), result(%d)", pid, type); + MAS_LOGD("[IN] mas send start streaming : pid(%d), type(%d)", pid, type); ret = mas_client_start_streaming_audio_data(pid, type); } @@ -459,7 +459,7 @@ int ma_service_dbus_stop_streaming_audio_data(DBusConnection* conn, DBusMessage* dbus_error_free(&err); ret = -1; //MAS_ERROR_OPERATION_FAILED; } else { - MAS_LOGD("[IN] mas stop streaming : pid(%d), result(%d)", pid, type); + MAS_LOGD("[IN] mas stop streaming : pid(%d), type(%d)", pid, type); ret = mas_client_stop_streaming_audio_data(pid, type); } @@ -490,7 +490,7 @@ int ma_service_dbus_update_voice_feedback_state(DBusConnection* conn, DBusMessag dbus_error_free(&err); ret = -1; //MAS_ERROR_OPERATION_FAILED; } else { - MAS_LOGD("[IN] mas update voice feedback : pid(%d), result(%d)", pid, state); + MAS_LOGD("[IN] mas update voice feedback : pid(%d), state(%d)", pid, state); ret = mas_client_update_voice_feedback_state(pid, state); } -- 2.7.4 From 398892c34e1d54b56f5968bf0a12bd5e7912505e Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 29 Oct 2019 11:11:34 +0900 Subject: [PATCH 06/16] Support background/foreground preprocessing selectively Change-Id: If8d8d875f35a08ec8463fe0549b0c9f35df0e2a8 --- inc/multi_assistant_service.h | 3 +- src/multi_assistant_service.c | 81 ++++++++++++++++++++++++++---------- src/multi_assistant_service_plugin.c | 2 +- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/inc/multi_assistant_service.h b/inc/multi_assistant_service.h index 0a79b99..0bbc37a 100644 --- a/inc/multi_assistant_service.h +++ b/inc/multi_assistant_service.h @@ -98,8 +98,7 @@ typedef enum { } PREPROCESSING_STATE; typedef enum { - PREPROCESSING_STATE_EVENT_WAKEUP, - PREPROCESSING_STATE_EVENT_ACTIVE_ASSISTANT_LAUNCHED, + PREPROCESSING_STATE_EVENT_ASSISTANT_ACTIVATED, PREPROCESSING_STATE_EVENT_PREPROCESSING_ALLOW_MODE_CHANGED, PREPROCESSING_STATE_EVENT_UTTERANCE_STREAMING_STARTED, PREPROCESSING_STATE_EVENT_FOLLOW_UP_STREAMING_STARTED, diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 278c60c..b2da67b 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -248,7 +248,7 @@ int mas_client_initialize(int pid) if (g_wakeup_maclient_appid && strncmp(g_wakeup_maclient_appid, appid, MAX_APPID_LEN) == 0) { g_wakeup_maclient_appid = NULL; mas_client_activate(pid); - mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_ACTIVE_ASSISTANT_LAUNCHED); + mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_ASSISTANT_ACTIVATED); } else { MAS_LOGE("[ERROR] g_wakeup_maclient_appid and appid differ : %s %s", (g_wakeup_maclient_appid ? g_wakeup_maclient_appid : "NULL"), appid); @@ -503,11 +503,9 @@ int mas_client_send_preprocessing_result(int pid, bool result) if (result) { MAS_LOGD("Preprocessing succeeded, bring (%s) to foreground", pid_appid); - mas_bring_client_to_foreground(pid_appid); mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_PREPROCESSING_SUCCEEDED); } else { MAS_LOGD("Preprocessing failed, bring (%s) to foreground", current_maclient_appid); - mas_bring_client_to_foreground(current_maclient_appid); mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_PREPROCESSING_FAILED); } @@ -1105,6 +1103,9 @@ ma_preprocessing_allow_mode_e get_preprocessing_allow_mode(const char* appid) return MA_PREPROCESSING_ALLOW_NONE; } +/* This might need to be read from settings in the future, but using macro for now */ +//#define BRING_PREPROCESSING_ASSISTANT_TO_FRONT + int mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT event) { const char* current_maclient_appid = NULL; @@ -1116,28 +1117,40 @@ int mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT event) ma_preprocessing_allow_mode_e mode = get_preprocessing_allow_mode(current_maclient_appid); switch (event) { - case PREPROCESSING_STATE_EVENT_WAKEUP: - case PREPROCESSING_STATE_EVENT_ACTIVE_ASSISTANT_LAUNCHED: + case PREPROCESSING_STATE_EVENT_ASSISTANT_ACTIVATED: { - if (!check_preprocessing_assistant_exists()) { - mas_bring_client_to_foreground(current_maclient_appid); - } +#ifndef BRING_PREPROCESSING_ASSISTANT_TO_FRONT + /* If there is no need to bring preprocessing assistant to front, + current_maclient should always be brought to front */ + mas_bring_client_to_foreground(current_maclient_appid); +#endif g_current_preprocessing_state = PREPROCESSING_STATE_WAKEUP_PREPROCESS_DISABLED; - if (MA_PREPROCESSING_ALLOW_UTTERANCE == mode || - MA_PREPROCESSING_ALLOW_ALL == mode) { - if (is_current_preprocessing_assistant(preprocessing_allow_appid)) { - g_current_preprocessing_state = PREPROCESSING_STATE_WAKEUP_PREPROCESS_ENABLED; + if (check_preprocessing_assistant_exists()) { + if (MA_PREPROCESSING_ALLOW_UTTERANCE == mode || + MA_PREPROCESSING_ALLOW_ALL == mode) { + if (is_current_preprocessing_assistant(preprocessing_allow_appid)) { + g_current_preprocessing_state = PREPROCESSING_STATE_WAKEUP_PREPROCESS_ENABLED; + } } + } else { +#ifdef BRING_PREPROCESSING_ASSISTANT_TO_FRONT + /* If preprocessing assistant does not exist, there is no way to enable + preprocessing assistant, so bring current maclient to front right away */ + mas_bring_client_to_foreground(current_maclient_appid); +#endif } } break; case PREPROCESSING_STATE_EVENT_PREPROCESSING_ALLOW_MODE_CHANGED: { g_current_preprocessing_state = PREPROCESSING_STATE_WAKEUP_PREPROCESS_DISABLED; - if (MA_PREPROCESSING_ALLOW_UTTERANCE == mode || - MA_PREPROCESSING_ALLOW_ALL == mode) { - if (is_current_preprocessing_assistant(preprocessing_allow_appid)) { - g_current_preprocessing_state = PREPROCESSING_STATE_WAKEUP_PREPROCESS_ENABLED; + /* Enable preprocessing mode only if the preprocessing assistant exists */ + if (check_preprocessing_assistant_exists()) { + if (MA_PREPROCESSING_ALLOW_UTTERANCE == mode || + MA_PREPROCESSING_ALLOW_ALL == mode) { + if (is_current_preprocessing_assistant(preprocessing_allow_appid)) { + g_current_preprocessing_state = PREPROCESSING_STATE_WAKEUP_PREPROCESS_ENABLED; + } } } } @@ -1147,27 +1160,53 @@ int mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT event) if (PREPROCESSING_STATE_WAKEUP_PREPROCESS_ENABLED == g_current_preprocessing_state) { g_current_preprocessing_state = PREPROCESSING_STATE_PREPROCESSING_UTTERANCE; } else if (PREPROCESSING_STATE_WAKEUP_PREPROCESS_DISABLED == g_current_preprocessing_state) { - mas_bring_client_to_foreground(current_maclient_appid); + /* If preprocessing assistant does not exist, the current_maclient + would have been brought to front already on wakeup event */ +#ifdef BRING_PREPROCESSING_ASSISTANT_TO_FRONT + if (check_preprocessing_assistant_exists()) { + mas_bring_client_to_foreground(current_maclient_appid); + } +#endif g_current_preprocessing_state = PREPROCESSING_STATE_NONE; } } break; case PREPROCESSING_STATE_EVENT_FOLLOW_UP_STREAMING_STARTED: { - if (MA_PREPROCESSING_ALLOW_FOLLOW_UP == mode || - MA_PREPROCESSING_ALLOW_ALL == mode) { - g_current_preprocessing_state = PREPROCESSING_STATE_PREPROCESSING_FOLLOW_UP; + g_current_preprocessing_state = PREPROCESSING_STATE_NONE; + if (check_preprocessing_assistant_exists()) { + if (MA_PREPROCESSING_ALLOW_FOLLOW_UP == mode || + MA_PREPROCESSING_ALLOW_ALL == mode) { + g_current_preprocessing_state = PREPROCESSING_STATE_PREPROCESSING_FOLLOW_UP; + } } } break; case PREPROCESSING_STATE_EVENT_PREPROCESSING_SUCCEEDED: { +#ifdef BRING_PREPROCESSING_ASSISTANT_TO_FRONT + if (PREPROCESSING_STATE_EVENT_UTTERANCE_STREAMING_STARTED == g_current_preprocessing_state || + PREPROCESSING_STATE_EVENT_FOLLOW_UP_STREAMING_STARTED == g_current_preprocessing_state) { + char* vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_PREPROCESSING_ASSISTANT_APPID); + MAS_LOGD("preprocessing_assistant_appid : %s", vconf_str); + if (vconf_str) { + mas_bring_client_to_foreground(vconf_str); + free(vconf_str); + vconf_str = NULL; + } + } +#endif g_current_preprocessing_state = PREPROCESSING_STATE_NONE; } break; case PREPROCESSING_STATE_EVENT_PREPROCESSING_FAILED: { - mas_bring_client_to_foreground(current_maclient_appid); +#ifdef BRING_PREPROCESSING_ASSISTANT_TO_FRONT + if (PREPROCESSING_STATE_EVENT_UTTERANCE_STREAMING_STARTED == g_current_preprocessing_state || + PREPROCESSING_STATE_EVENT_FOLLOW_UP_STREAMING_STARTED == g_current_preprocessing_state) { + mas_bring_client_to_foreground(current_maclient_appid); + } +#endif g_current_preprocessing_state = PREPROCESSING_STATE_NONE; } break; diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index a04366f..5e9e063 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -109,12 +109,12 @@ Eina_Bool process_wakeup_event_by_appid_timer(char* appid) if ((pid = mas_get_client_pid_by_appid(appid)) != -1) { mas_client_send_preprocessing_information(pid); mas_client_activate(pid); + mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_ASSISTANT_ACTIVATED); } else { // Appropriate MA Client not available, trying to launch new one MAS_LOGD("MA Client with appid %s does not exist, launching client", appid); mas_launch_client_by_appid(appid, CLIENT_LAUNCH_MODE_ACTIVATION); } - mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_WAKEUP); if (appid) free(appid); -- 2.7.4 From 871c00fcc97b27edfb84f723407a84286fae69a8 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 29 Oct 2019 16:03:40 +0900 Subject: [PATCH 07/16] Prelaunch default assistant on setting change event Change-Id: I1a8f96a408bff8668d81020876cdebad0623e969 --- inc/multi_assistant_service.h | 2 + inc/multi_assistant_service_plugin.h | 3 + inc/multi_wakeup_recognizer.h | 2 + plugins/wakeup-manager/inc/wakeup_manager.h | 22 +++++-- .../wakeup-manager/inc/wakeup_manager_wrapper.h | 4 ++ plugins/wakeup-manager/src/wakeup_manager.cpp | 67 ++++++++++++++++------ .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 37 +++++++++++- src/multi_assistant_service.c | 28 +++++---- src/multi_assistant_service_plugin.c | 32 +++++++++++ 9 files changed, 164 insertions(+), 33 deletions(-) diff --git a/inc/multi_assistant_service.h b/inc/multi_assistant_service.h index 0bbc37a..775245d 100644 --- a/inc/multi_assistant_service.h +++ b/inc/multi_assistant_service.h @@ -81,6 +81,8 @@ int mas_set_current_client_by_appid(const char *appid); int mas_launch_client_by_wakeup_word(const char *wakeup_word); +int mas_prelaunch_default_assistant(); + typedef enum { CLIENT_LAUNCH_MODE_ACTIVATION, CLIENT_LAUNCH_MODE_PRELAUNCH, diff --git a/inc/multi_assistant_service_plugin.h b/inc/multi_assistant_service_plugin.h index 7bfd761..dbfd673 100644 --- a/inc/multi_assistant_service_plugin.h +++ b/inc/multi_assistant_service_plugin.h @@ -169,6 +169,8 @@ typedef int (*wakeup_manager_set_previous_utterance_streaming_callback)(wakeup_s typedef int (*wakeup_manager_set_follow_up_streaming_callback)(wakeup_service_speech_streaming_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_SPEECH_STATUS_CALLBACK "wakeup_manager_set_speech_status_callback" typedef int (*wakeup_manager_set_speech_status_callback)(wakeup_service_speech_status_cb callback, void* user_data); +#define MA_WAKEUP_MANAGER_FUNC_SET_SETTING_CHANGED_CALLBACK "wakeup_manager_set_setting_changed_callback" +typedef int (*wakeup_manager_set_setting_changed_callback)(wakeup_service_setting_changed_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_ERROR_CALLBACK "wakeup_manager_set_error_callback" typedef int (*wakeup_manager_set_error_callback)(wakeup_service_error_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK "wakeup_manager_set_streaming_section_changed_callback" @@ -205,6 +207,7 @@ typedef struct { wakeup_manager_set_previous_utterance_streaming_callback set_previous_utterance_streaming_callback; wakeup_manager_set_follow_up_streaming_callback set_follow_up_streaming_callback; wakeup_manager_set_speech_status_callback set_speech_status_callback; + wakeup_manager_set_setting_changed_callback set_setting_changed_callback; wakeup_manager_set_error_callback set_error_callback; wakeup_manager_set_streaming_section_changed_callback set_streaming_section_changed_callback; } wakeup_manager_interface; diff --git a/inc/multi_wakeup_recognizer.h b/inc/multi_wakeup_recognizer.h index 6c3eb6f..b0732f9 100644 --- a/inc/multi_wakeup_recognizer.h +++ b/inc/multi_wakeup_recognizer.h @@ -99,6 +99,8 @@ typedef void (*wakeup_service_speech_streaming_cb)(wakeup_speech_streaming_event typedef void (*wakeup_service_speech_status_cb)(wakeup_speech_status_e status, void *user_data); +typedef void (*wakeup_service_setting_changed_cb)(void *user_data); + typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* user_data); typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index e6f5251..d574646 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -50,6 +50,12 @@ public: virtual void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0; }; +class ISettingValueObserver { +public: + virtual ~ISettingValueObserver() = default; + virtual void on_value_changed() = 0; +}; + /* If a wakeup event is raised by pressing a voice key, the wakeup_engine attribute of wakeup_event_info will have the following value */ #define WAKEUP_ENGINE_VOICE_KEY "voice_key" @@ -57,7 +63,7 @@ public: class CWakeupManager { public: - CWakeupManager(IWakeupEventObserver *observer); + CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingValueObserver* setting_observer); virtual ~CWakeupManager(); CWakeupManager(const CWakeupManager&) = delete; @@ -66,8 +72,11 @@ public: bool initialize(); bool deinitialize(); - void subscribe(IWakeupEventObserver *observer); - void unsubscribe(IWakeupEventObserver *observer); + void subscribe_wakeup_observer(IWakeupEventObserver* observer); + void unsubscribe_wakeup_observer(IWakeupEventObserver* observer); + + void subscribe_setting_observer(ISettingValueObserver* observer); + void unsubscribe_setting_observer(ISettingValueObserver* observer); bool activate(); bool deactivate(); @@ -107,9 +116,11 @@ public: CAudioManager* get_audio_manager(); CWakeupSettings* get_wakeup_settings(); - vector get_observers(); + vector get_wakeup_observers(); void set_last_wakeup_event_info(wakeup_event_info wakeup_info); + vector get_setting_observers(); + bool change_manager_state(wakeup_manager_state_e state); wakeup_manager_state_e get_manager_state(); @@ -178,7 +189,8 @@ private: map mAssistantSupportsCurrentLanguage; map mAssistantEnabled; - vector mObservers; + vector mWakeupObservers; + vector mSettingObservers; unique_ptr mWakeupPolicy; diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index 7f408dc..2df503e 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -63,6 +63,8 @@ typedef void (*wakeup_service_speech_streaming_cb)(wakeup_speech_streaming_event typedef void (*wakeup_service_speech_status_cb)(wakeup_service_speech_status_e status, void *user_data); +typedef void (*wakeup_service_setting_changed_cb)( void *user_data); + typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* user_data); typedef void (*wakeup_service_audio_data_require_status_cb)(bool require, void* user_data); @@ -146,6 +148,8 @@ EXPORT_API int wakeup_manager_set_follow_up_streaming_callback(wakeup_service_sp EXPORT_API int wakeup_manager_set_speech_status_callback(wakeup_service_speech_status_cb callback, void* user_data); +EXPORT_API int wakeup_manager_set_setting_changed_callback(wakeup_service_setting_changed_cb callback, void* user_data); + EXPORT_API int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* user_data); EXPORT_API int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streaming_section_changed_cb callback, void* user_data); diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 81ed6eb..9e16d36 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -54,12 +54,15 @@ static bool initialize_wakeup_event_info(wakeup_event_info* wakeup_info) return ret; } -CWakeupManager::CWakeupManager(IWakeupEventObserver *observer) +CWakeupManager::CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingValueObserver* setting_observer) { initialize_wakeup_event_info(&mLastWakeupEventInfo); - if (observer) { - subscribe(observer); + if (wakeup_observer) { + subscribe_wakeup_observer(wakeup_observer); + } + if (setting_observer) { + subscribe_setting_observer(setting_observer); } } @@ -136,16 +139,29 @@ bool CWakeupManager::deinitialize() return true; } -void CWakeupManager::subscribe(IWakeupEventObserver *observer) +void CWakeupManager::subscribe_wakeup_observer(IWakeupEventObserver *observer) { - mObservers.push_back(observer); + mWakeupObservers.push_back(observer); } -void CWakeupManager::unsubscribe(IWakeupEventObserver *observer) +void CWakeupManager::unsubscribe_wakeup_observer(IWakeupEventObserver *observer) { - auto iter = find(mObservers.begin(), mObservers.end(), observer); - if (iter != mObservers.end()) { - mObservers.erase(iter); + auto iter = find(mWakeupObservers.begin(), mWakeupObservers.end(), observer); + if (iter != mWakeupObservers.end()) { + mWakeupObservers.erase(iter); + } +} + +void CWakeupManager::subscribe_setting_observer(ISettingValueObserver* observer) +{ + mSettingObservers.push_back(observer); +} + +void CWakeupManager::unsubscribe_setting_observer(ISettingValueObserver* observer) +{ + auto iter = find(mSettingObservers.begin(), mSettingObservers.end(), observer); + if (iter != mSettingObservers.end()) { + mSettingObservers.erase(iter); } } @@ -469,7 +485,7 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) set_last_wakeup_event_info(wakeup_info); mWakeupEngineManager.set_selected_wakeup_info(wakeup_info); - for (const auto& observer : mObservers) { + for (const auto& observer : mWakeupObservers) { observer->on_wakeup(wakeup_info); } } @@ -498,9 +514,9 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) return true; } -vector CWakeupManager::get_observers() +vector CWakeupManager::get_wakeup_observers() { - return mObservers; + return mWakeupObservers; } void CWakeupManager::set_last_wakeup_event_info(wakeup_event_info wakeup_info) @@ -508,6 +524,11 @@ void CWakeupManager::set_last_wakeup_event_info(wakeup_event_info wakeup_info) mLastWakeupEventInfo = wakeup_info; } +vector CWakeupManager::get_setting_observers() +{ + return mSettingObservers; +} + static Eina_Bool streaming_duration_expired(void *data) { MWR_LOGD("[ENTER]"); @@ -534,7 +555,7 @@ static Eina_Bool streaming_duration_expired(void *data) } unsigned char final_buffer[2] = {'\0', }; - vector observers = wakeup_manager->get_observers(); + vector observers = wakeup_manager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_streaming_audio_data( WAKEUP_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer)); @@ -841,7 +862,7 @@ bool CWakeupManager::CEngineEventObserver::on_streaming_audio_data( { if (nullptr == mWakeupManager) return false; - vector observers = mWakeupManager->get_observers(); + vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_streaming_audio_data(event, buffer, len); } @@ -857,7 +878,7 @@ bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section( { if (nullptr == mWakeupManager) return false; - vector observers = mWakeupManager->get_observers(); + vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_audio_streaming_data_section(section); } @@ -878,7 +899,7 @@ void CWakeupManager::CPolicyEventObserver::on_wakeup(wakeup_event_info wakeup_in mWakeupManager->set_last_wakeup_event_info(wakeup_info); engine_manager->set_selected_wakeup_info(wakeup_info); - vector observers = mWakeupManager->get_observers(); + vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_wakeup(wakeup_info); } @@ -906,7 +927,7 @@ bool CWakeupManager::CAudioEventObserver::on_streaming_audio_data( { if (nullptr == mWakeupManager) return false; - vector observers = mWakeupManager->get_observers(); + vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_streaming_audio_data(event, buffer, len); } @@ -922,6 +943,10 @@ bool CWakeupManager::CSettingsEventObserver::on_voice_input_language_changed( { if (nullptr == mWakeupManager || nullptr == language) return false; mWakeupManager->set_language(std::string(language)); + vector observers = mWakeupManager->get_setting_observers(); + for (const auto& observer : observers) { + observer->on_value_changed(); + } return true; } @@ -930,6 +955,10 @@ bool CWakeupManager::CSettingsEventObserver::on_assistant_enabled_info_changed( { if (nullptr == mWakeupManager || nullptr == appid) return false; mWakeupManager->set_assistant_enabled(std::string(appid), enabled); + vector observers = mWakeupManager->get_setting_observers(); + for (const auto& observer : observers) { + observer->on_value_changed(); + } return true; } @@ -938,6 +967,10 @@ bool CWakeupManager::CSettingsEventObserver::on_default_assistant_appid_changed( { if (nullptr == mWakeupManager || nullptr == appid) return false; mWakeupManager->set_default_assistant(std::string(appid)); + vector observers = mWakeupManager->get_setting_observers(); + for (const auto& observer : observers) { + observer->on_value_changed(); + } return true; } diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 9853c34..1c72602 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -34,6 +34,9 @@ static void* g_follow_up_streaming_user_data; static wakeup_service_speech_status_cb g_speech_status_cb; static void* g_speech_status_user_data; +static wakeup_service_setting_changed_cb g_setting_changed_cb; +static void* g_setting_changed_user_data; + static wakeup_service_error_cb g_error_cb; static void* g_error_user_data; @@ -48,8 +51,14 @@ class CWakeupEventObserver : public IWakeupEventObserver void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override; }; +class CSettingValueObserver : public ISettingValueObserver +{ + void on_value_changed() override; +}; + static CWakeupEventObserver g_wakeup_event_observer; -static CWakeupManager g_wakeup_manager(&g_wakeup_event_observer); +static CSettingValueObserver g_setting_value_observer; +static CWakeupManager g_wakeup_manager(&g_wakeup_event_observer, &g_setting_value_observer); int wakeup_manager_initialize(void) { @@ -67,6 +76,9 @@ int wakeup_manager_initialize(void) g_speech_status_cb = NULL; g_speech_status_user_data = NULL; + g_setting_changed_cb = NULL; + g_setting_changed_user_data = NULL; + g_error_cb = NULL; g_error_user_data = NULL; @@ -481,6 +493,22 @@ int wakeup_manager_set_speech_status_callback(wakeup_service_speech_status_cb ca return 0; } +int wakeup_manager_set_setting_changed_callback(wakeup_service_setting_changed_cb callback, void* user_data) +{ + MWR_LOGD("[ENTER]"); + + if (NULL == callback) { + MWR_LOGE("[ERROR] Input parameter is NULL"); + return -1; + } + + g_setting_changed_cb = callback; + g_setting_changed_user_data = user_data; + + MWR_LOGD("[END]"); + return 0; +} + int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* user_data) { MWR_LOGD("[ENTER]"); @@ -549,3 +577,10 @@ void CWakeupEventObserver::on_audio_streaming_data_section( g_streaming_section_changed_cb(section, g_streaming_section_changed_user_data); } } + +void CSettingValueObserver::on_value_changed() +{ + if (g_setting_changed_cb) { + g_setting_changed_cb(g_setting_changed_user_data); + } +} \ No newline at end of file diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index b2da67b..35cc1ed 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -762,16 +762,7 @@ static int init_wakeup(void) #endif } - /* CHECK NEEDED : should the code segment below and activation logic above be moved to wakeup manger? */ - int prelaunch_mode; - int res = vconf_get_bool(WAKEUP_SETTINGS_KEY_PRELAUNCH_MODE, &prelaunch_mode); - if (0 == res && 0 != prelaunch_mode) { - const char *default_assistant = NULL; - if (0 == multi_assistant_service_plugin_get_default_assistant(&default_assistant)) { - MAS_LOGD("prelaunching default_assistant_appid : %s", default_assistant); - mas_launch_client_by_appid(default_assistant, CLIENT_LAUNCH_MODE_PRELAUNCH); - } - } + mas_prelaunch_default_assistant(); /* For the case of preprocessing assistant, it always have to be launched beforehand */ char *vconf_str; @@ -1080,6 +1071,23 @@ int mas_launch_client_by_wakeup_word(const char *wakeup_word) return mas_launch_client_by_appid(appid, CLIENT_LAUNCH_MODE_ACTIVATION); } +int mas_prelaunch_default_assistant() +{ + /* CHECK NEEDED : should the code segment below and activation logic above be moved to wakeup manger? */ + int prelaunch_mode; + int res = vconf_get_bool(WAKEUP_SETTINGS_KEY_PRELAUNCH_MODE, &prelaunch_mode); + if (0 == res && 0 != prelaunch_mode) { + const char *default_assistant = NULL; + if (0 == multi_assistant_service_plugin_get_default_assistant(&default_assistant)) { + if (0 == aul_app_is_running(default_assistant)) { + MAS_LOGD("prelaunching default_assistant_appid : %s", default_assistant); + mas_launch_client_by_appid(default_assistant, CLIENT_LAUNCH_MODE_PRELAUNCH); + } + } + } + return 0; +} + int mas_process_voice_key_event(bool pressed) { diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 5e9e063..8307104 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -337,6 +337,12 @@ static void __error_cb(int error, const char* err_msg, void* user_data) } } +static void __setting_changed_cb(void *user_data) +{ + mas_prelaunch_default_assistant(); + MAS_LOGD( "[SUCCESS] __setting_changed_cb is called"); +} + static void __streaming_section_changed_cb(ma_audio_streaming_data_section_e section, void* user_data) { MAS_LOGD( "[SUCCESS] __streaming_section_changed_cb is called, section(%d)", section); @@ -454,6 +460,9 @@ int multi_assistant_service_plugin_initialize(void) _wakeup_manager_interface.set_speech_status_callback = (wakeup_manager_set_speech_status_callback)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_SPEECH_STATUS_CALLBACK); + _wakeup_manager_interface.set_setting_changed_callback = + (wakeup_manager_set_setting_changed_callback)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_SET_SETTING_CHANGED_CALLBACK); _wakeup_manager_interface.set_error_callback = (wakeup_manager_set_error_callback)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_ERROR_CALLBACK); @@ -996,6 +1005,12 @@ int multi_assistant_service_plugin_set_callbacks(void) return ret; } + ret = multi_assistant_service_plugin_set_setting_changed_callback(__setting_changed_cb, NULL); + if (0 != ret) { + MAS_LOGE("Fail to set setting changed cb"); + return ret; + } + ret = multi_assistant_service_plugin_set_error_callback(__error_cb, NULL); if (0 != ret) { MAS_LOGE("Fail to set error cb"); @@ -1104,6 +1119,23 @@ int multi_assistant_service_plugin_set_speech_status_callback(wakeup_service_spe return ret; } +int multi_assistant_service_plugin_set_setting_changed_callback(wakeup_service_setting_changed_cb callback, void* user_data) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_set_setting_changed_callback func = _wakeup_manager_interface.set_setting_changed_callback; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_SETTING_CHANGED_CALLBACK); + } else { + ret = func(callback, user_data); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set setting_changed callback, ret(%d)", ret); + } + } + } + return ret; +} + int multi_assistant_service_plugin_set_error_callback(wakeup_service_error_cb callback, void* user_data) { int ret = -1; -- 2.7.4 From 02784ac4a8b72a15134b394c2bbee278eb27333c Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 29 Oct 2019 16:04:40 +0900 Subject: [PATCH 08/16] Bump version to 0.2.6 Change-Id: I0d42ae4371b25e7e24e8cd4329251ba39f672fe9 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 483d407..15f683c 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 9f783fc..127eeb5 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.5 +Version: 0.2.6 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 11a6ddfc6830fce8f1850c0dbab91a0dc614286e Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 31 Oct 2019 20:16:18 +0900 Subject: [PATCH 09/16] Finalize audio data when voice key is in pressed state When audio is finalized after voice key variable is set to false, the FINISH audio event will be fed to wakeup engines where the event itself is not useful for wakeup engines. Change-Id: Ifdd41be5e76a84b779d697edefd57203ac2bfe5a --- plugins/wakeup-manager/src/wakeup_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 9e16d36..1ac4694 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -491,8 +491,8 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) } } else if (MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_PUSH == event) { if (mVoiceKeyPressed != false) { - mVoiceKeyPressed = false; mAudioManager.finalize_audio_data(); + mVoiceKeyPressed = false; if (mWakeupEngineManager.get_audio_data_required()) { /* Restart recorder thread using appropriate recording device */ -- 2.7.4 From 199d497dec43a6f3b1a20844c64bf9feccd4bd55 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 31 Oct 2019 20:20:28 +0900 Subject: [PATCH 10/16] Change manager state right away on voice key release Change-Id: I97f074bfca65f0af8e770eae675dbf46fbb8713d --- plugins/wakeup-manager/src/wakeup_manager.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 1ac4694..12b82a8 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -494,6 +494,12 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) mAudioManager.finalize_audio_data(); mVoiceKeyPressed = false; + if (STREAMING_MODE::UTTERANCE == mStreamingMode) { + change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING); + } else { + change_manager_state(WAKEUP_MANAGER_STATE_LISTENING); + } + if (mWakeupEngineManager.get_audio_data_required()) { /* Restart recorder thread using appropriate recording device */ mAudioManager.stop_recording(true); -- 2.7.4 From d9c3427e05d7a12872ee2161071aa60494493bea Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 31 Oct 2019 20:22:42 +0900 Subject: [PATCH 11/16] Stop streaming only when necessary Streaming stop function was being called inside wakeup_manager state change function, which results invoking unnecessary streaming stop functions in many cases. Change-Id: I723b187375ce3a8d6c4b4933d6577c7a9be758d2 --- plugins/wakeup-manager/src/wakeup_manager.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 12b82a8..ece33b9 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -195,6 +195,10 @@ bool CWakeupManager::deactivate(void) mAudioManager.stop_recording(true); change_manager_state(WAKEUP_MANAGER_STATE_INACTIVE); + stop_streaming_utterance_data(); + stop_streaming_previous_utterance_data(); + stop_streaming_follow_up_data(); + MWR_LOGD("[END]"); return true; } @@ -356,15 +360,6 @@ bool CWakeupManager::set_streaming_mode(STREAMING_MODE mode) bool CWakeupManager::change_manager_state(wakeup_manager_state_e state) { - if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState || - WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState) { - if (WAKEUP_MANAGER_STATE_VOICE_FEEDBACK == state || - WAKEUP_MANAGER_STATE_LISTENING == state) { - stop_streaming_utterance_data(); - stop_streaming_previous_utterance_data(); - stop_streaming_follow_up_data(); - } - } mWakeupManagerState = state; mWakeupEngineManager.update_manager_state(state); return true; @@ -431,6 +426,9 @@ bool CWakeupManager::update_recognition_result(string appid, int result) if (WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState || WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { change_manager_state(WAKEUP_MANAGER_STATE_LISTENING); + stop_streaming_utterance_data(); + stop_streaming_previous_utterance_data(); + stop_streaming_follow_up_data(); } MWR_LOGD("[END]"); return true; -- 2.7.4 From f3721c04a7476621d70fc65f4ae35447a8fde0d8 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 31 Oct 2019 20:25:41 +0900 Subject: [PATCH 12/16] Bump version to 0.2.7 Change-Id: If861f52ac4920f04326c8a399adffa7c4f58c9bb --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 15f683c..d18e9c7 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 127eeb5..1dca5f5 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.6 +Version: 0.2.7 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From a2aa3a6561b314fbd9bda2d2996deebb9b59d62a Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 16:00:41 +0900 Subject: [PATCH 13/16] Apply secure options to compile flags Change-Id: Ia582efe7e1b7d7beb6b1fe989ada3f4f47ef6559 --- CMakeLists.txt | 4 ++-- plugins/wakeup-manager/CMakeLists.txt | 2 +- plugins/wakeup-manager/dependency-default/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59845fa..1cf3c04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,8 +36,8 @@ FOREACH(flag ${pkgs_CFLAGS}) ENDFOREACH(flag) SET(CMAKE_EXE_LINKER_FLAGS "-Wl,--as-needed -pie") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CFLAGS} -fPIE -std=c++11 -fvisibility=hidden -fvisibility-inlines-hidden") -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_CFLAGS} -fPIE -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIE -pie -std=c++11 -fvisibility=hidden -fvisibility-inlines-hidden") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_CFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIE -pie -fvisibility=hidden") # SET WARNNING OPTION ---------------------------------------------------------------------------- SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") diff --git a/plugins/wakeup-manager/CMakeLists.txt b/plugins/wakeup-manager/CMakeLists.txt index 752f085..7f07524 100644 --- a/plugins/wakeup-manager/CMakeLists.txt +++ b/plugins/wakeup-manager/CMakeLists.txt @@ -58,7 +58,7 @@ SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -fPIC -Wall" ) IF("${_TV_PRODUCT}" STREQUAL "TRUE") SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -DTV_PRODUCT") ENDIF() -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -std=c++11 -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIC -std=c++11 -fvisibility=hidden") SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") ADD_LIBRARY(${PROJECT_NAME} SHARED ${SRCS} ) diff --git a/plugins/wakeup-manager/dependency-default/CMakeLists.txt b/plugins/wakeup-manager/dependency-default/CMakeLists.txt index 163ede8..a08730d 100644 --- a/plugins/wakeup-manager/dependency-default/CMakeLists.txt +++ b/plugins/wakeup-manager/dependency-default/CMakeLists.txt @@ -49,7 +49,7 @@ ENDFOREACH(flag) SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -fPIC -Wall" ) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -std=c++11 -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIC -std=c++11 -fvisibility=hidden") SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") ADD_LIBRARY(${PROJECT_NAME} SHARED ${SRCS} ) -- 2.7.4 From dc1e7118f4ca47e7772e334d24de5c84c16a4654 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 16:01:59 +0900 Subject: [PATCH 14/16] Bump version to 0.2.8 Change-Id: I14024040e9cc66a2687f7bba6099f39c2f0dcc8f --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index d18e9c7..3e31401 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 1dca5f5..486ab55 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.7 +Version: 0.2.8 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 905b2b79394c84db6ac5f2051a6702f1e253e97f Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 4 Nov 2019 11:32:56 +0900 Subject: [PATCH 15/16] Add reload feature for updating pluings on-the-fly Change-Id: Ibf873b8861b22807f9b51f5f8e92efb82f8520ba --- .../src/dependency_default_audio.cpp | 36 ++++++- .../wakeup-manager/src/wakeup_audio_manager.cpp | 12 +++ .../wakeup-manager/src/wakeup_engine_manager.cpp | 7 ++ .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 117 +++++++++++++++------ src/multi_assistant_service.c | 54 +++++++--- src/multi_assistant_service_plugin.c | 2 + 6 files changed, 184 insertions(+), 44 deletions(-) diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp index e803739..d1662c8 100644 --- a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp @@ -17,6 +17,7 @@ static mas_proxy_interface g_proxy_interface; using namespace std; static audio_in_h g_audio_in = NULL; +static audio_io_state_e g_audio_io_state = AUDIO_IO_STATE_IDLE; static sound_stream_info_h g_volume_stream = NULL; static virtual_sound_stream_h g_virtual_sound_stream = NULL; @@ -111,6 +112,13 @@ static void _device_connection_changed_cb(sound_device_h device, bool isConnecte return; } +static void _audio_in_state_changed_cb(audio_in_h handle, audio_io_state_e previous, audio_io_state_e current, bool by_policy, void *user_data) +{ + if (handle == g_audio_in) { + g_audio_io_state = current; + } +} + void dependency_default_audio_initialize(mas_proxy_interface interfaces) { const int rate = 16000; @@ -155,12 +163,24 @@ void dependency_default_audio_initialize(mas_proxy_interface interfaces) return; } + ret = audio_in_set_state_changed_cb(g_audio_in, _audio_in_state_changed_cb, NULL); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGE("[Recorder ERROR] Fail to set state changed callback : %d", ret); + sound_manager_remove_device_connection_changed_cb(g_device_id); + sound_manager_destroy_stream_information(g_stream_info); + g_stream_info = NULL; + audio_in_destroy(g_audio_in); + g_audio_in = NULL; + return; + } + ret = audio_in_set_sound_stream_info(g_audio_in, g_stream_info); if (AUDIO_IO_ERROR_NONE != ret) { LOGE("[Recorder ERROR] Fail to set stream info : %d", ret); sound_manager_remove_device_connection_changed_cb(g_device_id); sound_manager_destroy_stream_information(g_stream_info); g_stream_info = NULL; + audio_in_unset_state_changed_cb(g_audio_in); audio_in_destroy(g_audio_in); g_audio_in = NULL; return; @@ -177,6 +197,7 @@ void dependency_default_audio_initialize(mas_proxy_interface interfaces) sound_manager_remove_device_connection_changed_cb(g_device_id); sound_manager_destroy_stream_information(g_stream_info); g_stream_info = NULL; + audio_in_unset_state_changed_cb(g_audio_in); audio_in_destroy(g_audio_in); g_audio_in = NULL; return; @@ -231,6 +252,11 @@ void dependency_default_audio_deinitialize() } if (g_audio_in) { + ret = audio_in_unset_state_changed_cb(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGD("[Recorder ERROR] Fail to unset state changed callback : %d", ret); + } + ret = audio_in_destroy(g_audio_in); if (AUDIO_IO_ERROR_NONE != ret) { LOGD("[Recorder ERROR] Fail to destroy audio : %d", ret); @@ -271,7 +297,10 @@ static void recorder_thread_func() void dependency_default_audio_start_recording() { if (g_audio_in) { - int ret = audio_in_resume(g_audio_in); + int ret = AUDIO_IO_ERROR_NONE; + if (AUDIO_IO_STATE_RUNNING != g_audio_io_state) { + ret = audio_in_resume(g_audio_in); + } if (AUDIO_IO_ERROR_NONE != ret) { LOGD("[Recorder ERROR] Fail to resume audio in : %d", ret); } @@ -289,7 +318,10 @@ void dependency_default_audio_stop_recording() g_recorder_thread.join(); } if (g_audio_in) { - int ret = audio_in_pause(g_audio_in); + int ret = AUDIO_IO_ERROR_NONE; + if (AUDIO_IO_STATE_RUNNING == g_audio_io_state) { + ret = audio_in_pause(g_audio_in); + } if (AUDIO_IO_ERROR_NONE != ret) { LOGD("[Recorder ERROR] Fail to pause audio in : %d", ret); } diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp index 8b68362..94ce56e 100644 --- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp @@ -68,6 +68,18 @@ int CAudioManager::initialize(void) int CAudioManager::deinitialize(void) { + clear_audio_data(); + if (mStreamingThread.joinable()) { + MWR_LOGD("mStreamingThread is joinable, trying join()"); + mStopStreamingThread.store(true); + try { + mStreamingThread.join(); + } catch (std::exception &e) { + MWR_LOGE("Exception thrown : %s", e.what()); + } + } + mStopStreamingThread.store(false); + sound_manager_remove_focus_state_watch_cb(mSoundFocusWatchId); return 0; diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index 5e1163c..ebc50a9 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -76,6 +76,13 @@ void CWakeupEngineManager::initialize() void CWakeupEngineManager::deinitialize() { + if (mStreamingThread.joinable()) { + MWR_LOGD("mStreamingThread is joinable, trying join()"); + mStopStreamingThread.store(true); + mStreamingThread.join(); + } + mStopStreamingThread.store(false); + for (auto& info : mEngineInfo) { try { if (info.interface.set_wakeup_event_callback) { diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 1c72602..77549bb 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -17,6 +17,8 @@ #include "wakeup_manager_main.h" #include "wakeup_manager.h" +#include + using namespace multiassistant::wakeup; static wakeup_service_wakeup_event_cb g_wakeup_event_cb; @@ -56,14 +58,29 @@ class CSettingValueObserver : public ISettingValueObserver void on_value_changed() override; }; -static CWakeupEventObserver g_wakeup_event_observer; -static CSettingValueObserver g_setting_value_observer; -static CWakeupManager g_wakeup_manager(&g_wakeup_event_observer, &g_setting_value_observer); +static std::unique_ptr g_wakeup_event_observer; +static std::unique_ptr g_setting_value_observer; +static std::unique_ptr g_wakeup_manager; int wakeup_manager_initialize(void) { MWR_LOGD("[ENTER]"); + try { + g_wakeup_event_observer.reset(new CWakeupEventObserver); + g_setting_value_observer.reset(new CSettingValueObserver); + if (NULL == g_wakeup_event_observer || NULL == g_setting_value_observer) { + return -1; + } + g_wakeup_manager.reset(new CWakeupManager(g_wakeup_event_observer.get(), g_setting_value_observer.get())); + if (NULL == g_wakeup_manager) { + return -1; + } + } catch (const std::exception& e) { + MWR_LOGE("Exception caught : %s", e.what()); + return -1; + } + g_wakeup_event_cb = NULL; g_wakeup_event_user_data = NULL; @@ -82,7 +99,7 @@ int wakeup_manager_initialize(void) g_error_cb = NULL; g_error_user_data = NULL; - g_wakeup_manager.initialize(); + g_wakeup_manager->initialize(); MWR_LOGD("[END]"); return 0; @@ -92,7 +109,18 @@ int wakeup_manager_deinitialize(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.deinitialize(); + if (nullptr == g_wakeup_manager) return -1; + + try { + g_wakeup_manager->deinitialize(); + g_wakeup_manager.reset(); + + g_wakeup_event_observer.reset(); + g_setting_value_observer.reset(); + } catch (const std::exception& e) { + MWR_LOGE("Exception caught : %s", e.what()); + return -1; + } MWR_LOGD("[END]"); return 0; @@ -104,7 +132,9 @@ int wakeup_manager_get_settings(ma_plugin_settings **settings, size_t *struct_si MWR_LOGD("[ERROR] Parameter is invalid, settings(%p), struct_size(%p)", settings, struct_size); return -1; } - CWakeupSettings* wakeup_settings = g_wakeup_manager.get_wakeup_settings(); + + if (nullptr == g_wakeup_manager) return -1; + CWakeupSettings* wakeup_settings = g_wakeup_manager->get_wakeup_settings(); if (wakeup_settings) { const int PLUGIN_VERSION = 1; static ma_plugin_settings current_settings; @@ -125,7 +155,9 @@ int wakeup_manager_add_assistant_wakeup_word(const char* appid, const char* wake MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), wakeup_word(%s), language(%s)", appid, wakeup_word, language); return -1; } - g_wakeup_manager.add_assistant_wakeup_word( + + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->add_assistant_wakeup_word( string{appid}, string{wakeup_word}, (language ? string{language} : string{})); MWR_LOGD("[END]"); @@ -141,7 +173,8 @@ int wakeup_manager_add_assistant_language(const char* appid, const char* languag return -1; } - g_wakeup_manager.add_assistant_language(string{appid}, string{language}); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->add_assistant_language(string{appid}, string{language}); MWR_LOGD("[DEBUG] language(%s)", language); @@ -158,8 +191,9 @@ int wakeup_manager_set_assistant_wakeup_engine(const char* appid, const char* en return -1; } + if (nullptr == g_wakeup_manager) return -1; MWR_LOGD("[DEBUG] appid(%s), wakeup engine(%s)", appid, engine); - g_wakeup_manager.set_assistant_wakeup_engine(string{appid}, string{engine}); + g_wakeup_manager->set_assistant_wakeup_engine(string{appid}, string{engine}); MWR_LOGD("[END]"); return 0; @@ -174,8 +208,9 @@ int wakeup_manager_set_default_assistant(const char* appid) return -1; } + if (nullptr == g_wakeup_manager) return -1; MWR_LOGD("[DEBUG] default_assistant appid(%s)", appid); - g_wakeup_manager.set_default_assistant(string{appid}); + g_wakeup_manager->set_default_assistant(string{appid}); MWR_LOGD("[END]"); return 0; @@ -184,7 +219,9 @@ int wakeup_manager_set_default_assistant(const char* appid) int wakeup_manager_get_default_assistant(const char** appid) { static string default_assistant; - default_assistant = g_wakeup_manager.get_default_assistant(); + + if (nullptr == g_wakeup_manager) return -1; + default_assistant = g_wakeup_manager->get_default_assistant(); if (default_assistant.empty()) { *appid = nullptr; @@ -203,7 +240,8 @@ int wakeup_manager_set_language(const char* language) return -1; } - g_wakeup_manager.set_language(string{language}); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_language(string{language}); MWR_LOGD("[END]"); return 0; @@ -211,7 +249,8 @@ int wakeup_manager_set_language(const char* language) int wakeup_manager_change_state(wakeup_manager_state_e state) { - g_wakeup_manager.change_manager_state(state); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->change_manager_state(state); return 0; } @@ -219,7 +258,8 @@ int wakeup_manager_activate(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.activate(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->activate(); MWR_LOGD("[END]"); return 0; @@ -229,7 +269,8 @@ int wakeup_manager_deactivate(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.deactivate(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->deactivate(); MWR_LOGD("[END]"); return 0; @@ -246,7 +287,8 @@ int wakeup_manager_update_voice_feedback_state(const char* appid, int state) appid_string = appid; } - g_wakeup_manager.update_voice_feedback_state(appid_string, state); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->update_voice_feedback_state(appid_string, state); MWR_LOGD("[END]"); return 0; @@ -264,7 +306,8 @@ int wakeup_manager_send_assistant_specific_command(const char* appid, const char if (appid) appid_string = appid; if (command) command_string = command; - g_wakeup_manager.send_assistant_specific_command(appid_string, command_string); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->send_assistant_specific_command(appid_string, command_string); MWR_LOGD("[END]"); return 0; @@ -281,7 +324,8 @@ int wakeup_manager_set_background_volume(const char* appid, double ratio) appid_string = appid; } - g_wakeup_manager.set_background_volume(appid_string, ratio); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_background_volume(appid_string, ratio); MWR_LOGD("[END]"); return 0; @@ -297,7 +341,9 @@ int wakeup_manager_update_recognition_result(const char* appid, int result) } else { appid_string = appid; } - g_wakeup_manager.update_recognition_result(appid_string, result); + + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->update_recognition_result(appid_string, result); MWR_LOGD("[END]"); return 0; @@ -307,7 +353,8 @@ int wakeup_manager_process_event(int event, void* data, int len) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.process_event(static_cast(event), data, len); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->process_event(static_cast(event), data, len); MWR_LOGD("[END]"); return 0; @@ -317,7 +364,8 @@ int wakeup_manager_start_streaming_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.start_streaming_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->start_streaming_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -327,7 +375,8 @@ int wakeup_manager_stop_streaming_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.stop_streaming_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->stop_streaming_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -337,7 +386,8 @@ int wakeup_manager_start_streaming_follow_up_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.start_streaming_follow_up_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->start_streaming_follow_up_data(); MWR_LOGD("[END]"); return 0; @@ -347,7 +397,8 @@ int wakeup_manager_stop_streaming_follow_up_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.stop_streaming_follow_up_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->stop_streaming_follow_up_data(); MWR_LOGD("[END]"); return 0; @@ -357,7 +408,8 @@ int wakeup_manager_start_streaming_previous_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.start_streaming_previous_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->start_streaming_previous_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -367,7 +419,8 @@ int wakeup_manager_stop_streaming_previous_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.stop_streaming_previous_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->stop_streaming_previous_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -382,7 +435,8 @@ int wakeup_manager_get_audio_format(int* rate, int* channel, int* audio_type) return -1; } - g_wakeup_manager.get_audio_format(rate, channel, audio_type); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->get_audio_format(rate, channel, audio_type); MWR_LOGD("[END] rate(%d), channel(%d), audio_type(%d)", *rate, *channel, *audio_type); return 0; @@ -397,7 +451,8 @@ int wakeup_manager_get_audio_source_type(char** type) return -1; } - g_wakeup_manager.get_audio_source_type(type); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->get_audio_source_type(type); MWR_LOGD("[END] type(%s)", *type); return 0; @@ -407,7 +462,8 @@ int wakeup_manager_set_wake_word_audio_require_flag(bool require) { MWR_LOGD("[ENTER] : %d", require); - g_wakeup_manager.set_wake_word_audio_require_flag(require); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_wake_word_audio_require_flag(require); MWR_LOGD("[END]"); return 0; @@ -543,7 +599,8 @@ int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streami int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len) { - g_wakeup_manager.feed_audio_data(event, buffer, len); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->feed_audio_data(event, buffer, len); return 0; } diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 35cc1ed..137c7de 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -677,15 +677,8 @@ static void mas_active_state_changed_cb(keynode_t* key, void* data) } } -static int init_wakeup(void) +static int init_plugin(void) { - MAS_LOGD("[Enter] init_wakeup"); - - int ret = mas_dbus_open_connection(); - if (0 != ret) { - MAS_LOGE("[ERROR] Fail to open connection"); - } - if (0 != multi_assistant_service_plugin_initialize()) { MAS_LOGE("Fail to ws intialize"); return -1; @@ -740,6 +733,18 @@ static int init_wakeup(void) return -1; } + return 0; +} + +static int deinit_plugin(void) +{ + if (0 != multi_assistant_service_plugin_deinitialize()) { + MAS_LOGE("Fail to deinitialize"); + } +} + +static int process_multi_assistant_activated_setting() +{ if (0 == vconf_notify_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb, NULL)) { /* Activate / deactivate according to the vconf key setting */ mas_active_state_changed_cb(NULL, NULL); @@ -761,6 +766,21 @@ static int init_wakeup(void) } #endif } + return 0; +} + +static int init_wakeup(void) +{ + MAS_LOGD("[Enter] init_wakeup"); + + int ret = mas_dbus_open_connection(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to open connection"); + } + + init_plugin(); + + process_multi_assistant_activated_setting(); mas_prelaunch_default_assistant(); @@ -786,6 +806,7 @@ static void deinit_wakeup(void) g_current_lang = NULL; } */ + deinit_plugin(); vconf_ignore_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb); @@ -793,10 +814,6 @@ static void deinit_wakeup(void) if (0 != ret) { MAS_LOGE("[ERROR] Fail to close connection"); } - - if (0 != multi_assistant_service_plugin_deinitialize()) { - MAS_LOGE("Fail to ws deinitialize"); - } } int mas_get_current_client_pid() @@ -1246,6 +1263,19 @@ void service_app_terminate(void *data) void service_app_control(app_control_h app_control, void *data) { // Todo: add your code here. + char *operation = NULL; + int ret = app_control_get_operation(app_control, &operation); + if (ret == APP_CONTROL_ERROR_NONE && operation) { + //strncmp(operation, APP_CONTROL_OPERATION_LAUNCH_ON_EVENT, strlen(operation)) == 0){ + char* value = NULL; + app_control_get_extra_data(app_control, "event", &value); + LOGD("operation : %s , %s", operation, value); + if (value && 0 == strcmp(value, "reload")) { + deinit_plugin(); + init_plugin(); + process_multi_assistant_activated_setting(); + } + } return; } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 8307104..b9f5f7a 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -511,6 +511,8 @@ int multi_assistant_service_plugin_deinitialize(void) MAS_LOGD("[Recorder ERROR] File not found!"); } #endif + multi_assistant_service_plugin_deactivate(); + int ret = -1; if (NULL != g_handle) { wakeup_manager_deinitialize func = _wakeup_manager_interface.deinitialize; -- 2.7.4 From 1393f5ccc126eee00404c22dbacadff58b93f9c4 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 19:07:56 +0900 Subject: [PATCH 16/16] Reload plugin modules depending on pkgmgr event Change-Id: I5c12c6600b80f2657e371b15c4eeac13c0044722 --- CMakeLists.txt | 1 + src/multi_assistant_service.c | 151 +++++++++++++++++++++++++++++++---- src/multi_assistant_service_plugin.c | 1 - 3 files changed, 137 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1cf3c04..1717bcf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ pkg_check_modules(pkgs REQUIRED libxml-2.0 multi-assistant vconf + pkgmgr-info ) # SET FLAG --------------------------------------------------------------------------------------- diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 137c7de..fc2f3cb 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -70,6 +72,7 @@ typedef struct { static int g_current_maclient_info = 0; static int g_current_preprocessing_maclient_info = -1; static const char *g_wakeup_maclient_appid = NULL; +static package_manager_h g_pkgmgr = NULL; static PREPROCESSING_STATE g_current_preprocessing_state = PREPROCESSING_STATE_NONE; @@ -738,12 +741,15 @@ static int init_plugin(void) static int deinit_plugin(void) { + if (0 != multi_assistant_service_plugin_deactivate()) { + MAS_LOGE("Fail to deactivate"); + } if (0 != multi_assistant_service_plugin_deinitialize()) { MAS_LOGE("Fail to deinitialize"); } } -static int process_multi_assistant_activated_setting() +static int process_activated_setting() { if (0 == vconf_notify_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb, NULL)) { /* Activate / deactivate according to the vconf key setting */ @@ -780,7 +786,7 @@ static int init_wakeup(void) init_plugin(); - process_multi_assistant_activated_setting(); + process_activated_setting(); mas_prelaunch_default_assistant(); @@ -1239,6 +1245,112 @@ int mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT event) return 0; } +static int pkg_app_list_cb(const pkgmgrinfo_appinfo_h handle, void *user_data) +{ + char *appid = NULL; + + int ret = pkgmgrinfo_appinfo_get_appid (handle, &appid); + if (PMINFO_R_OK == ret && NULL != appid) { + int *result = (int*)user_data; + if (result) { + for (int loop = 0; loop < MAX_MACLIENT_INFO_NUM;loop++) { + if (g_maclient_info[loop].used) { + LOGD("comparing appid : %s %s", g_maclient_info[loop].wakeup_engine, appid); + if (0 == strncmp(g_maclient_info[loop].wakeup_engine, appid, MAX_APPID_LEN)) { + *result = 1; + } + } + } + } + } else { + LOGE("pkgmgrinfo_appinfo_get_appid failed! error code=%d", ret); + return 0; + } + + return 0; +} + +/* +INFO: Package install/update/uninstall scenario +Install and Uninstall are obviously simple. + Install: just INSTALL + Uninstall: just UNINSTALL +Update package (change the source codes and Run As again), there are four scenarios: +1. UPDATE + Source code change +2. UNINSTALL -> INSTALL + This happens when Tizen IDE Property > Tizen SDK > Rapid Development Support > Check "Enable Project specific settings" + and change Application ID in tizen-manifest.xml file and Run As. +3. UPDATE -> INSTALL + This happens when Tizen IDE Property > Tizen SDK > Rapid Development Support > Uncheck "Enable Project specific settings" + and change Application ID in tizen-manifest.xml file and Run As. + At UPDATE event, pkgid (package parameter) is invalid... +4. UPDATE + Exceptionally, only UPDATE can be called when Application ID in tizen-manifest.xml file is changed. + At UPDATE event, pkgid (package parameter) is valid, and only appid is changed; the previous appid is invalid. +*/ +static void _package_manager_event_cb(const char *type, const char *package, package_manager_event_type_e event_type, package_manager_event_state_e event_state, int progress, package_manager_error_e error, void *user_data) +{ + int ret = 0; + uid_t uid = getuid (); + pkgmgrinfo_pkginfo_h handle = NULL; + static bool in_progress = false; + + if (!package || !type) + return; + + if (PACKAGE_MANAGER_EVENT_TYPE_UPDATE != event_type && + PACKAGE_MANAGER_EVENT_TYPE_INSTALL != event_type && + PACKAGE_MANAGER_EVENT_TYPE_UNINSTALL != event_type) + return; + + if (PACKAGE_MANAGER_EVENT_STATE_STARTED != event_state && + PACKAGE_MANAGER_EVENT_STATE_COMPLETED != event_state) + return; + + bool user = false; + LOGD("type=%s package=%s event_type=%d event_state=%d progress=%d error=%d", + type, package, event_type, event_state, progress, error); + ret = pkgmgrinfo_pkginfo_get_pkginfo(package, &handle); + if (ret != PMINFO_R_OK || NULL == handle) { + LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); + /* Try to get in user packages */ + ret = pkgmgrinfo_pkginfo_get_usr_pkginfo (package, uid, &handle); + if (ret != PMINFO_R_OK || NULL == handle) { + LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); + return; + } + } + + if (user) { + /* Try to get in user packages */ + pkgmgrinfo_appinfo_get_usr_list(handle, PMINFO_ALL_APP, pkg_app_list_cb, (void *)&ret, uid); + } else { + /* Try to get in global packages */ + pkgmgrinfo_appinfo_get_list(handle, PMINFO_ALL_APP, pkg_app_list_cb, (void *)&ret); + } + if (1 == ret) { + if (PACKAGE_MANAGER_EVENT_STATE_STARTED == event_state) { + LOGD("processing PACKAGE_MANAGER_EVENT_STATE_STARTED event"); + if (false == in_progress) { + in_progress = true; + deinit_plugin(); + } + } else if (PACKAGE_MANAGER_EVENT_STATE_COMPLETED == event_state) { + LOGD("processing PACKAGE_MANAGER_EVENT_STATE_COMPLETED event"); + if (true == in_progress) { + init_plugin(); + process_activated_setting(); + in_progress = false; + } + } + } + + pkgmgrinfo_pkginfo_destroy_pkginfo(handle); + + return; +} + bool service_app_create(void *data) { // Todo: add your code here. @@ -1250,11 +1362,33 @@ bool service_app_create(void *data) return false; } + if (!g_pkgmgr) { + int ret = package_manager_create(&g_pkgmgr); + if (ret == PACKAGE_MANAGER_ERROR_NONE) { + ret = package_manager_set_event_cb(g_pkgmgr, _package_manager_event_cb, NULL); + if (ret == PACKAGE_MANAGER_ERROR_NONE) { + LOGD("package_manager_set_event_cb succeeded."); + } + else { + LOGE("package_manager_set_event_cb failed(%d)", ret); + } + } + else { + LOGE("package_manager_create failed(%d)", ret); + } + } + return true; } void service_app_terminate(void *data) { + if (g_pkgmgr) { + package_manager_unset_event_cb(g_pkgmgr); + package_manager_destroy(g_pkgmgr); + g_pkgmgr = NULL; + } + // Todo: add your code here. deinit_wakeup(); return; @@ -1263,19 +1397,6 @@ void service_app_terminate(void *data) void service_app_control(app_control_h app_control, void *data) { // Todo: add your code here. - char *operation = NULL; - int ret = app_control_get_operation(app_control, &operation); - if (ret == APP_CONTROL_ERROR_NONE && operation) { - //strncmp(operation, APP_CONTROL_OPERATION_LAUNCH_ON_EVENT, strlen(operation)) == 0){ - char* value = NULL; - app_control_get_extra_data(app_control, "event", &value); - LOGD("operation : %s , %s", operation, value); - if (value && 0 == strcmp(value, "reload")) { - deinit_plugin(); - init_plugin(); - process_multi_assistant_activated_setting(); - } - } return; } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index b9f5f7a..e62a25f 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -511,7 +511,6 @@ int multi_assistant_service_plugin_deinitialize(void) MAS_LOGD("[Recorder ERROR] File not found!"); } #endif - multi_assistant_service_plugin_deactivate(); int ret = -1; if (NULL != g_handle) { -- 2.7.4