From d33f7d970102a49762b41d903ca14ca182f77f03 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 30 Sep 2019 17:15:36 +0900 Subject: [PATCH 01/16] Bump version to 0.1.2 Change-Id: I831ee98716d0eb27aa115696def6115bde9cf45b --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index dfe8e7d..a4ff719 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 0ec20cb..29fbf1e 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.1.1 +Version: 0.1.2 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 30b33bb42ebdd93bf4406494423ebbb8210381c8 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 25 Sep 2019 18:56:38 +0900 Subject: [PATCH 02/16] Implement wake word audio data embedding feature Change-Id: I224daddee40b6d1a80db2ffd0bdb926a9df3919b --- inc/multi_assistant_main.h | 2 + inc/multi_assistant_service_plugin.h | 6 +++ inc/multi_wakeup_recognizer.h | 4 +- plugins/wakeup-manager/inc/wakeup_engine_manager.h | 15 ++++++ plugins/wakeup-manager/inc/wakeup_manager.h | 3 ++ .../wakeup-manager/inc/wakeup_manager_wrapper.h | 7 +++ .../wakeup-manager/src/wakeup_engine_manager.cpp | 57 ++++++++++++++++++++ plugins/wakeup-manager/src/wakeup_manager.cpp | 23 +++++++++ .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 38 ++++++++++++++ src/multi_assistant_dbus.c | 55 ++++++++++++++++++++ src/multi_assistant_dbus_server.c | 31 +++++++++++ src/multi_assistant_service.c | 6 +++ src/multi_assistant_service_plugin.c | 60 ++++++++++++++++++++++ 13 files changed, 306 insertions(+), 1 deletion(-) diff --git a/inc/multi_assistant_main.h b/inc/multi_assistant_main.h index 4fcc7ef..42c4e2f 100644 --- a/inc/multi_assistant_main.h +++ b/inc/multi_assistant_main.h @@ -65,6 +65,7 @@ #define MA_METHOD_SET_BACKGROUND_VOLUME "ma_method_set_background_volume" #define MA_METHOD_SET_PREPROCESSING_ALLOW_MODE "ma_method_set_preprocessing_allow_mode" #define MA_METHOD_SEND_PREPROCESSING_RESULT "ma_method_send_preprocessing_result" +#define MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "ma_method_set_wake_word_audio_require_flag" #define MA_METHOD_ERROR "ma_method_error" #define MA_UI_METHOD_INITIALIZE "ma_ui_method_initialize" @@ -77,6 +78,7 @@ #define MAS_METHOD_WAKEUP_ENGINE_COMMAND "mas_method_wakeup_engine_command" #define MAS_METHOD_ERROR "mas_method_error" #define MAS_METHOD_SEND_PREPROCESSING_INFORMATION "mas_method_send_preprocessing_information" +#define MAS_METHOD_AUDIO_STREAMING_DATA_SECTION "mas_method_audio_streaming_data_section" #define MAS_UI_METHOD_SEND_ASR_RESULT "mas_ui_method_send_asr_result" #define MAS_UI_METHOD_SEND_RESULT "mas_ui_method_send_result" diff --git a/inc/multi_assistant_service_plugin.h b/inc/multi_assistant_service_plugin.h index 43db0f4..9bb703c 100644 --- a/inc/multi_assistant_service_plugin.h +++ b/inc/multi_assistant_service_plugin.h @@ -148,6 +148,8 @@ typedef int (*wakeup_manager_stop_streaming_follow_up_data)(void); typedef int (*wakeup_manager_get_audio_format)(int* rate, int* channel, int* audio_type); #define MA_WAKEUP_MANAGER_FUNC_GET_AUDIO_SOURCE_TYPE "wakeup_manager_get_audio_source_type" typedef int (*wakeup_manager_get_audio_source_type)(char** type); +#define MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "wakeup_manager_set_wake_word_audio_require_flag" +typedef int (*wakeup_manager_set_wake_word_audio_require_flag)(bool require); #define MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_manager_set_wakeup_event_callback" typedef int (*wakeup_manager_set_wakeup_event_callback)(wakeup_service_wakeup_event_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_UTTERANCE_STREAMING_CALLBACK "wakeup_manager_set_utterance_streaming_callback" @@ -160,6 +162,8 @@ typedef int (*wakeup_manager_set_follow_up_streaming_callback)(wakeup_service_sp typedef int (*wakeup_manager_set_speech_status_callback)(wakeup_service_speech_status_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_ERROR_CALLBACK "wakeup_manager_set_error_callback" typedef int (*wakeup_manager_set_error_callback)(wakeup_service_error_cb callback, void* user_data); +#define MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK "wakeup_manager_set_streaming_section_changed_callback" +typedef int (*wakeup_manager_set_streaming_section_changed_callback)(wakeup_service_streaming_section_changed_cb callback, void* user_data); typedef struct { wakeup_manager_initialize initialize; @@ -184,12 +188,14 @@ typedef struct { wakeup_manager_stop_streaming_follow_up_data stop_streaming_follow_up_data; wakeup_manager_get_audio_format get_audio_format; wakeup_manager_get_audio_source_type get_audio_source_type; + wakeup_manager_set_wake_word_audio_require_flag set_wake_word_audio_require_flag; wakeup_manager_set_wakeup_event_callback set_wakeup_event_callback; wakeup_manager_set_utterance_streaming_callback set_utterance_streaming_callback; wakeup_manager_set_previous_utterance_streaming_callback set_previous_utterance_streaming_callback; wakeup_manager_set_follow_up_streaming_callback set_follow_up_streaming_callback; wakeup_manager_set_speech_status_callback set_speech_status_callback; wakeup_manager_set_error_callback set_error_callback; + wakeup_manager_set_streaming_section_changed_callback set_streaming_section_changed_callback; } wakeup_manager_interface; #ifdef __cplusplus diff --git a/inc/multi_wakeup_recognizer.h b/inc/multi_wakeup_recognizer.h index 0b4287f..c818e1a 100644 --- a/inc/multi_wakeup_recognizer.h +++ b/inc/multi_wakeup_recognizer.h @@ -19,7 +19,7 @@ #define _MULTI_WAKEUP_RECOGNIZER_H_ #include - +#include #ifdef __cplusplus extern "C" { @@ -70,6 +70,8 @@ typedef void (*wakeup_service_speech_status_cb)(wakeup_speech_status_e status, v typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* user_data); +typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); + #ifdef __cplusplus } #endif diff --git a/plugins/wakeup-manager/inc/wakeup_engine_manager.h b/plugins/wakeup-manager/inc/wakeup_engine_manager.h index 3b262df..4e24631 100644 --- a/plugins/wakeup-manager/inc/wakeup_engine_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_engine_manager.h @@ -19,6 +19,8 @@ #include "wakeup_manager_wrapper.h" +#include + #include #include #include @@ -68,8 +70,14 @@ typedef int (*wakeup_engine_feed_audio_data)(long time, void* data, int len); typedef int (*wakeup_engine_get_utterance_data_count)(void); #define MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA "wakeup_engine_get_utterance_data" typedef int (*wakeup_engine_get_utterance_data)(int index, wakeup_speech_data *data); +#define MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA_COUNT "wakeup_engine_get_wake_word_data_count" +typedef int (*wakeup_engine_get_wake_word_data_count)(void); +#define MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA "wakeup_engine_get_wake_word_data" +typedef int (*wakeup_engine_get_wake_word_data)(int index, wakeup_speech_data *data); #define MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND "wakeup_engine_set_assistant_specific_command" typedef int (*wakeup_engine_set_assistant_specific_command)(const char* appid, const char* command); +#define MA_WAKEUP_ENGINE_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "wakeup_engine_set_wake_word_audio_require_flag" +typedef int (*wakeup_engine_set_wake_word_audio_require_flag)(bool require); #define MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_engine_set_wakeup_event_callback" typedef int (*wakeup_engine_set_wakeup_event_callback)(wakeup_service_wakeup_event_cb callback, void* user_data); #define MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK "wakeup_engine_set_speech_status_callback" @@ -98,8 +106,11 @@ typedef struct { wakeup_engine_feed_audio_data feed_audio_data; wakeup_engine_get_utterance_data_count get_utterance_data_count; wakeup_engine_get_utterance_data get_utterance_data; + wakeup_engine_get_wake_word_data_count get_wake_word_data_count; + wakeup_engine_get_wake_word_data get_wake_word_data; wakeup_engine_get_version get_version; wakeup_engine_set_assistant_specific_command set_assistant_specific_command; + wakeup_engine_set_wake_word_audio_require_flag set_wake_word_audio_require_flag; wakeup_engine_set_wakeup_event_callback set_wakeup_event_callback; wakeup_engine_set_speech_status_callback set_speech_status_callback; wakeup_engine_set_error_callback set_error_callback; @@ -117,6 +128,7 @@ public: virtual bool on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0; + virtual bool on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0; }; class CWakeupEngineManager @@ -140,6 +152,7 @@ public: bool set_language(string language); void set_assistant_activated(string appid, bool activated); + void set_wake_word_audio_require_flag(bool require); void start_streaming_current_utterance_data(); void stop_streaming_current_utterance_data(); @@ -182,6 +195,8 @@ private: thread mStreamingThread; atomic_bool mStopStreamingThread{false}; + + bool mWakeWordAudioRequired{false}; }; } // wakeup diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index 7d5e57b..645bcfc 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -49,6 +49,7 @@ public: virtual void on_wakeup(wakeup_event_info wakeup_info) = 0; virtual void on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0; + virtual void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0; }; /* If a wakeup event is raised by pressing a voice key, @@ -86,6 +87,7 @@ public: bool get_audio_source_type(char** type); bool set_language(string language); bool get_voice_key_pressed(); + bool set_wake_word_audio_require_flag(bool require); STREAMING_MODE get_streaming_mode(); bool set_streaming_mode(STREAMING_MODE mode); @@ -120,6 +122,7 @@ private: bool on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override; + bool on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override; void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; } private: diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index de080d0..a5acc99 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -22,6 +22,7 @@ #include #include #include +#include #include "wakeup_interfaces.h" @@ -53,6 +54,8 @@ typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* us typedef void (*wakeup_service_audio_data_require_status_cb)(bool require, void* user_data); +typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); + typedef enum { MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0, MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, @@ -114,6 +117,8 @@ EXPORT_API int wakeup_manager_get_audio_format(int *rate, int *channel, int *aud EXPORT_API int wakeup_manager_get_audio_source_type(char** type); +EXPORT_API int wakeup_manager_set_wake_word_audio_require_flag(bool require); + EXPORT_API int wakeup_manager_set_wakeup_event_callback(wakeup_service_wakeup_event_cb callback, void* user_data); EXPORT_API int wakeup_manager_set_utterance_streaming_callback(wakeup_service_speech_streaming_cb callback, void* user_data); @@ -126,6 +131,8 @@ EXPORT_API int wakeup_manager_set_speech_status_callback(wakeup_service_speech_s EXPORT_API int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* user_data); +EXPORT_API int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streaming_section_changed_cb callback, void* user_data); + /* Internal API declarations for dependency modules */ int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len); diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index db4a4ad..1b159aa 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -175,6 +175,17 @@ void CWakeupEngineManager::set_assistant_activated(string appid, bool activated) } } +void CWakeupEngineManager::set_wake_word_audio_require_flag(bool require) +{ + MWR_LOGD("[ENTER]"); + mWakeWordAudioRequired = require; + for (const auto& info : mEngineInfo) { + if (info.interface.set_wake_word_audio_require_flag) { + info.interface.set_wake_word_audio_require_flag(require); + } + } +} + void CWakeupEngineManager::streaming_speech_data_thread_func() { MWR_LOGD("[ENTER]"); @@ -195,6 +206,43 @@ void CWakeupEngineManager::streaming_speech_data_thread_func() int index = 0; bool finish_event_sent = false; + if (mWakeWordAudioRequired && + NULL != interface->get_wake_word_data && + NULL != interface->get_wake_word_data_count) { + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_audio_streaming_data_section(MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + int count = interface->get_wake_word_data_count(); + while (!(mStopStreamingThread.load()) && index < count) { + int ret = interface->get_wake_word_data(index, &speech_data); + if (0 == ret) { + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_streaming_audio_data( + speech_data.event, speech_data.buffer, speech_data.len)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + } else { + break; + } + index++; + } + index = 0; + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_audio_streaming_data_section(MA_AUDIO_STREAMING_DATA_SECTION_UTTERANCE)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + } + while (!(mStopStreamingThread.load())) { int ret = -1; int cnt = 0; @@ -512,9 +560,18 @@ void CWakeupEngineManager::add_engine(string name, string path) info.interface.get_utterance_data = (wakeup_engine_get_utterance_data)dlsym(info.engine_handle, MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA); + info.interface.get_wake_word_data_count = + (wakeup_engine_get_wake_word_data_count)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA_COUNT); + info.interface.get_wake_word_data = + (wakeup_engine_get_wake_word_data)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA); info.interface.set_assistant_specific_command = (wakeup_engine_set_assistant_specific_command)dlsym(info.engine_handle, MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND); + info.interface.set_wake_word_audio_require_flag = + (wakeup_engine_set_wake_word_audio_require_flag)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG); info.interface.set_wakeup_event_callback = (wakeup_engine_set_wakeup_event_callback)dlsym(info.engine_handle, MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK); diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index fec05e4..14373df 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -679,6 +679,16 @@ bool CWakeupManager::get_audio_source_type(char** type) return true; } +bool CWakeupManager::set_wake_word_audio_require_flag(bool require) +{ + MWR_LOGD("[ENTER]"); + + mWakeupEngineManager.set_wake_word_audio_require_flag(require); + + MWR_LOGD("[END]"); + return true; +} + CWakeupPolicy* CWakeupManager::get_wakeup_policy() { return mWakeupPolicy.get(); @@ -772,6 +782,19 @@ bool CWakeupManager::CEngineEventObserver::on_streaming_audio_data( return true; } +bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section( + ma_audio_streaming_data_section_e section) +{ + if (nullptr == mWakeupManager) return false; + + vector observers = mWakeupManager->get_observers(); + for (const auto& observer : observers) { + observer->on_audio_streaming_data_section(section); + } + + return true; +} + void CWakeupManager::CPolicyEventObserver::on_wakeup(wakeup_event_info wakeup_info) { if (nullptr == mWakeupManager) return; diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index dc33842..d639cf1 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -37,11 +37,15 @@ static void* g_speech_status_user_data; static wakeup_service_error_cb g_error_cb; static void* g_error_user_data; +static wakeup_service_streaming_section_changed_cb g_streaming_section_changed_cb; +static void* g_streaming_section_changed_user_data; + class CWakeupEventObserver : public IWakeupEventObserver { void on_wakeup(wakeup_event_info wakeup_info) override; void on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override; + void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override; }; static CWakeupEventObserver g_wakeup_event_observer; @@ -358,6 +362,16 @@ int wakeup_manager_get_audio_source_type(char** type) return 0; } +int wakeup_manager_set_wake_word_audio_require_flag(bool require) +{ + MWR_LOGD("[ENTER] : %d", require); + + g_wakeup_manager.set_wake_word_audio_require_flag(require); + + MWR_LOGD("[END]"); + return 0; +} + int wakeup_manager_set_wakeup_event_callback(wakeup_service_wakeup_event_cb callback, void* user_data) { MWR_LOGD("[ENTER]"); @@ -454,6 +468,22 @@ int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* us return 0; } +int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streaming_section_changed_cb callback, void* user_data) +{ + MWR_LOGD("[ENTER]"); + + if (NULL == callback) { + MWR_LOGE("[ERROR] Input parameter is NULL"); + return -1; + } + + g_streaming_section_changed_cb = callback; + g_streaming_section_changed_user_data = user_data; + + MWR_LOGD("[END]"); + return 0; +} + int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len) { g_wakeup_manager.feed_audio_data(event, buffer, len); @@ -482,3 +512,11 @@ void CWakeupEventObserver::on_streaming_audio_data( MWR_LOGI("[INFO] No service streaming callback"); } } + +void CWakeupEventObserver::on_audio_streaming_data_section( + ma_audio_streaming_data_section_e section) +{ + if (g_streaming_section_changed_cb) { + g_streaming_section_changed_cb(section, g_streaming_section_changed_user_data); + } +} diff --git a/src/multi_assistant_dbus.c b/src/multi_assistant_dbus.c index cd9fb36..bbf0cbf 100644 --- a/src/multi_assistant_dbus.c +++ b/src/multi_assistant_dbus.c @@ -341,6 +341,58 @@ int masc_dbus_send_preprocessing_information(int pid, const char* app_id) return 0; } +int masc_dbus_send_streaming_section_changed(int pid, int section) +{ + if (0 != __dbus_check()) { + return -1; //MAS_ERROR_OPERATION_FAILED; + } + + DBusMessage* msg; + + DBusError err; + dbus_error_init(&err); + + char service_name[64]; + memset(service_name, '\0', 64); + snprintf(service_name, 64, "%s_%d", MA_CLIENT_SERVICE_NAME, pid); + + msg = dbus_message_new_method_call( + service_name, + MA_CLIENT_SERVICE_OBJECT_PATH, + MA_CLIENT_SERVICE_INTERFACE, + MAS_METHOD_AUDIO_STREAMING_DATA_SECTION); + + static int count = 0; + if (NULL == msg) { + MAS_LOGE(">>>> Request mas send streaming section changed information : Fail to make message"); + return -1; // MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD(">>>> Request mas send streaming section changed information : %s", service_name); + } + + if (true != dbus_message_append_args(msg, + DBUS_TYPE_INT32, §ion, + DBUS_TYPE_INVALID)) { + dbus_message_unref(msg); + MAS_LOGE("[ERROR] Fail to append args"); + return -1; + } + + dbus_message_set_no_reply(msg, TRUE); + + if (1 != dbus_connection_send(g_conn_sender, msg, NULL)) { + MAS_LOGE("[Dbus ERROR] Fail to Send"); + return -1; // MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD("[Dbus DEBUG] Success to Send streaming section changed information : %d", section); + dbus_connection_flush(g_conn_sender); + } + + dbus_message_unref(msg); + + return 0; +} + int masc_ui_dbus_send_hello(void) { if (0 != __dbus_check()) { @@ -766,6 +818,9 @@ static Eina_Bool listener_event_callback(void* data, Ecore_Fd_Handler *fd_handle } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_SEND_PREPROCESSING_RESULT)) { ma_service_dbus_send_preprocessing_result(g_conn_listener, msg); + } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG)) { + ma_service_dbus_set_wake_word_audio_require_flag(g_conn_listener, msg); + } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_UI_METHOD_INITIALIZE)) { ma_service_ui_dbus_initialize(g_conn_listener, msg); diff --git a/src/multi_assistant_dbus_server.c b/src/multi_assistant_dbus_server.c index c2bff18..16ec6c8 100644 --- a/src/multi_assistant_dbus_server.c +++ b/src/multi_assistant_dbus_server.c @@ -632,6 +632,37 @@ int ma_service_dbus_send_preprocessing_result(DBusConnection* conn, DBusMessage* return 0; } +int ma_service_dbus_set_wake_word_audio_require_flag(DBusConnection* conn, DBusMessage* msg) +{ + DBusError err; + dbus_error_init(&err); + + int pid; + int ret = 0; + int require; + + dbus_message_get_args(msg, &err, + DBUS_TYPE_INT32, &pid, + DBUS_TYPE_INT32, &require, + DBUS_TYPE_INVALID); + + MAS_LOGD("[DEBUG] MAS SET WAKE WORD AUDIO REQUIRE FLAG"); + + if (dbus_error_is_set(&err)) { + MAS_LOGE("[IN ERROR] mas set wake word audio require flag : Get arguments error (%s)", err.message); + dbus_error_free(&err); + ret = -1; //MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD("[IN] mas set wake word audio require flag : pid(%d), require(%d)", pid, require); + ret = mas_client_set_wake_word_audio_require_flag(pid, (bool)require); + } + + MAS_LOGD("<<<<<"); + MAS_LOGD(" "); + + return 0; +} + int ma_service_ui_dbus_initialize(DBusConnection* conn, DBusMessage* msg) { DBusError err; diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 773cb7d..f5ffe23 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -488,6 +488,12 @@ int mas_client_send_preprocessing_result(int pid, bool result) return 0; } +int mas_client_set_wake_word_audio_require_flag(int pid, bool require) +{ + multi_assistant_service_plugin_set_wake_word_audio_require_flag(NULL, require); + return 0; +} + int mas_client_update_recognition_result(int pid, int state) { multi_assistant_service_plugin_update_recognition_result(NULL, state); diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index ac5b4ab..79f7aa5 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -331,6 +331,17 @@ static void __error_cb(int error, const char* err_msg, void* user_data) } } +static void __streaming_section_changed_cb(ma_audio_streaming_data_section_e section, void* user_data) +{ + MAS_LOGD( "[SUCCESS] __streaming_section_changed_cb is called, section(%d)", section); + + int pid = mas_get_current_client_pid(); + int ret = masc_dbus_send_streaming_section_changed(pid, (int)section); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to send streaming section changed information, ret(%d)", ret); + } +} + int multi_assistant_service_plugin_initialize(void) { MAS_LOGD( "[Enter]"); @@ -413,6 +424,9 @@ int multi_assistant_service_plugin_initialize(void) _wakeup_manager_interface.get_audio_source_type = (wakeup_manager_get_audio_source_type)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_GET_AUDIO_SOURCE_TYPE); + _wakeup_manager_interface.set_wake_word_audio_require_flag = + (wakeup_manager_set_wake_word_audio_require_flag)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG); _wakeup_manager_interface.set_wakeup_event_callback = (wakeup_manager_set_wakeup_event_callback)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_EVENT_CALLBACK); @@ -431,6 +445,9 @@ int multi_assistant_service_plugin_initialize(void) _wakeup_manager_interface.set_error_callback = (wakeup_manager_set_error_callback)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_ERROR_CALLBACK); + _wakeup_manager_interface.set_streaming_section_changed_callback = + (wakeup_manager_set_streaming_section_changed_callback)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK); int ret = -1; if (NULL != g_handle) { @@ -874,6 +891,25 @@ int multi_assistant_service_plugin_get_recording_audio_source_type(char** type) return ret; } +int multi_assistant_service_plugin_set_wake_word_audio_require_flag(const char* appid, bool require) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_set_wake_word_audio_require_flag func = _wakeup_manager_interface.set_wake_word_audio_require_flag; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG); + } else { + ret = func(require); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set wake word audio require flag, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + return ret; +} + int multi_assistant_service_plugin_set_callbacks(void) { int ret = multi_assistant_service_plugin_set_wakeup_event_callback(__wakeup_event_cb, NULL); @@ -911,6 +947,13 @@ int multi_assistant_service_plugin_set_callbacks(void) MAS_LOGE("Fail to set error cb"); return ret; } + + ret = multi_assistant_service_plugin_set_streaming_section_changed_callback(__streaming_section_changed_cb, NULL); + if (0 != ret) { + MAS_LOGE("Fail to set streaming section changed cb"); + return ret; + } + return 0; } @@ -1023,3 +1066,20 @@ int multi_assistant_service_plugin_set_error_callback(wakeup_service_error_cb ca } return ret; } + +int multi_assistant_service_plugin_set_streaming_section_changed_callback(wakeup_service_streaming_section_changed_cb callback, void* user_data) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_set_streaming_section_changed_callback func = _wakeup_manager_interface.set_streaming_section_changed_callback; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK); + } else { + ret = func(callback, user_data); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set streaming section changed callback, ret(%d)", ret); + } + } + } + return ret; +} -- 2.7.4 From 5b74c715ddfbf48f422280a8bc911de39eec36cc Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 2 Oct 2019 16:54:55 +0900 Subject: [PATCH 03/16] Bump version to 0.2.0 Change-Id: Ia567ca3a49734d5e6a54eac64df0df316dfbe360 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index a4ff719..4220817 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 29fbf1e..494261e 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.1.2 +Version: 0.2.0 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 0254595df66b7b2073cf17c6379a521b5edd0367 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 18 Oct 2019 13:25:49 +0900 Subject: [PATCH 04/16] Fix defects detected by static analysis tool Change-Id: I93f7686679318de3f42ec3d416b9f76db11602d5 --- src/multi_assistant_service.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index f5ffe23..4d69c3d 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -1054,8 +1054,15 @@ int mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT event) if (!check_preprocessing_assistant_exists()) { mas_bring_client_to_foreground(current_maclient_appid); } + g_current_preprocessing_state = PREPROCESSING_STATE_WAKEUP_PREPROCESS_DISABLED; + if (MA_PREPROCESSING_ALLOW_UTTERANCE == mode || + MA_PREPROCESSING_ALLOW_ALL == mode) { + if (is_current_preprocessing_assistant(preprocessing_allow_appid)) { + g_current_preprocessing_state = PREPROCESSING_STATE_WAKEUP_PREPROCESS_ENABLED; + } + } } - /* Intentionally omitted break statement here */ + break; case PREPROCESSING_STATE_EVENT_PREPROCESSING_ALLOW_MODE_CHANGED: { g_current_preprocessing_state = PREPROCESSING_STATE_WAKEUP_PREPROCESS_DISABLED; -- 2.7.4 From b05b257b1129411ec58718d7955c70d170a532b2 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 15 Oct 2019 16:52:22 +0900 Subject: [PATCH 05/16] Remove unused function Change-Id: I82e60ea5837b4a9fecd1006d904d4de8dfd0a6e0 --- inc/multi_assistant_service.h | 2 -- src/multi_assistant_service.c | 5 ----- 2 files changed, 7 deletions(-) diff --git a/inc/multi_assistant_service.h b/inc/multi_assistant_service.h index 2c496e0..9ca7460 100644 --- a/inc/multi_assistant_service.h +++ b/inc/multi_assistant_service.h @@ -57,8 +57,6 @@ int mas_client_set_preprocessing_allow_mode(int pid, int mode, const char* appid int mas_client_send_preprocessing_result(int pid, bool result); -int mas_client_update_recognition_result(int pid, int result); - int mas_ui_client_initialize(int pid); int mas_ui_client_deinitialize(int pid); diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 4d69c3d..f0754a0 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -491,12 +491,7 @@ int mas_client_send_preprocessing_result(int pid, bool result) int mas_client_set_wake_word_audio_require_flag(int pid, bool require) { multi_assistant_service_plugin_set_wake_word_audio_require_flag(NULL, require); - return 0; -} -int mas_client_update_recognition_result(int pid, int state) -{ - multi_assistant_service_plugin_update_recognition_result(NULL, state); return 0; } -- 2.7.4 From b21e35c2627ac40377a3edc6d54bed4a90c0583f Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 15 Oct 2019 16:55:08 +0900 Subject: [PATCH 06/16] Add missing implementation for resolving appid from pid Change-Id: Idfb73b15e7a116f93be61e5f8c6f0e1afa5f6095 --- src/multi_assistant_service.c | 48 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index f0754a0..073967b 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -369,7 +369,13 @@ int mas_client_send_result(int pid, char* display_text, char* utterance_text, ch MAS_LOGE("[ERROR] Fail to send result, ret(%d)", ret); } - multi_assistant_service_plugin_update_recognition_result(NULL, MA_RECOGNITION_RESULT_EVENT_SUCCESS); + const char* pid_appid = NULL; + char buf[MAX_APPID_LEN] = {'\0',}; + if (AUL_R_OK == aul_app_get_appid_bypid(pid, buf, sizeof(buf))) { + buf[MAX_APPID_LEN - 1] = '\0'; + pid_appid = buf; + } + multi_assistant_service_plugin_update_recognition_result(pid_appid, MA_RECOGNITION_RESULT_EVENT_SUCCESS); return ret; } @@ -382,7 +388,13 @@ int mas_client_send_recognition_result(int pid, int result) MAS_LOGE("[ERROR] Fail to send recognition result, ret(%d)", ret); } - multi_assistant_service_plugin_update_recognition_result(NULL, result); + const char* pid_appid = NULL; + char buf[MAX_APPID_LEN] = {'\0',}; + if (AUL_R_OK == aul_app_get_appid_bypid(pid, buf, sizeof(buf))) { + buf[MAX_APPID_LEN - 1] = '\0'; + pid_appid = buf; + } + multi_assistant_service_plugin_update_recognition_result(pid_appid, result); return ret; } @@ -426,19 +438,37 @@ int mas_client_stop_streaming_audio_data(int pid, int type) int mas_client_update_voice_feedback_state(int pid, int state) { - multi_assistant_service_plugin_update_voice_feedback_state(NULL, state); + const char* pid_appid = NULL; + char buf[MAX_APPID_LEN] = {'\0',}; + if (AUL_R_OK == aul_app_get_appid_bypid(pid, buf, sizeof(buf))) { + buf[MAX_APPID_LEN - 1] = '\0'; + pid_appid = buf; + } + multi_assistant_service_plugin_update_voice_feedback_state(pid_appid, state); return 0; } int mas_client_send_assistant_specific_command(int pid, const char *command) { - multi_assistant_service_plugin_send_assistant_specific_command(NULL, command); + const char* pid_appid = NULL; + char buf[MAX_APPID_LEN] = {'\0',}; + if (AUL_R_OK == aul_app_get_appid_bypid(pid, buf, sizeof(buf))) { + buf[MAX_APPID_LEN - 1] = '\0'; + pid_appid = buf; + } + multi_assistant_service_plugin_send_assistant_specific_command(pid_appid, command); return 0; } int mas_client_set_background_volume(int pid, double ratio) { - multi_assistant_service_plugin_set_background_volume(NULL, ratio); + const char* pid_appid = NULL; + char buf[MAX_APPID_LEN] = {'\0',}; + if (AUL_R_OK == aul_app_get_appid_bypid(pid, buf, sizeof(buf))) { + buf[MAX_APPID_LEN - 1] = '\0'; + pid_appid = buf; + } + multi_assistant_service_plugin_set_background_volume(pid_appid, ratio); return 0; } @@ -490,8 +520,14 @@ int mas_client_send_preprocessing_result(int pid, bool result) int mas_client_set_wake_word_audio_require_flag(int pid, bool require) { - multi_assistant_service_plugin_set_wake_word_audio_require_flag(NULL, require); + const char* pid_appid = NULL; + char buf[MAX_APPID_LEN] = {'\0',}; + if (AUL_R_OK == aul_app_get_appid_bypid(pid, buf, sizeof(buf))) { + buf[MAX_APPID_LEN - 1] = '\0'; + pid_appid = buf; + } + multi_assistant_service_plugin_set_wake_word_audio_require_flag(pid_appid, require); return 0; } -- 2.7.4 From eb61e7647b8354f5b59ec12ba30ab0ad8d4c30aa Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 2 Oct 2019 20:46:25 +0900 Subject: [PATCH 07/16] Add support for delivering preprocessing info and result Change-Id: Iab6dc1d92a23379d5697092fb295edba4f21376a --- inc/multi_assistant_main.h | 1 + inc/multi_assistant_service.h | 2 + src/multi_assistant_dbus.c | 54 +++++++++++++++++++++++++ src/multi_assistant_service.c | 78 ++++++++++++++++++++++-------------- src/multi_assistant_service_plugin.c | 7 ++++ 5 files changed, 111 insertions(+), 31 deletions(-) diff --git a/inc/multi_assistant_main.h b/inc/multi_assistant_main.h index 42c4e2f..6c04aa6 100644 --- a/inc/multi_assistant_main.h +++ b/inc/multi_assistant_main.h @@ -79,6 +79,7 @@ #define MAS_METHOD_ERROR "mas_method_error" #define MAS_METHOD_SEND_PREPROCESSING_INFORMATION "mas_method_send_preprocessing_information" #define MAS_METHOD_AUDIO_STREAMING_DATA_SECTION "mas_method_audio_streaming_data_section" +#define MAS_METHOD_SEND_PREPROCESSING_RESULT "mas_method_send_preprocessing_result" #define MAS_UI_METHOD_SEND_ASR_RESULT "mas_ui_method_send_asr_result" #define MAS_UI_METHOD_SEND_RESULT "mas_ui_method_send_result" diff --git a/inc/multi_assistant_service.h b/inc/multi_assistant_service.h index 9ca7460..0a79b99 100644 --- a/inc/multi_assistant_service.h +++ b/inc/multi_assistant_service.h @@ -65,6 +65,8 @@ int mas_ui_client_change_assistant(const char* appid); int mas_get_current_client_pid(); +int mas_get_current_preprocessing_client_pid(); + int mas_get_client_pid_by_wakeup_word(const char *wakeup_word); int mas_get_client_pid_by_appid(const char *appid); diff --git a/src/multi_assistant_dbus.c b/src/multi_assistant_dbus.c index bbf0cbf..ac92dba 100644 --- a/src/multi_assistant_dbus.c +++ b/src/multi_assistant_dbus.c @@ -393,6 +393,60 @@ int masc_dbus_send_streaming_section_changed(int pid, int section) return 0; } +int masc_dbus_send_preprocessing_result(int pid, bool result) +{ + if (0 != __dbus_check()) { + return -1; //MAS_ERROR_OPERATION_FAILED; + } + + DBusMessage* msg; + + DBusError err; + dbus_error_init(&err); + + char service_name[64]; + memset(service_name, '\0', 64); + snprintf(service_name, 64, "%s_%d", MA_CLIENT_SERVICE_NAME, pid); + + msg = dbus_message_new_method_call( + service_name, + MA_CLIENT_SERVICE_OBJECT_PATH, + MA_CLIENT_SERVICE_INTERFACE, + MAS_METHOD_SEND_PREPROCESSING_RESULT); + + static int count = 0; + if (NULL == msg) { + MAS_LOGE(">>>> Request mas send preprocessing result : Fail to make message"); + return -1; // MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD(">>>> Request mas send preprocessing result : %s", service_name); + } + + int temp_result = result; + + if (true != dbus_message_append_args(msg, + DBUS_TYPE_INT32, &temp_result, + DBUS_TYPE_INVALID)) { + dbus_message_unref(msg); + MAS_LOGE("[ERROR] Fail to append args"); + return -1; + } + + dbus_message_set_no_reply(msg, TRUE); + + if (1 != dbus_connection_send(g_conn_sender, msg, NULL)) { + MAS_LOGE("[Dbus ERROR] Fail to Send"); + return -1; // MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD("[Dbus DEBUG] Success to Send preprocessing result : %d", temp_result); + dbus_connection_flush(g_conn_sender); + } + + dbus_message_unref(msg); + + return 0; +} + int masc_ui_dbus_send_hello(void) { if (0 != __dbus_check()) { diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 073967b..2c5d692 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -68,6 +68,7 @@ typedef struct { } ma_client_s; static int g_current_maclient_info = 0; +static int g_current_preprocessing_maclient_info = -1; static const char *g_wakeup_maclient_appid = NULL; static PREPROCESSING_STATE g_current_preprocessing_state = PREPROCESSING_STATE_NONE; @@ -170,7 +171,10 @@ bool check_preprocessing_assistant_exists() for (int loop = 0; loop < MAX_MACLIENT_INFO_NUM; loop++) { if (g_maclient_info[loop].used) { if (strncmp(vconf_str, g_maclient_info[loop].appid, MAX_APPID_LEN) == 0) { - ret = true; + ma_client_s* client = ma_client_find_by_appid(vconf_str); + if (client && client->pid > 0) { + ret = true; + } } } } @@ -178,6 +182,8 @@ bool check_preprocessing_assistant_exists() vconf_str = NULL; } + MAS_LOGD("result : %d", ret); + return ret; } @@ -235,12 +241,12 @@ int mas_client_initialize(int pid) current_maclient_appid = g_maclient_info[g_current_maclient_info].appid; } + mas_client_send_preprocessing_information(pid); if (current_maclient_appid && 0 == strncmp(current_maclient_appid, appid, MAX_APPID_LEN)) { MAS_LOGD("MA client with current maclient appid connected!"); if (g_wakeup_maclient_appid && strncmp(g_wakeup_maclient_appid, appid, MAX_APPID_LEN) == 0) { g_wakeup_maclient_appid = NULL; - mas_client_send_preprocessing_information(pid); mas_client_activate(pid); mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_ACTIVE_ASSISTANT_LAUNCHED); } else { @@ -292,38 +298,16 @@ int mas_client_get_audio_source_type(int pid, char** type) return ret; } -bool check_pid_is_current_maclient(int pid) -{ - bool ret = false; - char appid[MAX_APPID_LEN] = {'\0',}; - if (AUL_R_OK == aul_app_get_appid_bypid(pid, appid, sizeof(appid))) { - appid[MAX_APPID_LEN - 1] = '\0'; - const char *current_maclient_appid = NULL; - if (g_current_maclient_info >= 0 && g_current_maclient_info < MAX_MACLIENT_INFO_NUM) { - current_maclient_appid = g_maclient_info[g_current_maclient_info].appid; - } - - if (current_maclient_appid && 0 == strncmp(current_maclient_appid, appid, MAX_APPID_LEN)) { - ret = true; - } else { - MAS_LOGE("appid %s does not match with current MA Client", appid); - } - } - return ret; -} - int mas_client_send_preprocessing_information(int pid) { int ret = -1; MAS_LOGD("[Enter] pid(%d)", pid); - if (check_pid_is_current_maclient(pid)) { - char* vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_PREPROCESSING_ASSISTANT_APPID); - MAS_LOGD("preprocessing_assistant_appid : %s", vconf_str); - ret = masc_dbus_send_preprocessing_information(pid, vconf_str); - free(vconf_str); - vconf_str = NULL; - } + char* vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_PREPROCESSING_ASSISTANT_APPID); + MAS_LOGD("preprocessing_assistant_appid : %s", vconf_str); + ret = masc_dbus_send_preprocessing_information(pid, vconf_str); + free(vconf_str); + vconf_str = NULL; return ret; } @@ -510,11 +494,26 @@ int mas_client_send_preprocessing_result(int pid, bool result) } if (!is_current_preprocessing_assistant(pid_appid)) return -1; + const char *current_maclient_appid = NULL; + if (g_current_maclient_info >= 0 && g_current_maclient_info < MAX_MACLIENT_INFO_NUM) { + current_maclient_appid = g_maclient_info[g_current_maclient_info].appid; + } + if (result) { + MAS_LOGD("Preprocessing succeeded, bring (%s) to foreground", pid_appid); + mas_bring_client_to_foreground(pid_appid); mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_PREPROCESSING_SUCCEEDED); } else { + MAS_LOGD("Preprocessing failed, bring (%s) to foreground", current_maclient_appid); + mas_bring_client_to_foreground(current_maclient_appid); mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_PREPROCESSING_FAILED); } + + ma_client_s* client = ma_client_find_by_appid(current_maclient_appid); + if (client) { + masc_dbus_send_preprocessing_result(client->pid, result); + } + return 0; } @@ -616,6 +615,10 @@ int __mas_assistant_info_cb(const char* appid, const char* name, const char* ico strncpy(g_maclient_info[index].appid, appid, MAX_APPID_LEN); g_maclient_info[index].appid[MAX_APPID_LEN - 1] = '\0'; + if (is_current_preprocessing_assistant(g_maclient_info[index].appid)) { + g_current_preprocessing_maclient_info = index; + } + for (loop = 0;loop < MAX_WAKEUP_WORDS_NUM;loop++) { if (loop < cnt_wakeup && wakeup_list[loop]) { MAS_LOGD("wakeup_list(%d)(%s)(%s)", loop, wakeup_list[loop], wakeup_language[loop]); @@ -790,8 +793,21 @@ int mas_get_current_client_pid() { int ret = -1; if (g_current_maclient_info >= 0 && g_current_maclient_info < MAX_MACLIENT_INFO_NUM) { - const char *current_maclient_appid = g_maclient_info[g_current_maclient_info].appid; - ma_client_s* client = ma_client_find_by_appid(current_maclient_appid); + const char *appid = g_maclient_info[g_current_maclient_info].appid; + ma_client_s* client = ma_client_find_by_appid(appid); + if (client) { + ret = client->pid; + } + } + return ret; +} + +int mas_get_current_preprocessing_client_pid() +{ + int ret = -1; + if (g_current_preprocessing_maclient_info >= 0 && g_current_preprocessing_maclient_info < MAX_MACLIENT_INFO_NUM) { + const char *appid = g_maclient_info[g_current_preprocessing_maclient_info].appid; + ma_client_s* client = ma_client_find_by_appid(appid); if (client) { ret = client->pid; } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 79f7aa5..700b3f2 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -286,6 +286,7 @@ static void __audio_streaming_cb(wakeup_speech_streaming_event_e event, unsigned ++count; int pid = mas_get_current_client_pid(); + int preprocessing_pid = mas_get_current_preprocessing_client_pid(); if (pid == -1) { MAS_LOGE("[ERROR] Fail to retrieve pid of current MA client"); } else { @@ -294,6 +295,12 @@ static void __audio_streaming_cb(wakeup_speech_streaming_event_e event, unsigned if (0 != ret) { MAS_LOGE("[ERROR] Fail to send speech data, ret(%d)", ret); } + if (pid != preprocessing_pid && -1 != preprocessing_pid) { + int ret = masc_dbus_send_streaming_audio_data(preprocessing_pid, event, buffer, len); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to send speech data to preprocessing client, ret(%d)", ret); + } + } } } -- 2.7.4 From c5609dcb88e7623a15dd44897681cac75f1e7a26 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 15 Oct 2019 21:08:42 +0900 Subject: [PATCH 08/16] Add prelaunch functionality for preprocessing assistant Change-Id: Idf167b0293ea1783c22b9ab151fbce1d52769a63 --- src/multi_assistant_service.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 2c5d692..a6a8c0a 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -764,6 +764,16 @@ static int init_wakeup(void) } } + /* For the case of preprocessing assistant, it always have to be launched beforehand */ + char *vconf_str; + vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_PREPROCESSING_ASSISTANT_APPID); + if (vconf_str) { + MAS_LOGD("prelaunching preprocessing_assistant_appid : %s", vconf_str); + mas_launch_client_by_appid(vconf_str, CLIENT_LAUNCH_MODE_PRELAUNCH); + free(vconf_str); + vconf_str = NULL; + } + return 0; } -- 2.7.4 From ce5f8f324192d9cec13056b71a55c20bf3691a34 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 16 Oct 2019 16:38:12 +0900 Subject: [PATCH 09/16] Add sample code for terminating host process Change-Id: I4106710af29912e58bd0c4d0123cc1348f8ecc69 --- .../dependency-default/src/dependency_default.cpp | 21 +++++++++++++++++++ .../src/dependency_default_audio.cpp | 24 ++++++++++++++-------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp index be13250..edff076 100644 --- a/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp @@ -2,9 +2,16 @@ #include "dependency_default_audio.h" #include "dependency_default_button.h" +#include +#include + +#include + static mas_proxy_interface g_proxy_interface; const int g_dependency_version = 1; +static bool g_should_exit = false; + int mas_dependency_initialize(mas_proxy_interface interface, int *dependency_version) { g_proxy_interface = interface; @@ -16,6 +23,20 @@ int mas_dependency_initialize(mas_proxy_interface interface, int *dependency_ver *dependency_version = g_dependency_version; } + /* If the service process has to terminate in certain circumstances, + set g_should_exit variable to true as below. */ + // g_should_exit = true; + + if (g_should_exit) { + LOGE("g_should_exit value contains positive value, registering timer"); + std::thread([]() { + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + LOGE("Now trying to exit"); + mas_dependency_deinitialize(); + service_app_exit_without_restart(); + }).detach(); + } + return 0; } diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp index 1bc67c5..e803739 100644 --- a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp @@ -190,6 +190,8 @@ void dependency_default_audio_initialize(mas_proxy_interface interfaces) void dependency_default_audio_deinitialize() { + dependency_default_audio_stop_recording(); + int ret = 0; if (g_virtual_sound_stream) { sound_manager_stop_virtual_stream(g_virtual_sound_stream); @@ -268,13 +270,15 @@ static void recorder_thread_func() void dependency_default_audio_start_recording() { - int ret = audio_in_resume(g_audio_in); - if (AUDIO_IO_ERROR_NONE != ret) { - LOGD("[Recorder ERROR] Fail to resume audio in : %d", ret); + if (g_audio_in) { + int ret = audio_in_resume(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGD("[Recorder ERROR] Fail to resume audio in : %d", ret); + } + g_stop_recorder_thread.store(false); + LOGD("Starting recorder thread"); + g_recorder_thread = thread(recorder_thread_func); } - g_stop_recorder_thread.store(false); - LOGD("Starting recorder thread"); - g_recorder_thread = thread(recorder_thread_func); } void dependency_default_audio_stop_recording() @@ -284,9 +288,11 @@ void dependency_default_audio_stop_recording() g_stop_recorder_thread.store(true); g_recorder_thread.join(); } - int ret = audio_in_pause(g_audio_in); - if (AUDIO_IO_ERROR_NONE != ret) { - LOGD("[Recorder ERROR] Fail to pause audio in : %d", ret); + if (g_audio_in) { + int ret = audio_in_pause(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGD("[Recorder ERROR] Fail to pause audio in : %d", ret); + } } } -- 2.7.4 From 30753d14f8ffb4552ed2934156c0af508e8f0888 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 18 Oct 2019 13:43:18 +0900 Subject: [PATCH 10/16] Bump version to 0.2.1 Change-Id: If694bf9dbc732a183e70e7e64df7094356748643 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 4220817..c7ff283 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 494261e..663f768 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.0 +Version: 0.2.1 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 2fb1daf0e9c8ea41f082cb3afe7ef1a6086e7f66 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 21 Oct 2019 20:55:41 +0900 Subject: [PATCH 11/16] Deactivate wakeup engines with no activated assistant If there is a wakeup engine that no activate assistant specified as its wakeup event source, deactivate it since it would consume resources unnecessarily. Change-Id: I5b88fbe0b7644bdb5601216f46ccedc6566a7938 --- inc/multi_assistant_service_plugin.h | 11 ++ plugins/wakeup-manager/inc/wakeup_engine_manager.h | 1 + plugins/wakeup-manager/inc/wakeup_manager.h | 13 ++- .../wakeup-manager/inc/wakeup_manager_wrapper.h | 4 + plugins/wakeup-manager/inc/wakeup_settings.h | 9 +- .../wakeup-manager/src/wakeup_engine_manager.cpp | 3 +- plugins/wakeup-manager/src/wakeup_manager.cpp | 96 +++++++++++++++- .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 29 +++++ plugins/wakeup-manager/src/wakeup_settings.cpp | 122 ++++++++++++++++++++- src/multi_assistant_service.c | 30 +++-- src/multi_assistant_service_plugin.c | 48 ++++++++ 11 files changed, 343 insertions(+), 23 deletions(-) diff --git a/inc/multi_assistant_service_plugin.h b/inc/multi_assistant_service_plugin.h index 9bb703c..7bfd761 100644 --- a/inc/multi_assistant_service_plugin.h +++ b/inc/multi_assistant_service_plugin.h @@ -53,6 +53,10 @@ int multi_assistant_service_plugin_add_assistant_language(const char* appid, con int multi_assistant_service_plugin_set_assistant_wakeup_engine(const char* appid, const char* engine); +int multi_assistant_service_plugin_set_default_assistant(const char* appid); + +int multi_assistant_service_plugin_get_default_assistant(const char** appid); + int multi_assistant_service_plugin_activate(void); int multi_assistant_service_plugin_deactivate(void); @@ -99,6 +103,7 @@ int multi_assistant_service_plugin_set_speech_status_callback(wakeup_service_spe int multi_assistant_service_plugin_set_error_callback(wakeup_service_error_cb callback, void* user_data); + #define MA_WAKEUP_MANAGER_PATH tzplatform_mkpath(tzplatform_getid("TZ_SYS_RO_SHARE"), "multiassistant/") #define MA_DEFAULT_WAKEUP_MANAGER_FILENAME "libma-wakeup-manager.so" @@ -116,6 +121,10 @@ typedef int (*wakeup_manager_add_assistant_wakeup_word)(const char* appid, const typedef int (*wakeup_manager_add_assistant_language)(const char* appid, const char* language); #define MA_WAKEUP_MANAGER_FUNC_SET_ASSISTANT_WAKEUP_ENGINE "wakeup_manager_set_assistant_wakeup_engine" typedef int (*wakeup_manager_set_assistant_wakeup_engine)(const char* appid, const char* engine); +#define MA_WAKEUP_MANAGER_FUNC_SET_DEFAULT_ASSISTANT "wakeup_manager_set_default_assistant" +typedef int (*wakeup_manager_set_default_assistant)(const char* appid); +#define MA_WAKEUP_MANAGER_FUNC_GET_DEFAULT_ASSISTANT "wakeup_manager_get_default_assistant" +typedef int (*wakeup_manager_get_default_assistant)(const char** appid); #define MA_WAKEUP_MANAGER_FUNC_SET_LANGUAGE "wakeup_manager_set_language" typedef int (*wakeup_manager_set_language)(const char* language); #define MA_WAKEUP_MANAGER_FUNC_ACTIVATE "wakeup_manager_activate" @@ -172,6 +181,8 @@ typedef struct { wakeup_manager_add_assistant_wakeup_word add_assistant_wakeup_word; wakeup_manager_add_assistant_language add_assistant_language; wakeup_manager_set_assistant_wakeup_engine set_assistant_wakeup_engine; + wakeup_manager_set_default_assistant set_default_assistant; + wakeup_manager_get_default_assistant get_default_assistant; wakeup_manager_set_language set_language; wakeup_manager_activate activate; wakeup_manager_deactivate deactivate; diff --git a/plugins/wakeup-manager/inc/wakeup_engine_manager.h b/plugins/wakeup-manager/inc/wakeup_engine_manager.h index 4e24631..49d8e22 100644 --- a/plugins/wakeup-manager/inc/wakeup_engine_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_engine_manager.h @@ -152,6 +152,7 @@ public: bool set_language(string language); void set_assistant_activated(string appid, bool activated); + bool get_assistant_activated(string appid); void set_wake_word_audio_require_flag(bool require); void start_streaming_current_utterance_data(); diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index 645bcfc..d682387 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -78,6 +78,11 @@ public: bool add_assistant_wakeup_word(string appid, string wakeup_word, string language); bool set_assistant_wakeup_engine(string appid, string engine); + bool set_assistant_enabled(string appid, bool enabled); + bool get_assistant_enabled(string appid); + bool set_default_assistant(string appid); + string get_default_assistant(); + bool update_voice_feedback_state(string appid, bool state); bool send_assistant_specific_command(string appid, string command); bool set_background_volume(string appid, double ratio); @@ -157,6 +162,8 @@ private: { public: bool on_voice_input_language_changed(const char* language) override; + bool on_assistant_enabled_info_changed(const char* appid, bool enabled) override; + bool on_default_assistant_appid_changed(const char* appid) override; void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; } private: @@ -170,7 +177,8 @@ private: vector languageList; } AssistantLanguageInfo; vector mAssistantLanguageInfo; - map mAssistantActivated; + map mAssistantSupportsCurrentLanguage; + map mAssistantEnabled; vector mObservers; @@ -190,6 +198,7 @@ private: bool mVoiceKeyPressed{false}; string mCurrentLanguage; + string mCurrentDefaultAssistant; STREAMING_MODE mStreamingMode{STREAMING_MODE::NONE}; Ecore_Timer* mStreamingDurationTimer{nullptr}; @@ -202,4 +211,4 @@ private: } // wakeup } // multiassistant -#endif /* _WAKEUP_MANAGER_H_ */ +#endif /* _WAKEUP_MANAGER_H_ */ \ No newline at end of file diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index a5acc99..c566d83 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -85,6 +85,10 @@ EXPORT_API int wakeup_manager_add_assistant_language(const char* appid, const ch EXPORT_API int wakeup_manager_set_assistant_wakeup_engine(const char* appid, const char *engine); +EXPORT_API int wakeup_manager_set_default_assistant(const char* appid); + +EXPORT_API int wakeup_manager_get_default_assistant(const char** appid); + EXPORT_API int wakeup_manager_set_language(const char* language); EXPORT_API int wakeup_manager_activate(void); diff --git a/plugins/wakeup-manager/inc/wakeup_settings.h b/plugins/wakeup-manager/inc/wakeup_settings.h index e99c72e..99c8fd1 100644 --- a/plugins/wakeup-manager/inc/wakeup_settings.h +++ b/plugins/wakeup-manager/inc/wakeup_settings.h @@ -30,8 +30,6 @@ namespace wakeup using namespace std; -#define DEFAULT_ASSISTANT_APPID "org.tizen.voice-app" - #define WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID "db/multi-assistant/default_assistant_appid" #define WAKEUP_SETTINGS_KEY_UI_PANEL_ENABLED "db/multi-assistant/ui_panel_enabled" #define WAKEUP_SETTINGS_KEY_CONVERSATION_TIMEOUT "db/multi-assistant/conversation_timeout" @@ -47,6 +45,8 @@ class ISettingsEventObserver public: virtual ~ISettingsEventObserver() = default; virtual bool on_voice_input_language_changed(const char* language) = 0; + virtual bool on_assistant_enabled_info_changed(const char* appid, bool enabled) = 0; + virtual bool on_default_assistant_appid_changed(const char* appid) = 0; }; class CWakeupSettings @@ -77,14 +77,15 @@ public: private: vector mObservers; - string mDefaultAssistantAppid{DEFAULT_ASSISTANT_APPID}; + string mDefaultAssistantAppid; bool mUiPanelEnabled{true}; float mConversationTimeout{5.0}; bool mMultipleMode{true}; - vector mEnabledAssistants{DEFAULT_ASSISTANT_APPID}; + vector mEnabledAssistants; float mWakeupPolicyDelay{0.1}; vector mWakeupPolicyPriority; // No priority by default float mStreamingDurationMax{10.0}; + string mVoiceInputLanguage; }; } // wakeup diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index 1b159aa..46e9a1d 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -413,7 +413,8 @@ void CWakeupEngineManager::engine_set_assistant_specific_command(string appid, s void CWakeupEngineManager::engine_feed_audio_data(long time, void* data, int len) { for (const auto& info : mEngineInfo) { - if (info.audio_data_require_status && + if (info.activated && + info.audio_data_require_status && info.interface.feed_audio_data) { int ret = info.interface.feed_audio_data(time, data, len); if (0 != ret) { diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 14373df..81ed6eb 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -95,6 +95,7 @@ bool CWakeupManager::initialize() mAudioEventObserver.set_wakeup_manager(this); mSettingsEventObserver.set_wakeup_manager(this); + mWakeupSettings.subscribe(&mSettingsEventObserver); mWakeupSettings.initialize(); mAudioEventObserver.set_wakeup_engine_manager(&mWakeupEngineManager); @@ -112,8 +113,6 @@ bool CWakeupManager::initialize() dependency_resolver_initialize(interface); - mWakeupSettings.subscribe(&mSettingsEventObserver); - MWR_LOGD("[END]"); return true; } @@ -200,6 +199,12 @@ bool CWakeupManager::add_assistant_language(string appid, string language) info.languageList.push_back(language); mAssistantLanguageInfo.push_back(info); } + if (0 == mCurrentLanguage.compare(language)) { + mAssistantSupportsCurrentLanguage[appid] = true; + if (true == mAssistantEnabled[appid]) { + mWakeupEngineManager.set_assistant_activated(appid, true); + } + } MWR_LOGD("[END]"); return true; } @@ -224,6 +229,56 @@ bool CWakeupManager::set_assistant_wakeup_engine(string appid, string engine) return true; } +bool CWakeupManager::set_assistant_enabled(string appid, bool enabled) +{ + MWR_LOGD("[ENTER]"); + + mAssistantEnabled[appid] = enabled; + bool activated = enabled; + if (false == mAssistantSupportsCurrentLanguage[appid]) { + activated = false; + } + if (0 == appid.compare(mCurrentDefaultAssistant)) { + activated = true; + } + mWakeupEngineManager.set_assistant_activated(appid, activated); + + MWR_LOGD("[END]"); + return true; +} + +bool CWakeupManager::set_default_assistant(string appid) +{ + MWR_LOGD("[ENTER]"); + + /* Check if previous default assistant has to be deactivated */ + bool activated = true; + if (false == mAssistantSupportsCurrentLanguage[mCurrentDefaultAssistant]) { + activated = false; + } + if (false == mAssistantEnabled[mCurrentDefaultAssistant]) { + activated = false; + } + mWakeupEngineManager.set_assistant_activated(mCurrentDefaultAssistant, activated); + + /* New default assistant has to be activated no matter what */ + mWakeupEngineManager.set_assistant_activated(appid, true); + mCurrentDefaultAssistant = appid; + + MWR_LOGD("[END]"); + return true; +} + +string CWakeupManager::get_default_assistant() +{ + return mCurrentDefaultAssistant; +} + +bool CWakeupManager::get_assistant_enabled(string appid) +{ + return mAssistantEnabled[appid]; +} + bool CWakeupManager::set_language(string language) { bool ret = false; @@ -242,12 +297,19 @@ bool CWakeupManager::set_language(string language) } } if(false == found) { - mAssistantActivated[info.appid] = false; + mAssistantSupportsCurrentLanguage[info.appid] = false; } else { - mAssistantActivated[info.appid] = true; + mAssistantSupportsCurrentLanguage[info.appid] = true; } - mWakeupEngineManager.set_assistant_activated(info.appid, found); + bool activated = found; + if (false == mAssistantEnabled[info.appid]) { + activated = false; + } + if (0 == info.appid.compare(mCurrentDefaultAssistant)) { + activated = true; + } + mWakeupEngineManager.set_assistant_activated(info.appid, activated); } mWakeupEngineManager.set_language(language); @@ -718,6 +780,14 @@ bool CWakeupManager::CEngineEventObserver::on_wakeup_event(string engine_name, w { MWR_LOGD("[ENTER]"); if (nullptr == mWakeupManager) return false; + if (nullptr == wakeup_info.wakeup_appid) return false; + + if (0 != mWakeupManager->get_default_assistant().compare(wakeup_info.wakeup_appid)) { + if (false == mWakeupManager->get_assistant_enabled(string{wakeup_info.wakeup_appid})) { + MWR_LOGE("Wakeup event with deactivated appid : %s", wakeup_info.wakeup_appid); + return false; + } + } CWakeupPolicy* policy = mWakeupManager->get_wakeup_policy(); if (policy) { @@ -855,5 +925,21 @@ bool CWakeupManager::CSettingsEventObserver::on_voice_input_language_changed( return true; } +bool CWakeupManager::CSettingsEventObserver::on_assistant_enabled_info_changed( + const char* appid, bool enabled) +{ + if (nullptr == mWakeupManager || nullptr == appid) return false; + mWakeupManager->set_assistant_enabled(std::string(appid), enabled); + return true; +} + +bool CWakeupManager::CSettingsEventObserver::on_default_assistant_appid_changed( + const char* appid) +{ + if (nullptr == mWakeupManager || nullptr == appid) return false; + mWakeupManager->set_default_assistant(std::string(appid)); + return true; +} + } // wakeup } // multiassistant diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index d639cf1..9853c34 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -153,6 +153,35 @@ int wakeup_manager_set_assistant_wakeup_engine(const char* appid, const char* en return 0; } +int wakeup_manager_set_default_assistant(const char* appid) +{ + MWR_LOGD("[ENTER]"); + + if (NULL == appid) { + MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)", appid); + return -1; + } + + MWR_LOGD("[DEBUG] default_assistant appid(%s)", appid); + g_wakeup_manager.set_default_assistant(string{appid}); + + MWR_LOGD("[END]"); + return 0; +} + +int wakeup_manager_get_default_assistant(const char** appid) +{ + static string default_assistant; + default_assistant = g_wakeup_manager.get_default_assistant(); + + if (default_assistant.empty()) { + *appid = nullptr; + } else { + *appid = default_assistant.c_str(); + } + return 0; +} + int wakeup_manager_set_language(const char* language) { MWR_LOGD("[ENTER]"); diff --git a/plugins/wakeup-manager/src/wakeup_settings.cpp b/plugins/wakeup-manager/src/wakeup_settings.cpp index 0787f09..c68d6ba 100644 --- a/plugins/wakeup-manager/src/wakeup_settings.cpp +++ b/plugins/wakeup-manager/src/wakeup_settings.cpp @@ -9,6 +9,13 @@ namespace multiassistant namespace wakeup { +/* Utility function for checking if an element exists in a container */ +template +static auto contains(const C& v, const T& x) -> decltype(end(v), true) +{ + return end(v) != find(begin(v), end(v), x); +} + CWakeupSettings::CWakeupSettings() { } @@ -17,8 +24,9 @@ CWakeupSettings::~CWakeupSettings() { } -static void wakeup_setting_input_language_changed_cb(keynode_t *node, void* data) +static void wakeup_setting_input_language_changed_cb(keynode_t* node, void* data) { + MWR_LOGD("[ENTER]"); if (nullptr == node) return; CWakeupSettings* settings = static_cast(data); @@ -39,6 +47,84 @@ static void wakeup_setting_input_language_changed_cb(keynode_t *node, void* data } } +static void wakeup_setting_enabled_assistants_changed_cb(keynode_t* node, void* data) +{ + MWR_LOGD("[ENTER]"); + if (nullptr == node) return; + + CWakeupSettings* settings = static_cast(data); + if (nullptr == settings) return; + + if (VCONF_TYPE_STRING == node->type) { + vector newlyAddedAssistants; + vector newlyRemovedAssistants; + const char* value = static_cast(node->value.s); + if (value) { + vector previouslyEnabledAssistants = settings->get_enabled_assistants(); + vector currentlyEnabledAssistants; + string token; + istringstream iss(value); + currentlyEnabledAssistants.clear(); + while (getline(iss, token, ';')) { + currentlyEnabledAssistants.push_back(token); + MWR_LOGD("enabled_assistants : %s", token.c_str()); + } + + for (const auto& assistant : currentlyEnabledAssistants) { + if (!contains(previouslyEnabledAssistants, assistant)) { + newlyAddedAssistants.push_back(assistant); + } + } + for (const auto& assistant : previouslyEnabledAssistants) { + if (!contains(currentlyEnabledAssistants, assistant)) { + newlyRemovedAssistants.push_back(assistant); + } + } + } + + vector observers = settings->get_observers(); + for (const auto& observer : observers) { + if (observer) { + for (const auto& assistant : newlyAddedAssistants) { + if (!observer->on_assistant_enabled_info_changed(assistant.c_str(), true)) { + LOGW("[Settings WARNING] One of the observer returned false"); + } + } + for (const auto& assistant : newlyRemovedAssistants) { + if (!observer->on_assistant_enabled_info_changed(assistant.c_str(), false)) { + LOGW("[Settings WARNING] One of the observer returned false"); + } + } + } + } + } else { + LOGE("[Settings ERROR] the value type is not string : %d", node->type); + } +} + +static void wakeup_setting_default_assistant_appid_changed_cb(keynode_t* node, void* data) +{ + MWR_LOGD("[ENTER]"); + if (nullptr == node) return; + + CWakeupSettings* settings = static_cast(data); + if (nullptr == settings) return; + + if (VCONF_TYPE_STRING == node->type) { + const char* value = static_cast(node->value.s); + vector observers = settings->get_observers(); + for (const auto& observer : observers) { + if (observer) { + if (!observer->on_default_assistant_appid_changed(value)) { + LOGW("[Settings WARNING] One of the observer returned false"); + } + } + } + } else { + LOGE("[Settings ERROR] the value type is not string : %d", node->type); + } +} + void CWakeupSettings::initialize() { int vconf_ret; @@ -50,6 +136,13 @@ void CWakeupSettings::initialize() if (vconf_str) { mDefaultAssistantAppid = vconf_str; MWR_LOGD("default_assistant_appid : %s", mDefaultAssistantAppid.c_str()); + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_default_assistant_appid_changed(vconf_str)) { + LOGW("[Settings WARNING] One of the observer returned false"); + } + } + } free(vconf_str); vconf_str = nullptr; } @@ -76,6 +169,13 @@ void CWakeupSettings::initialize() while (getline(iss, token, ';')) { mEnabledAssistants.push_back(token); MWR_LOGD("enabled_assistants : %s", token.c_str()); + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_assistant_enabled_info_changed(token.c_str(), true)) { + LOGW("[Settings WARNING] One of the observer returned false"); + } + } + } } free(vconf_str); vconf_str = nullptr; @@ -102,14 +202,34 @@ void CWakeupSettings::initialize() mStreamingDurationMax = vconf_double; MWR_LOGD("streaming_duration_max : %f", mStreamingDurationMax); } + vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_VOICE_INPUT_LANGUAGE); + if (vconf_str) { + mVoiceInputLanguage = vconf_str; + MWR_LOGD("voice input language : %s", mVoiceInputLanguage.c_str()); + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_voice_input_language_changed(vconf_str)) { + LOGW("[Settings WARNING] One of the observer returned false"); + } + } + } + free(vconf_str); + vconf_str = nullptr; + } vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_VOICE_INPUT_LANGUAGE, wakeup_setting_input_language_changed_cb, this); + vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS, + wakeup_setting_enabled_assistants_changed_cb, this); + vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID, + wakeup_setting_default_assistant_appid_changed_cb, this); } void CWakeupSettings::deinitialize() { vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_VOICE_INPUT_LANGUAGE, NULL); + vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS, NULL); + vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID, NULL); } void CWakeupSettings::subscribe(ISettingsEventObserver *observer) diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index a6a8c0a..5b2b220 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -37,8 +37,9 @@ static const char *g_current_lang = "en_US"; +#define ENABLE_MULTI_ASSISTANT_BY_DEFAULT + #define MULTI_ASSISTANT_SETTINGS_ACTIVATED "db/multi-assistant/activated" -#define WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID "db/multi-assistant/default_assistant_appid" #define WAKEUP_SETTINGS_KEY_PREPROCESSING_ASSISTANT_APPID "db/multi-assistant/preprocessing_assistant_appid" #define WAKEUP_SETTINGS_KEY_PRELAUNCH_MODE "db/multi-assistant/prelaunch_mode" @@ -744,23 +745,32 @@ static int init_wakeup(void) /* Activate / deactivate according to the vconf key setting */ mas_active_state_changed_cb(NULL, NULL); } else { +#ifdef ENABLE_MULTI_ASSISTANT_BY_DEFAULT /* Multi-assistant needs to be enabled by default, unless disabled explicitly */ multi_assistant_service_plugin_activate(); - vconf_set_bool(MULTI_ASSISTANT_SETTINGS_ACTIVATED, 1); - vconf_notify_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb, NULL); + const char *default_assistant = NULL; + if (0 == multi_assistant_service_plugin_get_default_assistant(&default_assistant)) { + if (NULL == default_assistant) { + if (g_maclient_info[0].used) { + default_assistant = g_maclient_info[0].appid; + MAS_LOGW("No default assistant, setting %s as default", default_assistant); + multi_assistant_service_plugin_set_default_assistant(default_assistant); + } else { + MAS_LOGE("No default assistant, and no assistant installed"); + } + } + } +#endif } /* CHECK NEEDED : should the code segment below and activation logic above be moved to wakeup manger? */ int prelaunch_mode; int res = vconf_get_bool(WAKEUP_SETTINGS_KEY_PRELAUNCH_MODE, &prelaunch_mode); if (0 == res && 0 != prelaunch_mode) { - char *vconf_str; - vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID); - if (vconf_str) { - MAS_LOGD("prelaunching default_assistant_appid : %s", vconf_str); - mas_launch_client_by_appid(vconf_str, CLIENT_LAUNCH_MODE_PRELAUNCH); - free(vconf_str); - vconf_str = NULL; + const char *default_assistant = NULL; + if (0 == multi_assistant_service_plugin_get_default_assistant(&default_assistant)) { + MAS_LOGD("prelaunching default_assistant_appid : %s", default_assistant); + mas_launch_client_by_appid(default_assistant, CLIENT_LAUNCH_MODE_PRELAUNCH); } } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 700b3f2..518e5e5 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -383,6 +383,12 @@ int multi_assistant_service_plugin_initialize(void) _wakeup_manager_interface.set_assistant_wakeup_engine = (wakeup_manager_set_assistant_wakeup_engine)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_ASSISTANT_WAKEUP_ENGINE); + _wakeup_manager_interface.set_default_assistant = + (wakeup_manager_set_default_assistant)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_SET_DEFAULT_ASSISTANT); + _wakeup_manager_interface.get_default_assistant = + (wakeup_manager_get_default_assistant)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_GET_DEFAULT_ASSISTANT); _wakeup_manager_interface.set_language = (wakeup_manager_set_language)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_LANGUAGE); @@ -613,6 +619,48 @@ int multi_assistant_service_plugin_set_assistant_wakeup_engine(const char* appid return ret; } +int multi_assistant_service_plugin_set_default_assistant(const char* appid) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_set_default_assistant func = _wakeup_manager_interface.set_default_assistant; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_DEFAULT_ASSISTANT); + } else { + ret = func(appid); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set default assistant(%s), ret(%d)", appid, ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + return ret; +} + +int multi_assistant_service_plugin_get_default_assistant(const char** appid) +{ + int ret = -1; + if (NULL == appid) { + MAS_LOGE("[ERROR] appid is not valid"); + return ret; + } + if (NULL != g_handle) { + wakeup_manager_get_default_assistant func = _wakeup_manager_interface.get_default_assistant; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_GET_DEFAULT_ASSISTANT); + } else { + ret = func(appid); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to get default assistant, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + return ret; +} + int multi_assistant_service_plugin_activate(void) { int ret = -1; -- 2.7.4 From 63461547d0e4d312920a8afce5d3666c816a608d Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 22 Oct 2019 18:09:05 +0900 Subject: [PATCH 12/16] Ignore enabled assistants setting in single mode Change-Id: If8832442ec3c9cfd56f8823a5c0636f284190132 --- plugins/wakeup-manager/inc/wakeup_settings.h | 2 +- plugins/wakeup-manager/src/wakeup_settings.cpp | 40 +++++++++++++++----------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/plugins/wakeup-manager/inc/wakeup_settings.h b/plugins/wakeup-manager/inc/wakeup_settings.h index 99c8fd1..13ecbb4 100644 --- a/plugins/wakeup-manager/inc/wakeup_settings.h +++ b/plugins/wakeup-manager/inc/wakeup_settings.h @@ -80,7 +80,7 @@ private: string mDefaultAssistantAppid; bool mUiPanelEnabled{true}; float mConversationTimeout{5.0}; - bool mMultipleMode{true}; + bool mMultipleMode{false}; vector mEnabledAssistants; float mWakeupPolicyDelay{0.1}; vector mWakeupPolicyPriority; // No priority by default diff --git a/plugins/wakeup-manager/src/wakeup_settings.cpp b/plugins/wakeup-manager/src/wakeup_settings.cpp index c68d6ba..96a006f 100644 --- a/plugins/wakeup-manager/src/wakeup_settings.cpp +++ b/plugins/wakeup-manager/src/wakeup_settings.cpp @@ -161,24 +161,26 @@ void CWakeupSettings::initialize() mMultipleMode = vconf_bool; MWR_LOGD("multiple_mode : %s", (mMultipleMode ? "true" : "false")); } - vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS); - if (vconf_str) { - string token; - istringstream iss(vconf_str); - mEnabledAssistants.clear(); - while (getline(iss, token, ';')) { - mEnabledAssistants.push_back(token); - MWR_LOGD("enabled_assistants : %s", token.c_str()); - for (const auto& observer : mObservers) { - if (observer) { - if (!observer->on_assistant_enabled_info_changed(token.c_str(), true)) { - LOGW("[Settings WARNING] One of the observer returned false"); + if (true == mMultipleMode) { + vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS); + if (vconf_str) { + string token; + istringstream iss(vconf_str); + mEnabledAssistants.clear(); + while (getline(iss, token, ';')) { + mEnabledAssistants.push_back(token); + MWR_LOGD("enabled_assistants : %s", token.c_str()); + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_assistant_enabled_info_changed(token.c_str(), true)) { + LOGW("[Settings WARNING] One of the observer returned false"); + } } } } + free(vconf_str); + vconf_str = nullptr; } - free(vconf_str); - vconf_str = nullptr; } vconf_ret = vconf_get_dbl(WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_DELAY, &vconf_double); if (0 == vconf_ret) { @@ -219,17 +221,21 @@ void CWakeupSettings::initialize() vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_VOICE_INPUT_LANGUAGE, wakeup_setting_input_language_changed_cb, this); - vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS, - wakeup_setting_enabled_assistants_changed_cb, this); vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID, wakeup_setting_default_assistant_appid_changed_cb, this); + if (true == mMultipleMode) { + vconf_notify_key_changed(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS, + wakeup_setting_enabled_assistants_changed_cb, this); + } } void CWakeupSettings::deinitialize() { vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_VOICE_INPUT_LANGUAGE, NULL); - vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS, NULL); vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID, NULL); + if (true == mMultipleMode) { + vconf_ignore_key_changed(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS, NULL); + } } void CWakeupSettings::subscribe(ISettingsEventObserver *observer) -- 2.7.4 From 0c6a47d8546af3e403e7a0908af1287222c24d2d Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 22 Oct 2019 18:14:26 +0900 Subject: [PATCH 13/16] Bump version to 0.2.2 Change-Id: I21cb1ebde99f3234b0f3c96a1cef201e9bee5b4f --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index c7ff283..c7c4f14 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 663f768..f6d0b3c 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.1 +Version: 0.2.2 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From a31117e83974901f41b952766e94c4dcff4bd819 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 24 Oct 2019 16:31:22 +0900 Subject: [PATCH 14/16] Add exception handlers for the functions in external modules Change-Id: Iae5619b299fc0a6ff737b042f90d23d13a3864c0 --- plugins/wakeup-manager/src/dependency_resolver.cpp | 65 +++++++-- .../wakeup-manager/src/wakeup_engine_manager.cpp | 150 ++++++++++++++------- 2 files changed, 156 insertions(+), 59 deletions(-) diff --git a/plugins/wakeup-manager/src/dependency_resolver.cpp b/plugins/wakeup-manager/src/dependency_resolver.cpp index 551290e..43b080d 100644 --- a/plugins/wakeup-manager/src/dependency_resolver.cpp +++ b/plugins/wakeup-manager/src/dependency_resolver.cpp @@ -22,6 +22,8 @@ #include #include +#include + #include "multi_assistant_main.h" #include "dependency_resolver.h" @@ -93,7 +95,12 @@ int dependency_resolver_initialize(mas_proxy_interface interface) if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_INITIALIZE); } else { - ret = func(interface, &dependency_version); + try { + ret = func(interface, &dependency_version); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_INITIALIZE, e.what()); + } if (0 != ret) { MAS_LOGE("[ERROR] Fail to initialize, ret(%d)", ret); } @@ -114,7 +121,12 @@ int dependency_resolver_deinitialize(void) if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_DEINITIALIZE); } else { - ret = func(); + try { + ret = func(); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_DEINITIALIZE, e.what()); + } if (0 != ret) { MAS_LOGE("[ERROR] Fail to deinitialize, ret(%d)", ret); } @@ -137,7 +149,12 @@ int dependency_resolver_set_error_callback(mas_dependency_error_cb callback, voi if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_SET_ERROR_CALLBACK); } else { - ret = func(callback, user_data); + try { + ret = func(callback, user_data); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_DEINITIALIZE, e.what()); + } if (0 != ret) { MAS_LOGE("[ERROR] Fail to set error callback(%p, %p), ret(%d)", callback, user_data, ret); } @@ -157,7 +174,12 @@ int dependency_resolver_start_recording(void) if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_START_RECORDING); } else { - ret = func(); + try { + ret = func(); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_DEINITIALIZE, e.what()); + } if (0 != ret) { MAS_LOGE("[ERROR] Fail to start recording, ret(%d)", ret); } @@ -177,7 +199,12 @@ int dependency_resolver_stop_recording(void) if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_STOP_RECORDING); } else { - ret = func(); + try { + ret = func(); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_DEINITIALIZE, e.what()); + } if (0 != ret) { MAS_LOGE("[ERROR] Fail to stop recording, ret(%d)", ret); } @@ -198,7 +225,12 @@ int dependency_resolver_set_recording_session(unsigned int session) if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_SET_RECORDING_SESSION); } else { - ret = func(session); + try { + ret = func(session); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_DEINITIALIZE, e.what()); + } if (0 != ret) { MAS_LOGE("[ERROR] Fail to set recording session, ret(%d)", ret); } @@ -218,7 +250,12 @@ int dependency_resolver_set_background_volume(double ratio) if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_SET_BACKGROUND_VOLUME); } else { - ret = func(ratio); + try { + ret = func(ratio); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_DEINITIALIZE, e.what()); + } if (0 != ret) { MAS_LOGE("[ERROR] Fail to set background volume to %f, ret(%d)", ratio, ret); } @@ -238,7 +275,12 @@ int dependency_resolver_get_audio_format(int* rate, int* channel, int* audio_typ if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_GET_AUDIO_FORMAT); } else { - ret = func(rate, channel, audio_type); + try { + ret = func(rate, channel, audio_type); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_DEINITIALIZE, e.what()); + } if (0 != ret) { MAS_LOGE("[ERROR] Fail to get audio format, ret(%d)", ret); } @@ -258,7 +300,12 @@ int dependency_resolver_get_audio_source_type(char** type) if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_GET_AUDIO_SOURCE_TYPE); } else { - ret = func(type); + try { + ret = func(type); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_DEINITIALIZE, e.what()); + } if (0 != ret) { MAS_LOGE("[ERROR] Fail to get audio source type, ret(%d)", ret); } diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index 46e9a1d..5e1163c 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -77,20 +77,25 @@ void CWakeupEngineManager::initialize() void CWakeupEngineManager::deinitialize() { for (auto& info : mEngineInfo) { - if (info.interface.set_wakeup_event_callback) { - info.interface.set_wakeup_event_callback(nullptr, nullptr); - } - if (info.interface.set_speech_status_callback) { - info.interface.set_speech_status_callback(nullptr, nullptr); - } - if (info.interface.set_error_callback) { - info.interface.set_error_callback(nullptr, nullptr); - } - if (info.interface.set_audio_data_require_status_callback) { - info.interface.set_audio_data_require_status_callback(nullptr, nullptr); - } - if (info.interface.deinitialize) { - info.interface.deinitialize(); + try { + if (info.interface.set_wakeup_event_callback) { + info.interface.set_wakeup_event_callback(nullptr, nullptr); + } + if (info.interface.set_speech_status_callback) { + info.interface.set_speech_status_callback(nullptr, nullptr); + } + if (info.interface.set_error_callback) { + info.interface.set_error_callback(nullptr, nullptr); + } + if (info.interface.set_audio_data_require_status_callback) { + info.interface.set_audio_data_require_status_callback(nullptr, nullptr); + } + if (info.interface.deinitialize) { + info.interface.deinitialize(); + } + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); } if (info.engine_handle) { dlclose(info.engine_handle); @@ -138,7 +143,12 @@ bool CWakeupEngineManager::set_language(string language) { for (const auto& info : mEngineInfo) { if (info.interface.set_language) { - info.interface.set_language(language.c_str()); + try { + info.interface.set_language(language.c_str()); + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); + } } } return true; @@ -164,9 +174,19 @@ void CWakeupEngineManager::set_assistant_activated(string appid, bool activated) info.activated = (info.activated_assistants.size() > 0); if (previously_activated != info.activated) { if (info.activated) { - info.interface.activate(); + try { + info.interface.activate(); + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); + } } else { - info.interface.deactivate(); + try { + info.interface.deactivate(); + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); + } } /* Activated status changed, need to update audio_data_require_status too */ on_audio_data_require_status(info.engine_name, info.audio_data_require_status); @@ -181,7 +201,12 @@ void CWakeupEngineManager::set_wake_word_audio_require_flag(bool require) mWakeWordAudioRequired = require; for (const auto& info : mEngineInfo) { if (info.interface.set_wake_word_audio_require_flag) { - info.interface.set_wake_word_audio_require_flag(require); + try { + info.interface.set_wake_word_audio_require_flag(require); + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); + } } } } @@ -336,7 +361,12 @@ void CWakeupEngineManager::update_manager_state(wakeup_manager_state_e state) { for (const auto& info : mEngineInfo) { if (info.interface.update_manager_state) { - info.interface.update_manager_state(state); + try { + info.interface.update_manager_state(state); + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); + } } } } @@ -392,7 +422,12 @@ void CWakeupEngineManager::engine_add_wakeup_word(string appid, string wakeup_wo bool found = contains(info.assistant_list, appid); if (found) { if (info.interface.add_wakeup_word) { - info.interface.add_wakeup_word(appid.c_str(), wakeup_word.c_str(), language.c_str()); + try { + info.interface.add_wakeup_word(appid.c_str(), wakeup_word.c_str(), language.c_str()); + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); + } } } } @@ -404,7 +439,12 @@ void CWakeupEngineManager::engine_set_assistant_specific_command(string appid, s bool found = contains(info.assistant_list, appid); if (found) { if (info.interface.set_assistant_specific_command) { - info.interface.set_assistant_specific_command(appid.c_str(), command.c_str()); + try { + info.interface.set_assistant_specific_command(appid.c_str(), command.c_str()); + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); + } } } } @@ -416,9 +456,14 @@ void CWakeupEngineManager::engine_feed_audio_data(long time, void* data, int len if (info.activated && info.audio_data_require_status && info.interface.feed_audio_data) { - int ret = info.interface.feed_audio_data(time, data, len); - if (0 != ret) { - LOGE("[ERROR] Fail to feed speech data, ret(%d) : %s", ret, info.engine_name.c_str()); + try { + int ret = info.interface.feed_audio_data(time, data, len); + if (0 != ret) { + LOGE("[ERROR] Fail to feed speech data, ret(%d) : %s", ret, info.engine_name.c_str()); + } + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); } } } @@ -618,35 +663,40 @@ void CWakeupEngineManager::add_engine(string name, string path) user_data.engine_name = info.engine_name; callback_user_data.push_back(user_data); - if (info.interface.set_wakeup_event_callback) { - info.interface.set_wakeup_event_callback( - [](wakeup_event_info info, void* user_data) { - CallbackUserData *data = static_cast(user_data); - if (nullptr == data) return; - if (nullptr == data->manager) return; - info.wakeup_engine = data->engine_name.c_str(); - data->manager->on_wakeup_event(data->engine_name, info); - }, &(callback_user_data.back())); - } + try { + if (info.interface.set_wakeup_event_callback) { + info.interface.set_wakeup_event_callback( + [](wakeup_event_info info, void* user_data) { + CallbackUserData *data = static_cast(user_data); + if (nullptr == data) return; + if (nullptr == data->manager) return; + info.wakeup_engine = data->engine_name.c_str(); + data->manager->on_wakeup_event(data->engine_name, info); + }, &(callback_user_data.back())); + } - if (info.interface.set_audio_data_require_status_callback) { - info.interface.set_audio_data_require_status_callback( - [](bool require, void* user_data) { - CallbackUserData *data = static_cast(user_data); - if (nullptr == data) return; - if (nullptr == data->manager) return; - data->manager->on_audio_data_require_status(data->engine_name, require); - }, &(callback_user_data.back())); - } + if (info.interface.set_audio_data_require_status_callback) { + info.interface.set_audio_data_require_status_callback( + [](bool require, void* user_data) { + CallbackUserData *data = static_cast(user_data); + if (nullptr == data) return; + if (nullptr == data->manager) return; + data->manager->on_audio_data_require_status(data->engine_name, require); + }, &(callback_user_data.back())); + } - if (info.interface.initialize) { - info.interface.initialize(); - } - if (info.interface.get_version) { - int version; - if (0 == info.interface.get_version(&version)) { - info.version = version; + if (info.interface.initialize) { + info.interface.initialize(); + } + if (info.interface.get_version) { + int version; + if (0 == info.interface.get_version(&version)) { + info.version = version; + } } + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); } } -- 2.7.4 From b22e1d111f1cf6a6840be16e44149e27b920d52d Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 24 Oct 2019 16:36:25 +0900 Subject: [PATCH 15/16] Bump version to 0.2.3 Change-Id: I6019ce410e76fb78c2a525d4929379a1342e48d9 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index c7c4f14..4ae038a 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index f6d0b3c..19ee155 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.2 +Version: 0.2.3 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 54f7b4364bcdfc7d925fb4f261ce1b3345af847b Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 24 Oct 2019 21:33:18 +0900 Subject: [PATCH 16/16] Remove multi-assistant header inclusion Change-Id: I13635bc2bbd080e6dc2706a057f807a2f29f51d7 --- inc/multi_wakeup_recognizer.h | 33 +++++++++++++++++++++- plugins/wakeup-manager/inc/wakeup_engine_manager.h | 2 -- plugins/wakeup-manager/inc/wakeup_manager.h | 2 -- .../wakeup-manager/inc/wakeup_manager_wrapper.h | 15 +++++++++- src/multi_assistant_service.c | 1 - src/multi_assistant_service_plugin.c | 1 - 6 files changed, 46 insertions(+), 8 deletions(-) diff --git a/inc/multi_wakeup_recognizer.h b/inc/multi_wakeup_recognizer.h index c818e1a..6c3eb6f 100644 --- a/inc/multi_wakeup_recognizer.h +++ b/inc/multi_wakeup_recognizer.h @@ -19,7 +19,6 @@ #define _MULTI_WAKEUP_RECOGNIZER_H_ #include -#include #ifdef __cplusplus extern "C" { @@ -61,6 +60,38 @@ typedef enum { WAKEUP_ASR_RESULT_EVENT_ERROR /**< Event when the recognition has failed */ } wakeup_asr_result_event_e; +typedef enum { + MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE = 0, /**< Current utterance */ + MA_AUDIO_STREAMING_DATA_TYPE_PREVIOUS_UTTERANCE, /**< Previous utterance */ + MA_AUDIO_STREAMING_DATA_TYPE_FOLLOW_UP_SPEECH /**< Follow-up speech */ +} ma_audio_streaming_data_type_e; + +typedef enum { + MA_ACTIVE_STATE_INACTIVE = 0, /**< 'Inactive' state */ + MA_ACTIVE_STATE_ACTIVE, /**< 'Active' state */ + MA_ACTIVE_STATE_PREPROCESSING, /**< 'Preprocessing' state */ +} ma_active_state_e; + +typedef enum { + MA_RECOGNITION_RESULT_EVENT_SUCCESS = 0, /**< Recognition succeeded */ + MA_RECOGNITION_RESULT_EVENT_EMPTY_TEXT, /**< No text result recognized */ + MA_RECOGNITION_RESULT_EVENT_ERROR, /**< Unknown error occurred */ + MA_RECOGNITION_RESULT_EVENT_FALSE_TRIGGER /**< Turned out to be a false trigger */ +} ma_recognition_result_event_e; + +typedef enum { + MA_PREPROCESSING_ALLOW_NONE = 0, /**< No preprocessing allowed */ + MA_PREPROCESSING_ALLOW_UTTERANCE, /**< Preprocessing allowed for utterance audio only */ + MA_PREPROCESSING_ALLOW_FOLLOW_UP, /**< Preprocessing allowed for follow-up audio only */ + MA_PREPROCESSING_ALLOW_ALL, /**< Preprocessing allowed for all audio */ +} ma_preprocessing_allow_mode_e; + +typedef enum { + MA_AUDIO_STREAMING_DATA_SECTION_UTTERANCE = 0, /**< Utterance section started */ + MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD, /**< Wake word section started */ +} ma_audio_streaming_data_section_e; + +typedef void (*ma_audio_streaming_data_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); typedef void (*wakeup_service_wakeup_event_cb)(wakeup_event_info wakeup_info, const char* wakeup_word, void* user_data); diff --git a/plugins/wakeup-manager/inc/wakeup_engine_manager.h b/plugins/wakeup-manager/inc/wakeup_engine_manager.h index 49d8e22..c768f9b 100644 --- a/plugins/wakeup-manager/inc/wakeup_engine_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_engine_manager.h @@ -19,8 +19,6 @@ #include "wakeup_manager_wrapper.h" -#include - #include #include #include diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index d682387..e6f5251 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -24,8 +24,6 @@ #include "wakeup_audio_manager.h" #include "wakeup_policy_default.h" -#include "multi_assistant_common.h" - #include #include diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index c566d83..7f408dc 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -22,7 +22,6 @@ #include #include #include -#include #include "wakeup_interfaces.h" @@ -44,6 +43,20 @@ typedef enum { WAKEUP_MANAGER_STATE_VOICE_FEEDBACK = 4 } wakeup_manager_state_e; +typedef enum { + MA_PREPROCESSING_ALLOW_NONE = 0, /**< No preprocessing allowed */ + MA_PREPROCESSING_ALLOW_UTTERANCE, /**< Preprocessing allowed for utterance audio only */ + MA_PREPROCESSING_ALLOW_FOLLOW_UP, /**< Preprocessing allowed for follow-up audio only */ + MA_PREPROCESSING_ALLOW_ALL, /**< Preprocessing allowed for all audio */ +} ma_preprocessing_allow_mode_e; + +typedef enum { + MA_AUDIO_STREAMING_DATA_SECTION_UTTERANCE = 0, /**< Utterance section started */ + MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD, /**< Wake word section started */ +} ma_audio_streaming_data_section_e; + +typedef void (*ma_audio_streaming_data_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); + typedef void (*wakeup_service_wakeup_event_cb)(wakeup_event_info wakeup_info, void* user_data); typedef void (*wakeup_service_speech_streaming_cb)(wakeup_speech_streaming_event_e event, void* buffer, int len, void *user_data); diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 5b2b220..0baf44e 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -33,7 +33,6 @@ #include "multi_assistant_service_plugin.h" #include "multi_assistant_dbus.h" #include "multi_assistant_config.h" -#include "multi_assistant_common.h" static const char *g_current_lang = "en_US"; diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 518e5e5..a04366f 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -32,7 +32,6 @@ #include "multi_assistant_service.h" #include "multi_assistant_service_plugin.h" #include "multi_assistant_dbus.h" -#include "multi_assistant_common.h" /* Sound buf save for test */ #if 0 -- 2.7.4