From a2aa3a6561b314fbd9bda2d2996deebb9b59d62a Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 16:00:41 +0900 Subject: [PATCH 01/16] Apply secure options to compile flags Change-Id: Ia582efe7e1b7d7beb6b1fe989ada3f4f47ef6559 --- CMakeLists.txt | 4 ++-- plugins/wakeup-manager/CMakeLists.txt | 2 +- plugins/wakeup-manager/dependency-default/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59845fa..1cf3c04 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,8 +36,8 @@ FOREACH(flag ${pkgs_CFLAGS}) ENDFOREACH(flag) SET(CMAKE_EXE_LINKER_FLAGS "-Wl,--as-needed -pie") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CFLAGS} -fPIE -std=c++11 -fvisibility=hidden -fvisibility-inlines-hidden") -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_CFLAGS} -fPIE -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIE -pie -std=c++11 -fvisibility=hidden -fvisibility-inlines-hidden") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_CFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIE -pie -fvisibility=hidden") # SET WARNNING OPTION ---------------------------------------------------------------------------- SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") diff --git a/plugins/wakeup-manager/CMakeLists.txt b/plugins/wakeup-manager/CMakeLists.txt index 752f085..7f07524 100644 --- a/plugins/wakeup-manager/CMakeLists.txt +++ b/plugins/wakeup-manager/CMakeLists.txt @@ -58,7 +58,7 @@ SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -fPIC -Wall" ) IF("${_TV_PRODUCT}" STREQUAL "TRUE") SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -DTV_PRODUCT") ENDIF() -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -std=c++11 -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIC -std=c++11 -fvisibility=hidden") SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") ADD_LIBRARY(${PROJECT_NAME} SHARED ${SRCS} ) diff --git a/plugins/wakeup-manager/dependency-default/CMakeLists.txt b/plugins/wakeup-manager/dependency-default/CMakeLists.txt index 163ede8..a08730d 100644 --- a/plugins/wakeup-manager/dependency-default/CMakeLists.txt +++ b/plugins/wakeup-manager/dependency-default/CMakeLists.txt @@ -49,7 +49,7 @@ ENDFOREACH(flag) SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -fPIC -Wall" ) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -std=c++11 -fvisibility=hidden") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fstack-protector-strong -D_FORTIFY_SOURCE=2 -Wl,-z,relro -fPIC -std=c++11 -fvisibility=hidden") SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") ADD_LIBRARY(${PROJECT_NAME} SHARED ${SRCS} ) -- 2.7.4 From dc1e7118f4ca47e7772e334d24de5c84c16a4654 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 16:01:59 +0900 Subject: [PATCH 02/16] Bump version to 0.2.8 Change-Id: I14024040e9cc66a2687f7bba6099f39c2f0dcc8f --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index d18e9c7..3e31401 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 1dca5f5..486ab55 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.7 +Version: 0.2.8 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 905b2b79394c84db6ac5f2051a6702f1e253e97f Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 4 Nov 2019 11:32:56 +0900 Subject: [PATCH 03/16] Add reload feature for updating pluings on-the-fly Change-Id: Ibf873b8861b22807f9b51f5f8e92efb82f8520ba --- .../src/dependency_default_audio.cpp | 36 ++++++- .../wakeup-manager/src/wakeup_audio_manager.cpp | 12 +++ .../wakeup-manager/src/wakeup_engine_manager.cpp | 7 ++ .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 117 +++++++++++++++------ src/multi_assistant_service.c | 54 +++++++--- src/multi_assistant_service_plugin.c | 2 + 6 files changed, 184 insertions(+), 44 deletions(-) diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp index e803739..d1662c8 100644 --- a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp @@ -17,6 +17,7 @@ static mas_proxy_interface g_proxy_interface; using namespace std; static audio_in_h g_audio_in = NULL; +static audio_io_state_e g_audio_io_state = AUDIO_IO_STATE_IDLE; static sound_stream_info_h g_volume_stream = NULL; static virtual_sound_stream_h g_virtual_sound_stream = NULL; @@ -111,6 +112,13 @@ static void _device_connection_changed_cb(sound_device_h device, bool isConnecte return; } +static void _audio_in_state_changed_cb(audio_in_h handle, audio_io_state_e previous, audio_io_state_e current, bool by_policy, void *user_data) +{ + if (handle == g_audio_in) { + g_audio_io_state = current; + } +} + void dependency_default_audio_initialize(mas_proxy_interface interfaces) { const int rate = 16000; @@ -155,12 +163,24 @@ void dependency_default_audio_initialize(mas_proxy_interface interfaces) return; } + ret = audio_in_set_state_changed_cb(g_audio_in, _audio_in_state_changed_cb, NULL); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGE("[Recorder ERROR] Fail to set state changed callback : %d", ret); + sound_manager_remove_device_connection_changed_cb(g_device_id); + sound_manager_destroy_stream_information(g_stream_info); + g_stream_info = NULL; + audio_in_destroy(g_audio_in); + g_audio_in = NULL; + return; + } + ret = audio_in_set_sound_stream_info(g_audio_in, g_stream_info); if (AUDIO_IO_ERROR_NONE != ret) { LOGE("[Recorder ERROR] Fail to set stream info : %d", ret); sound_manager_remove_device_connection_changed_cb(g_device_id); sound_manager_destroy_stream_information(g_stream_info); g_stream_info = NULL; + audio_in_unset_state_changed_cb(g_audio_in); audio_in_destroy(g_audio_in); g_audio_in = NULL; return; @@ -177,6 +197,7 @@ void dependency_default_audio_initialize(mas_proxy_interface interfaces) sound_manager_remove_device_connection_changed_cb(g_device_id); sound_manager_destroy_stream_information(g_stream_info); g_stream_info = NULL; + audio_in_unset_state_changed_cb(g_audio_in); audio_in_destroy(g_audio_in); g_audio_in = NULL; return; @@ -231,6 +252,11 @@ void dependency_default_audio_deinitialize() } if (g_audio_in) { + ret = audio_in_unset_state_changed_cb(g_audio_in); + if (AUDIO_IO_ERROR_NONE != ret) { + LOGD("[Recorder ERROR] Fail to unset state changed callback : %d", ret); + } + ret = audio_in_destroy(g_audio_in); if (AUDIO_IO_ERROR_NONE != ret) { LOGD("[Recorder ERROR] Fail to destroy audio : %d", ret); @@ -271,7 +297,10 @@ static void recorder_thread_func() void dependency_default_audio_start_recording() { if (g_audio_in) { - int ret = audio_in_resume(g_audio_in); + int ret = AUDIO_IO_ERROR_NONE; + if (AUDIO_IO_STATE_RUNNING != g_audio_io_state) { + ret = audio_in_resume(g_audio_in); + } if (AUDIO_IO_ERROR_NONE != ret) { LOGD("[Recorder ERROR] Fail to resume audio in : %d", ret); } @@ -289,7 +318,10 @@ void dependency_default_audio_stop_recording() g_recorder_thread.join(); } if (g_audio_in) { - int ret = audio_in_pause(g_audio_in); + int ret = AUDIO_IO_ERROR_NONE; + if (AUDIO_IO_STATE_RUNNING == g_audio_io_state) { + ret = audio_in_pause(g_audio_in); + } if (AUDIO_IO_ERROR_NONE != ret) { LOGD("[Recorder ERROR] Fail to pause audio in : %d", ret); } diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp index 8b68362..94ce56e 100644 --- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp @@ -68,6 +68,18 @@ int CAudioManager::initialize(void) int CAudioManager::deinitialize(void) { + clear_audio_data(); + if (mStreamingThread.joinable()) { + MWR_LOGD("mStreamingThread is joinable, trying join()"); + mStopStreamingThread.store(true); + try { + mStreamingThread.join(); + } catch (std::exception &e) { + MWR_LOGE("Exception thrown : %s", e.what()); + } + } + mStopStreamingThread.store(false); + sound_manager_remove_focus_state_watch_cb(mSoundFocusWatchId); return 0; diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index 5e1163c..ebc50a9 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -76,6 +76,13 @@ void CWakeupEngineManager::initialize() void CWakeupEngineManager::deinitialize() { + if (mStreamingThread.joinable()) { + MWR_LOGD("mStreamingThread is joinable, trying join()"); + mStopStreamingThread.store(true); + mStreamingThread.join(); + } + mStopStreamingThread.store(false); + for (auto& info : mEngineInfo) { try { if (info.interface.set_wakeup_event_callback) { diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 1c72602..77549bb 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -17,6 +17,8 @@ #include "wakeup_manager_main.h" #include "wakeup_manager.h" +#include + using namespace multiassistant::wakeup; static wakeup_service_wakeup_event_cb g_wakeup_event_cb; @@ -56,14 +58,29 @@ class CSettingValueObserver : public ISettingValueObserver void on_value_changed() override; }; -static CWakeupEventObserver g_wakeup_event_observer; -static CSettingValueObserver g_setting_value_observer; -static CWakeupManager g_wakeup_manager(&g_wakeup_event_observer, &g_setting_value_observer); +static std::unique_ptr g_wakeup_event_observer; +static std::unique_ptr g_setting_value_observer; +static std::unique_ptr g_wakeup_manager; int wakeup_manager_initialize(void) { MWR_LOGD("[ENTER]"); + try { + g_wakeup_event_observer.reset(new CWakeupEventObserver); + g_setting_value_observer.reset(new CSettingValueObserver); + if (NULL == g_wakeup_event_observer || NULL == g_setting_value_observer) { + return -1; + } + g_wakeup_manager.reset(new CWakeupManager(g_wakeup_event_observer.get(), g_setting_value_observer.get())); + if (NULL == g_wakeup_manager) { + return -1; + } + } catch (const std::exception& e) { + MWR_LOGE("Exception caught : %s", e.what()); + return -1; + } + g_wakeup_event_cb = NULL; g_wakeup_event_user_data = NULL; @@ -82,7 +99,7 @@ int wakeup_manager_initialize(void) g_error_cb = NULL; g_error_user_data = NULL; - g_wakeup_manager.initialize(); + g_wakeup_manager->initialize(); MWR_LOGD("[END]"); return 0; @@ -92,7 +109,18 @@ int wakeup_manager_deinitialize(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.deinitialize(); + if (nullptr == g_wakeup_manager) return -1; + + try { + g_wakeup_manager->deinitialize(); + g_wakeup_manager.reset(); + + g_wakeup_event_observer.reset(); + g_setting_value_observer.reset(); + } catch (const std::exception& e) { + MWR_LOGE("Exception caught : %s", e.what()); + return -1; + } MWR_LOGD("[END]"); return 0; @@ -104,7 +132,9 @@ int wakeup_manager_get_settings(ma_plugin_settings **settings, size_t *struct_si MWR_LOGD("[ERROR] Parameter is invalid, settings(%p), struct_size(%p)", settings, struct_size); return -1; } - CWakeupSettings* wakeup_settings = g_wakeup_manager.get_wakeup_settings(); + + if (nullptr == g_wakeup_manager) return -1; + CWakeupSettings* wakeup_settings = g_wakeup_manager->get_wakeup_settings(); if (wakeup_settings) { const int PLUGIN_VERSION = 1; static ma_plugin_settings current_settings; @@ -125,7 +155,9 @@ int wakeup_manager_add_assistant_wakeup_word(const char* appid, const char* wake MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), wakeup_word(%s), language(%s)", appid, wakeup_word, language); return -1; } - g_wakeup_manager.add_assistant_wakeup_word( + + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->add_assistant_wakeup_word( string{appid}, string{wakeup_word}, (language ? string{language} : string{})); MWR_LOGD("[END]"); @@ -141,7 +173,8 @@ int wakeup_manager_add_assistant_language(const char* appid, const char* languag return -1; } - g_wakeup_manager.add_assistant_language(string{appid}, string{language}); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->add_assistant_language(string{appid}, string{language}); MWR_LOGD("[DEBUG] language(%s)", language); @@ -158,8 +191,9 @@ int wakeup_manager_set_assistant_wakeup_engine(const char* appid, const char* en return -1; } + if (nullptr == g_wakeup_manager) return -1; MWR_LOGD("[DEBUG] appid(%s), wakeup engine(%s)", appid, engine); - g_wakeup_manager.set_assistant_wakeup_engine(string{appid}, string{engine}); + g_wakeup_manager->set_assistant_wakeup_engine(string{appid}, string{engine}); MWR_LOGD("[END]"); return 0; @@ -174,8 +208,9 @@ int wakeup_manager_set_default_assistant(const char* appid) return -1; } + if (nullptr == g_wakeup_manager) return -1; MWR_LOGD("[DEBUG] default_assistant appid(%s)", appid); - g_wakeup_manager.set_default_assistant(string{appid}); + g_wakeup_manager->set_default_assistant(string{appid}); MWR_LOGD("[END]"); return 0; @@ -184,7 +219,9 @@ int wakeup_manager_set_default_assistant(const char* appid) int wakeup_manager_get_default_assistant(const char** appid) { static string default_assistant; - default_assistant = g_wakeup_manager.get_default_assistant(); + + if (nullptr == g_wakeup_manager) return -1; + default_assistant = g_wakeup_manager->get_default_assistant(); if (default_assistant.empty()) { *appid = nullptr; @@ -203,7 +240,8 @@ int wakeup_manager_set_language(const char* language) return -1; } - g_wakeup_manager.set_language(string{language}); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_language(string{language}); MWR_LOGD("[END]"); return 0; @@ -211,7 +249,8 @@ int wakeup_manager_set_language(const char* language) int wakeup_manager_change_state(wakeup_manager_state_e state) { - g_wakeup_manager.change_manager_state(state); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->change_manager_state(state); return 0; } @@ -219,7 +258,8 @@ int wakeup_manager_activate(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.activate(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->activate(); MWR_LOGD("[END]"); return 0; @@ -229,7 +269,8 @@ int wakeup_manager_deactivate(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.deactivate(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->deactivate(); MWR_LOGD("[END]"); return 0; @@ -246,7 +287,8 @@ int wakeup_manager_update_voice_feedback_state(const char* appid, int state) appid_string = appid; } - g_wakeup_manager.update_voice_feedback_state(appid_string, state); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->update_voice_feedback_state(appid_string, state); MWR_LOGD("[END]"); return 0; @@ -264,7 +306,8 @@ int wakeup_manager_send_assistant_specific_command(const char* appid, const char if (appid) appid_string = appid; if (command) command_string = command; - g_wakeup_manager.send_assistant_specific_command(appid_string, command_string); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->send_assistant_specific_command(appid_string, command_string); MWR_LOGD("[END]"); return 0; @@ -281,7 +324,8 @@ int wakeup_manager_set_background_volume(const char* appid, double ratio) appid_string = appid; } - g_wakeup_manager.set_background_volume(appid_string, ratio); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_background_volume(appid_string, ratio); MWR_LOGD("[END]"); return 0; @@ -297,7 +341,9 @@ int wakeup_manager_update_recognition_result(const char* appid, int result) } else { appid_string = appid; } - g_wakeup_manager.update_recognition_result(appid_string, result); + + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->update_recognition_result(appid_string, result); MWR_LOGD("[END]"); return 0; @@ -307,7 +353,8 @@ int wakeup_manager_process_event(int event, void* data, int len) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.process_event(static_cast(event), data, len); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->process_event(static_cast(event), data, len); MWR_LOGD("[END]"); return 0; @@ -317,7 +364,8 @@ int wakeup_manager_start_streaming_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.start_streaming_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->start_streaming_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -327,7 +375,8 @@ int wakeup_manager_stop_streaming_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.stop_streaming_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->stop_streaming_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -337,7 +386,8 @@ int wakeup_manager_start_streaming_follow_up_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.start_streaming_follow_up_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->start_streaming_follow_up_data(); MWR_LOGD("[END]"); return 0; @@ -347,7 +397,8 @@ int wakeup_manager_stop_streaming_follow_up_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.stop_streaming_follow_up_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->stop_streaming_follow_up_data(); MWR_LOGD("[END]"); return 0; @@ -357,7 +408,8 @@ int wakeup_manager_start_streaming_previous_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.start_streaming_previous_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->start_streaming_previous_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -367,7 +419,8 @@ int wakeup_manager_stop_streaming_previous_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_wakeup_manager.stop_streaming_previous_utterance_data(); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->stop_streaming_previous_utterance_data(); MWR_LOGD("[END]"); return 0; @@ -382,7 +435,8 @@ int wakeup_manager_get_audio_format(int* rate, int* channel, int* audio_type) return -1; } - g_wakeup_manager.get_audio_format(rate, channel, audio_type); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->get_audio_format(rate, channel, audio_type); MWR_LOGD("[END] rate(%d), channel(%d), audio_type(%d)", *rate, *channel, *audio_type); return 0; @@ -397,7 +451,8 @@ int wakeup_manager_get_audio_source_type(char** type) return -1; } - g_wakeup_manager.get_audio_source_type(type); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->get_audio_source_type(type); MWR_LOGD("[END] type(%s)", *type); return 0; @@ -407,7 +462,8 @@ int wakeup_manager_set_wake_word_audio_require_flag(bool require) { MWR_LOGD("[ENTER] : %d", require); - g_wakeup_manager.set_wake_word_audio_require_flag(require); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_wake_word_audio_require_flag(require); MWR_LOGD("[END]"); return 0; @@ -543,7 +599,8 @@ int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streami int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len) { - g_wakeup_manager.feed_audio_data(event, buffer, len); + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->feed_audio_data(event, buffer, len); return 0; } diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 35cc1ed..137c7de 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -677,15 +677,8 @@ static void mas_active_state_changed_cb(keynode_t* key, void* data) } } -static int init_wakeup(void) +static int init_plugin(void) { - MAS_LOGD("[Enter] init_wakeup"); - - int ret = mas_dbus_open_connection(); - if (0 != ret) { - MAS_LOGE("[ERROR] Fail to open connection"); - } - if (0 != multi_assistant_service_plugin_initialize()) { MAS_LOGE("Fail to ws intialize"); return -1; @@ -740,6 +733,18 @@ static int init_wakeup(void) return -1; } + return 0; +} + +static int deinit_plugin(void) +{ + if (0 != multi_assistant_service_plugin_deinitialize()) { + MAS_LOGE("Fail to deinitialize"); + } +} + +static int process_multi_assistant_activated_setting() +{ if (0 == vconf_notify_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb, NULL)) { /* Activate / deactivate according to the vconf key setting */ mas_active_state_changed_cb(NULL, NULL); @@ -761,6 +766,21 @@ static int init_wakeup(void) } #endif } + return 0; +} + +static int init_wakeup(void) +{ + MAS_LOGD("[Enter] init_wakeup"); + + int ret = mas_dbus_open_connection(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to open connection"); + } + + init_plugin(); + + process_multi_assistant_activated_setting(); mas_prelaunch_default_assistant(); @@ -786,6 +806,7 @@ static void deinit_wakeup(void) g_current_lang = NULL; } */ + deinit_plugin(); vconf_ignore_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb); @@ -793,10 +814,6 @@ static void deinit_wakeup(void) if (0 != ret) { MAS_LOGE("[ERROR] Fail to close connection"); } - - if (0 != multi_assistant_service_plugin_deinitialize()) { - MAS_LOGE("Fail to ws deinitialize"); - } } int mas_get_current_client_pid() @@ -1246,6 +1263,19 @@ void service_app_terminate(void *data) void service_app_control(app_control_h app_control, void *data) { // Todo: add your code here. + char *operation = NULL; + int ret = app_control_get_operation(app_control, &operation); + if (ret == APP_CONTROL_ERROR_NONE && operation) { + //strncmp(operation, APP_CONTROL_OPERATION_LAUNCH_ON_EVENT, strlen(operation)) == 0){ + char* value = NULL; + app_control_get_extra_data(app_control, "event", &value); + LOGD("operation : %s , %s", operation, value); + if (value && 0 == strcmp(value, "reload")) { + deinit_plugin(); + init_plugin(); + process_multi_assistant_activated_setting(); + } + } return; } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 8307104..b9f5f7a 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -511,6 +511,8 @@ int multi_assistant_service_plugin_deinitialize(void) MAS_LOGD("[Recorder ERROR] File not found!"); } #endif + multi_assistant_service_plugin_deactivate(); + int ret = -1; if (NULL != g_handle) { wakeup_manager_deinitialize func = _wakeup_manager_interface.deinitialize; -- 2.7.4 From 1393f5ccc126eee00404c22dbacadff58b93f9c4 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 19:07:56 +0900 Subject: [PATCH 04/16] Reload plugin modules depending on pkgmgr event Change-Id: I5c12c6600b80f2657e371b15c4eeac13c0044722 --- CMakeLists.txt | 1 + src/multi_assistant_service.c | 151 +++++++++++++++++++++++++++++++---- src/multi_assistant_service_plugin.c | 1 - 3 files changed, 137 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1cf3c04..1717bcf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ pkg_check_modules(pkgs REQUIRED libxml-2.0 multi-assistant vconf + pkgmgr-info ) # SET FLAG --------------------------------------------------------------------------------------- diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 137c7de..fc2f3cb 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -70,6 +72,7 @@ typedef struct { static int g_current_maclient_info = 0; static int g_current_preprocessing_maclient_info = -1; static const char *g_wakeup_maclient_appid = NULL; +static package_manager_h g_pkgmgr = NULL; static PREPROCESSING_STATE g_current_preprocessing_state = PREPROCESSING_STATE_NONE; @@ -738,12 +741,15 @@ static int init_plugin(void) static int deinit_plugin(void) { + if (0 != multi_assistant_service_plugin_deactivate()) { + MAS_LOGE("Fail to deactivate"); + } if (0 != multi_assistant_service_plugin_deinitialize()) { MAS_LOGE("Fail to deinitialize"); } } -static int process_multi_assistant_activated_setting() +static int process_activated_setting() { if (0 == vconf_notify_key_changed(MULTI_ASSISTANT_SETTINGS_ACTIVATED, mas_active_state_changed_cb, NULL)) { /* Activate / deactivate according to the vconf key setting */ @@ -780,7 +786,7 @@ static int init_wakeup(void) init_plugin(); - process_multi_assistant_activated_setting(); + process_activated_setting(); mas_prelaunch_default_assistant(); @@ -1239,6 +1245,112 @@ int mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT event) return 0; } +static int pkg_app_list_cb(const pkgmgrinfo_appinfo_h handle, void *user_data) +{ + char *appid = NULL; + + int ret = pkgmgrinfo_appinfo_get_appid (handle, &appid); + if (PMINFO_R_OK == ret && NULL != appid) { + int *result = (int*)user_data; + if (result) { + for (int loop = 0; loop < MAX_MACLIENT_INFO_NUM;loop++) { + if (g_maclient_info[loop].used) { + LOGD("comparing appid : %s %s", g_maclient_info[loop].wakeup_engine, appid); + if (0 == strncmp(g_maclient_info[loop].wakeup_engine, appid, MAX_APPID_LEN)) { + *result = 1; + } + } + } + } + } else { + LOGE("pkgmgrinfo_appinfo_get_appid failed! error code=%d", ret); + return 0; + } + + return 0; +} + +/* +INFO: Package install/update/uninstall scenario +Install and Uninstall are obviously simple. + Install: just INSTALL + Uninstall: just UNINSTALL +Update package (change the source codes and Run As again), there are four scenarios: +1. UPDATE + Source code change +2. UNINSTALL -> INSTALL + This happens when Tizen IDE Property > Tizen SDK > Rapid Development Support > Check "Enable Project specific settings" + and change Application ID in tizen-manifest.xml file and Run As. +3. UPDATE -> INSTALL + This happens when Tizen IDE Property > Tizen SDK > Rapid Development Support > Uncheck "Enable Project specific settings" + and change Application ID in tizen-manifest.xml file and Run As. + At UPDATE event, pkgid (package parameter) is invalid... +4. UPDATE + Exceptionally, only UPDATE can be called when Application ID in tizen-manifest.xml file is changed. + At UPDATE event, pkgid (package parameter) is valid, and only appid is changed; the previous appid is invalid. +*/ +static void _package_manager_event_cb(const char *type, const char *package, package_manager_event_type_e event_type, package_manager_event_state_e event_state, int progress, package_manager_error_e error, void *user_data) +{ + int ret = 0; + uid_t uid = getuid (); + pkgmgrinfo_pkginfo_h handle = NULL; + static bool in_progress = false; + + if (!package || !type) + return; + + if (PACKAGE_MANAGER_EVENT_TYPE_UPDATE != event_type && + PACKAGE_MANAGER_EVENT_TYPE_INSTALL != event_type && + PACKAGE_MANAGER_EVENT_TYPE_UNINSTALL != event_type) + return; + + if (PACKAGE_MANAGER_EVENT_STATE_STARTED != event_state && + PACKAGE_MANAGER_EVENT_STATE_COMPLETED != event_state) + return; + + bool user = false; + LOGD("type=%s package=%s event_type=%d event_state=%d progress=%d error=%d", + type, package, event_type, event_state, progress, error); + ret = pkgmgrinfo_pkginfo_get_pkginfo(package, &handle); + if (ret != PMINFO_R_OK || NULL == handle) { + LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); + /* Try to get in user packages */ + ret = pkgmgrinfo_pkginfo_get_usr_pkginfo (package, uid, &handle); + if (ret != PMINFO_R_OK || NULL == handle) { + LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); + return; + } + } + + if (user) { + /* Try to get in user packages */ + pkgmgrinfo_appinfo_get_usr_list(handle, PMINFO_ALL_APP, pkg_app_list_cb, (void *)&ret, uid); + } else { + /* Try to get in global packages */ + pkgmgrinfo_appinfo_get_list(handle, PMINFO_ALL_APP, pkg_app_list_cb, (void *)&ret); + } + if (1 == ret) { + if (PACKAGE_MANAGER_EVENT_STATE_STARTED == event_state) { + LOGD("processing PACKAGE_MANAGER_EVENT_STATE_STARTED event"); + if (false == in_progress) { + in_progress = true; + deinit_plugin(); + } + } else if (PACKAGE_MANAGER_EVENT_STATE_COMPLETED == event_state) { + LOGD("processing PACKAGE_MANAGER_EVENT_STATE_COMPLETED event"); + if (true == in_progress) { + init_plugin(); + process_activated_setting(); + in_progress = false; + } + } + } + + pkgmgrinfo_pkginfo_destroy_pkginfo(handle); + + return; +} + bool service_app_create(void *data) { // Todo: add your code here. @@ -1250,11 +1362,33 @@ bool service_app_create(void *data) return false; } + if (!g_pkgmgr) { + int ret = package_manager_create(&g_pkgmgr); + if (ret == PACKAGE_MANAGER_ERROR_NONE) { + ret = package_manager_set_event_cb(g_pkgmgr, _package_manager_event_cb, NULL); + if (ret == PACKAGE_MANAGER_ERROR_NONE) { + LOGD("package_manager_set_event_cb succeeded."); + } + else { + LOGE("package_manager_set_event_cb failed(%d)", ret); + } + } + else { + LOGE("package_manager_create failed(%d)", ret); + } + } + return true; } void service_app_terminate(void *data) { + if (g_pkgmgr) { + package_manager_unset_event_cb(g_pkgmgr); + package_manager_destroy(g_pkgmgr); + g_pkgmgr = NULL; + } + // Todo: add your code here. deinit_wakeup(); return; @@ -1263,19 +1397,6 @@ void service_app_terminate(void *data) void service_app_control(app_control_h app_control, void *data) { // Todo: add your code here. - char *operation = NULL; - int ret = app_control_get_operation(app_control, &operation); - if (ret == APP_CONTROL_ERROR_NONE && operation) { - //strncmp(operation, APP_CONTROL_OPERATION_LAUNCH_ON_EVENT, strlen(operation)) == 0){ - char* value = NULL; - app_control_get_extra_data(app_control, "event", &value); - LOGD("operation : %s , %s", operation, value); - if (value && 0 == strcmp(value, "reload")) { - deinit_plugin(); - init_plugin(); - process_multi_assistant_activated_setting(); - } - } return; } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index b9f5f7a..e62a25f 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -511,7 +511,6 @@ int multi_assistant_service_plugin_deinitialize(void) MAS_LOGD("[Recorder ERROR] File not found!"); } #endif - multi_assistant_service_plugin_deactivate(); int ret = -1; if (NULL != g_handle) { -- 2.7.4 From eb1c183dc66625bcb50ef700045b3044ca54cdf0 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 8 Nov 2019 15:02:09 +0900 Subject: [PATCH 05/16] Fix bug recording_session is not changed properly Change-Id: Ia2ad3c215cb6bb2b2f65872122d3c4b8aff33115 --- plugins/wakeup-manager/src/wakeup_manager.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index ece33b9..1fa4387 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -422,6 +422,9 @@ bool CWakeupManager::set_background_volume(string appid, double ratio) bool CWakeupManager::update_recognition_result(string appid, int result) { MWR_LOGD("[ENTER]"); + if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { + mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD); + } mWakeupEngineManager.update_recognition_result(appid, result); if (WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState || WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { @@ -893,20 +896,24 @@ bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section( void CWakeupManager::CPolicyEventObserver::on_wakeup(wakeup_event_info wakeup_info) { if (nullptr == mWakeupManager) return; + + CAudioManager *audio_manager = mWakeupManager->get_audio_manager(); + CWakeupEngineManager *engine_manager = mWakeupManager->get_engine_manager(); + if (nullptr == audio_manager || nullptr == engine_manager) return; + mWakeupManager->stop_streaming_utterance_data(); mWakeupManager->stop_streaming_previous_utterance_data(); mWakeupManager->stop_streaming_follow_up_data(); mWakeupManager->change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE); - CWakeupEngineManager *engine_manager = mWakeupManager->get_engine_manager(); - if (nullptr == engine_manager) return; - mWakeupManager->set_last_wakeup_event_info(wakeup_info); engine_manager->set_selected_wakeup_info(wakeup_info); vector observers = mWakeupManager->get_wakeup_observers(); for (const auto& observer : observers) { observer->on_wakeup(wakeup_info); } + + audio_manager->set_recording_session(RECORDING_SESSION_UTTERANCE); } bool CWakeupManager::CAudioEventObserver::on_recording_audio_data(long time, void* data, int len) -- 2.7.4 From dca681cbde530a626a227afe1a004ce75f3a0b43 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 7 Nov 2019 19:16:30 +0900 Subject: [PATCH 06/16] Bump version to 0.2.9 Change-Id: I1febcfbce71db18e29d69652af547d2910518ec8 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 3e31401..4dc2a2c 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 486ab55..039a724 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.8 +Version: 0.2.9 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From d2da0780b6b6ab297f28db11816f24ae3e29301d Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 13 Nov 2019 19:56:26 +0900 Subject: [PATCH 07/16] Add support for setting assistant language Change-Id: Id7d6e6c48541088ec7004fcc2db27c3d473afdb3 --- inc/multi_assistant_main.h | 1 + inc/multi_assistant_service_plugin.h | 3 +++ plugins/wakeup-manager/inc/wakeup_engine_manager.h | 1 + plugins/wakeup-manager/inc/wakeup_manager.h | 1 + .../wakeup-manager/inc/wakeup_manager_wrapper.h | 2 ++ .../wakeup-manager/src/wakeup_engine_manager.cpp | 23 ++++++++++++++++ plugins/wakeup-manager/src/wakeup_manager.cpp | 10 +++++++ .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 20 ++++++++++++++ src/multi_assistant_dbus.c | 3 +++ src/multi_assistant_dbus_server.c | 31 ++++++++++++++++++++++ src/multi_assistant_service.c | 14 ++++++++++ src/multi_assistant_service_plugin.c | 22 +++++++++++++++ 12 files changed, 131 insertions(+) diff --git a/inc/multi_assistant_main.h b/inc/multi_assistant_main.h index 6c04aa6..8acd240 100644 --- a/inc/multi_assistant_main.h +++ b/inc/multi_assistant_main.h @@ -66,6 +66,7 @@ #define MA_METHOD_SET_PREPROCESSING_ALLOW_MODE "ma_method_set_preprocessing_allow_mode" #define MA_METHOD_SEND_PREPROCESSING_RESULT "ma_method_send_preprocessing_result" #define MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "ma_method_set_wake_word_audio_require_flag" +#define MA_METHOD_SET_ASSISTANT_LANGUAGE "ma_method_set_assistant_language" #define MA_METHOD_ERROR "ma_method_error" #define MA_UI_METHOD_INITIALIZE "ma_ui_method_initialize" diff --git a/inc/multi_assistant_service_plugin.h b/inc/multi_assistant_service_plugin.h index dbfd673..48a9233 100644 --- a/inc/multi_assistant_service_plugin.h +++ b/inc/multi_assistant_service_plugin.h @@ -159,6 +159,8 @@ typedef int (*wakeup_manager_get_audio_format)(int* rate, int* channel, int* aud typedef int (*wakeup_manager_get_audio_source_type)(char** type); #define MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "wakeup_manager_set_wake_word_audio_require_flag" typedef int (*wakeup_manager_set_wake_word_audio_require_flag)(bool require); +#define MA_WAKEUP_MANAGER_FUNC_SET_ASSISTANT_LANGUAGE "wakeup_manager_set_assistant_language" +typedef int (*wakeup_manager_set_assistant_language)(const char* appid, const char* language); #define MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_manager_set_wakeup_event_callback" typedef int (*wakeup_manager_set_wakeup_event_callback)(wakeup_service_wakeup_event_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_UTTERANCE_STREAMING_CALLBACK "wakeup_manager_set_utterance_streaming_callback" @@ -202,6 +204,7 @@ typedef struct { wakeup_manager_get_audio_format get_audio_format; wakeup_manager_get_audio_source_type get_audio_source_type; wakeup_manager_set_wake_word_audio_require_flag set_wake_word_audio_require_flag; + wakeup_manager_set_assistant_language set_assistant_language; wakeup_manager_set_wakeup_event_callback set_wakeup_event_callback; wakeup_manager_set_utterance_streaming_callback set_utterance_streaming_callback; wakeup_manager_set_previous_utterance_streaming_callback set_previous_utterance_streaming_callback; diff --git a/plugins/wakeup-manager/inc/wakeup_engine_manager.h b/plugins/wakeup-manager/inc/wakeup_engine_manager.h index c768f9b..d9d478c 100644 --- a/plugins/wakeup-manager/inc/wakeup_engine_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_engine_manager.h @@ -149,6 +149,7 @@ public: void set_selected_wakeup_info(wakeup_event_info wakeup_info); bool set_language(string language); + bool set_assistant_language(string appid, string language); void set_assistant_activated(string appid, bool activated); bool get_assistant_activated(string appid); void set_wake_word_audio_require_flag(bool require); diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index d574646..c6623a2 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -84,6 +84,7 @@ public: bool add_assistant_language(string appid, string language); bool add_assistant_wakeup_word(string appid, string wakeup_word, string language); bool set_assistant_wakeup_engine(string appid, string engine); + bool set_assistant_language(string appid, string language); bool set_assistant_enabled(string appid, bool enabled); bool get_assistant_enabled(string appid); diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index 2df503e..d6d20a5 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -138,6 +138,8 @@ EXPORT_API int wakeup_manager_get_audio_source_type(char** type); EXPORT_API int wakeup_manager_set_wake_word_audio_require_flag(bool require); +EXPORT_API int wakeup_manager_set_assistant_language(const char* appid, const char *language); + EXPORT_API int wakeup_manager_set_wakeup_event_callback(wakeup_service_wakeup_event_cb callback, void* user_data); EXPORT_API int wakeup_manager_set_utterance_streaming_callback(wakeup_service_speech_streaming_cb callback, void* user_data); diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index ebc50a9..cef7b6d 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -161,6 +161,29 @@ bool CWakeupEngineManager::set_language(string language) return true; } +bool CWakeupEngineManager::set_assistant_language(string appid, string language) +{ + for (auto& info : mEngineInfo) { + const auto& iter = find_if(info.assistant_list.begin(), info.assistant_list.end(), + [appid](const string& assistant) { + return (0 == assistant.compare(appid)); + }); + + /* If the appid is in the assistant list */ + if (info.assistant_list.end() != iter) { + try { + int ret = info.interface.set_language(language.c_str()); + MWR_LOGD("set_language returned %d : %s %s %s", + ret, appid.c_str(), info.engine_name.c_str(), language.c_str()); + } catch (const std::exception& e) { + MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", + info.engine_name.c_str(), e.what()); + } + } + } + return true; +} + void CWakeupEngineManager::set_assistant_activated(string appid, bool activated) { MWR_LOGD("[ENTER] : %s %d", appid.c_str(), activated); diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 1fa4387..a6a6281 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -249,6 +249,16 @@ bool CWakeupManager::set_assistant_wakeup_engine(string appid, string engine) return true; } +bool CWakeupManager::set_assistant_language(string appid, string language) +{ + MWR_LOGD("[ENTER] : %s, %s", appid.c_str(), language.c_str()); + + mWakeupEngineManager.set_assistant_language(appid, language); + + MWR_LOGD("[END]"); + return true; +} + bool CWakeupManager::set_assistant_enabled(string appid, bool enabled) { MWR_LOGD("[ENTER]"); diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 77549bb..49b24eb 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -469,6 +469,26 @@ int wakeup_manager_set_wake_word_audio_require_flag(bool require) return 0; } +int wakeup_manager_set_assistant_language(const char* appid, const char* language) +{ + MWR_LOGD("[ENTER]"); + + string appid_string; + string language_string; + if (NULL == appid || NULL == language) { + MWR_LOGE("[ERROR] Parameter is invalid, appid(%s), language(%s)", appid, language); + return -1; + } + appid_string = appid; + language_string = language; + + if (nullptr == g_wakeup_manager) return -1; + g_wakeup_manager->set_assistant_language(appid_string, language_string); + + MWR_LOGD("[END]"); + return 0; +} + int wakeup_manager_set_wakeup_event_callback(wakeup_service_wakeup_event_cb callback, void* user_data) { MWR_LOGD("[ENTER]"); diff --git a/src/multi_assistant_dbus.c b/src/multi_assistant_dbus.c index ac92dba..53130d8 100644 --- a/src/multi_assistant_dbus.c +++ b/src/multi_assistant_dbus.c @@ -875,6 +875,9 @@ static Eina_Bool listener_event_callback(void* data, Ecore_Fd_Handler *fd_handle } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG)) { ma_service_dbus_set_wake_word_audio_require_flag(g_conn_listener, msg); + } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_SET_ASSISTANT_LANGUAGE)) { + ma_service_dbus_set_assistant_language(g_conn_listener, msg); + } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_UI_METHOD_INITIALIZE)) { ma_service_ui_dbus_initialize(g_conn_listener, msg); diff --git a/src/multi_assistant_dbus_server.c b/src/multi_assistant_dbus_server.c index 0d00f3e..40c9e13 100644 --- a/src/multi_assistant_dbus_server.c +++ b/src/multi_assistant_dbus_server.c @@ -663,6 +663,37 @@ int ma_service_dbus_set_wake_word_audio_require_flag(DBusConnection* conn, DBusM return 0; } +int ma_service_dbus_set_assistant_language(DBusConnection* conn, DBusMessage* msg) +{ + DBusError err; + dbus_error_init(&err); + + int pid; + int ret = 0; + char* language; + + dbus_message_get_args(msg, &err, + DBUS_TYPE_INT32, &pid, + DBUS_TYPE_STRING, &language, + DBUS_TYPE_INVALID); + + MAS_LOGD("[DEBUG] MAS SET ASSISTANT LANGUAGE"); + + if (dbus_error_is_set(&err)) { + MAS_LOGE("[IN ERROR] mas set assistant language : Get arguments error (%s)", err.message); + dbus_error_free(&err); + ret = -1; //MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD("[IN] mas set assistant language : pid(%d), language(%s)", pid, language); + ret = mas_client_set_assistant_language(pid, language); + } + + MAS_LOGD("<<<<<"); + MAS_LOGD(" "); + + return 0; +} + int ma_service_ui_dbus_initialize(DBusConnection* conn, DBusMessage* msg) { DBusError err; diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index fc2f3cb..94b2fb4 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -533,6 +533,20 @@ int mas_client_set_wake_word_audio_require_flag(int pid, bool require) return 0; } +int mas_client_set_assistant_language(int pid, const char* language) +{ + const char* pid_appid = NULL; + char buf[MAX_APPID_LEN] = {'\0',}; + int ret = aul_app_get_appid_bypid(pid, buf, sizeof(buf)); + if (AUL_R_OK == ret) { + buf[MAX_APPID_LEN - 1] = '\0'; + pid_appid = buf; + } + + multi_assistant_service_plugin_assistant_language(pid_appid, language); + return 0; +} + int mas_ui_client_initialize(int pid) { MAS_LOGD("[Enter] pid(%d)", pid); diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index e62a25f..e7640f6 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -445,6 +445,9 @@ int multi_assistant_service_plugin_initialize(void) _wakeup_manager_interface.set_wake_word_audio_require_flag = (wakeup_manager_set_wake_word_audio_require_flag)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG); + _wakeup_manager_interface.set_assistant_language = + (wakeup_manager_set_assistant_language)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_SET_ASSISTANT_LANGUAGE); _wakeup_manager_interface.set_wakeup_event_callback = (wakeup_manager_set_wakeup_event_callback)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_EVENT_CALLBACK); @@ -974,6 +977,25 @@ int multi_assistant_service_plugin_set_wake_word_audio_require_flag(const char* return ret; } +int multi_assistant_service_plugin_assistant_language(const char* appid, const char* language) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_set_assistant_language func = _wakeup_manager_interface.set_assistant_language; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_ASSISTANT_LANGUAGE); + } else { + ret = func(appid, language); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set assistant language, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + return ret; +} + int multi_assistant_service_plugin_set_callbacks(void) { int ret = multi_assistant_service_plugin_set_wakeup_event_callback(__wakeup_event_cb, NULL); -- 2.7.4 From a329e2482b363b5986482a876cddbcf83ab8f16d Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 18 Nov 2019 11:19:36 +0900 Subject: [PATCH 08/16] Fix defects detected by static analysis tool Change-Id: Ic5c1e6a0068ad243476f62b016131a9e0c5b9e1f --- src/multi_assistant_service.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 94b2fb4..96ff563 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -761,6 +761,7 @@ static int deinit_plugin(void) if (0 != multi_assistant_service_plugin_deinitialize()) { MAS_LOGE("Fail to deinitialize"); } + return 0; } static int process_activated_setting() @@ -1329,6 +1330,7 @@ static void _package_manager_event_cb(const char *type, const char *package, pac if (ret != PMINFO_R_OK || NULL == handle) { LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); /* Try to get in user packages */ + user = true; ret = pkgmgrinfo_pkginfo_get_usr_pkginfo (package, uid, &handle); if (ret != PMINFO_R_OK || NULL == handle) { LOGW("Failed to call pkgmgrinfo_pkginfo_get_pkginfo & get_usr_pkginfo(\"%s\",~) returned %d, uid : %d", package, ret, getuid ()); -- 2.7.4 From 301719685888589864b56ba4e52ba0eda987d7ee Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 19 Nov 2019 11:41:42 +0900 Subject: [PATCH 09/16] Add logs for tracking proper destruction Change-Id: If35c356ee2c1c5849d0b82d1a470076fc45cb370 --- plugins/wakeup-manager/src/wakeup_engine_manager.cpp | 11 +++++++++-- plugins/wakeup-manager/src/wakeup_manager.cpp | 5 +++-- plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp | 4 ++-- src/multi_assistant_service.c | 11 +++++++++-- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index cef7b6d..896057f 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -27,6 +27,7 @@ CWakeupEngineManager::CWakeupEngineManager() CWakeupEngineManager::~CWakeupEngineManager() { + MWR_LOGI("Wakeup Engine Manager is now being destroyed"); } CWakeupEngineManager::CWakeupEngineManager(IEngineEventObserver *observer) : CWakeupEngineManager() @@ -76,6 +77,7 @@ void CWakeupEngineManager::initialize() void CWakeupEngineManager::deinitialize() { + MWR_LOGI("[START]"); if (mStreamingThread.joinable()) { MWR_LOGD("mStreamingThread is joinable, trying join()"); mStopStreamingThread.store(true); @@ -97,20 +99,25 @@ void CWakeupEngineManager::deinitialize() if (info.interface.set_audio_data_require_status_callback) { info.interface.set_audio_data_require_status_callback(nullptr, nullptr); } + MWR_LOGI("Trying to deinitialize engine : %s", info.engine_name.c_str()); if (info.interface.deinitialize) { - info.interface.deinitialize(); + int ret = info.interface.deinitialize(); + MWR_LOGI("Deinitialization [%s] returned %d", info.engine_name.c_str(), ret); } } catch (const std::exception& e) { MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s", info.engine_name.c_str(), e.what()); } if (info.engine_handle) { - dlclose(info.engine_handle); + int ret = dlclose(info.engine_handle); + MWR_LOGI("Closing [%s] returned %d, [%s]", info.engine_name.c_str(), ret, + ((0 == ret) ? "" : dlerror())); info.engine_handle = nullptr; } } mSelectedEngine = nullptr; mEngineInfo.clear(); + MWR_LOGI("[END]"); } void CWakeupEngineManager::subscribe(IEngineEventObserver *observer) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index a6a6281..75fa067 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -68,6 +68,7 @@ CWakeupManager::CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingVa CWakeupManager::~CWakeupManager() { + MWR_LOGI("Wakeup Manager is now being destroyed"); } void CWakeupManager::initialize_wakeup_policy() @@ -122,7 +123,7 @@ bool CWakeupManager::initialize() bool CWakeupManager::deinitialize() { - MWR_LOGD("[ENTER]"); + MWR_LOGI("[ENTER]"); dependency_resolver_deinitialize(); @@ -135,7 +136,7 @@ bool CWakeupManager::deinitialize() mWakeupSettings.deinitialize(); mAssistantLanguageInfo.clear(); - MWR_LOGD("[END]"); + MWR_LOGI("[END]"); return true; } diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 49b24eb..0d30121 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -107,7 +107,7 @@ int wakeup_manager_initialize(void) int wakeup_manager_deinitialize(void) { - MWR_LOGD("[ENTER]"); + MWR_LOGI("[ENTER]"); if (nullptr == g_wakeup_manager) return -1; @@ -122,7 +122,7 @@ int wakeup_manager_deinitialize(void) return -1; } - MWR_LOGD("[END]"); + MWR_LOGI("[END]"); return 0; } diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 96ff563..8c1101f 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -755,12 +755,14 @@ static int init_plugin(void) static int deinit_plugin(void) { + MAS_LOGI("[ENTER]"); if (0 != multi_assistant_service_plugin_deactivate()) { MAS_LOGE("Fail to deactivate"); } if (0 != multi_assistant_service_plugin_deinitialize()) { MAS_LOGE("Fail to deinitialize"); } + MAS_LOGI("[END]"); return 0; } @@ -820,7 +822,7 @@ static int init_wakeup(void) static void deinit_wakeup(void) { - MAS_LOGD("[Enter] deinit_wakeup "); + MAS_LOGI("[Enter] deinit_wakeup"); /* if (NULL != g_current_lang) { free(g_current_lang); @@ -835,6 +837,7 @@ static void deinit_wakeup(void) if (0 != ret) { MAS_LOGE("[ERROR] Fail to close connection"); } + MAS_LOGI("[END]"); } int mas_get_current_client_pid() @@ -1399,6 +1402,7 @@ bool service_app_create(void *data) void service_app_terminate(void *data) { + MAS_LOGI("[ENTER]"); if (g_pkgmgr) { package_manager_unset_event_cb(g_pkgmgr); package_manager_destroy(g_pkgmgr); @@ -1407,6 +1411,7 @@ void service_app_terminate(void *data) // Todo: add your code here. deinit_wakeup(); + MAS_LOGI("[END]"); return; } @@ -1456,5 +1461,7 @@ int main(int argc, char* argv[]) service_app_add_event_handler(&handlers[APP_EVENT_LANGUAGE_CHANGED], APP_EVENT_LANGUAGE_CHANGED, service_app_lang_changed, &ad); service_app_add_event_handler(&handlers[APP_EVENT_REGION_FORMAT_CHANGED], APP_EVENT_REGION_FORMAT_CHANGED, service_app_region_changed, &ad); - return service_app_main(argc, argv, &event_callback, ad); + int ret = service_app_main(argc, argv, &event_callback, ad); + LOGI("Main function exits with : %d", ret); + return ret; } -- 2.7.4 From 30582d1b78a1021aab437eef6a2636524fe21c85 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 19 Nov 2019 15:08:06 +0900 Subject: [PATCH 10/16] Bump version to 0.2.10 Change-Id: Icd0479f15c231519edb6fcf0aa2bbf2d0630a821 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 4dc2a2c..adb0d85 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 039a724..b7c668a 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.9 +Version: 0.2.10 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 6a8832bf758b34175c4ff8b05f520978afcf2e64 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 19 Nov 2019 19:02:40 +0900 Subject: [PATCH 11/16] Set WAKE_WORD recording session on stop streaming request Change-Id: I6f74d289a518272bc080b1135f0e5cfb9ca752f7 --- plugins/wakeup-manager/src/wakeup_manager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 75fa067..84327d0 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -633,6 +633,7 @@ bool CWakeupManager::stop_streaming_utterance_data() } if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING); + mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD); } mAudioManager.stop_streaming_current_utterance_data(); mWakeupEngineManager.stop_streaming_current_utterance_data(); -- 2.7.4 From 7527a62d2d10e01349894c8102f9bfc57537de42 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 20 Nov 2019 17:26:11 +0900 Subject: [PATCH 12/16] Bump version to 0.2.11 Change-Id: Icbd7638feb6b776411a2985fe51c2816ae7dcaed --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index adb0d85..0aa3e02 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index b7c668a..a87311f 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.10 +Version: 0.2.11 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 06fbd080ad64d24b5c498cc0a9ef27d544240dbc Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 15 Nov 2019 15:38:32 +0900 Subject: [PATCH 13/16] Restart recording on follow-up streaming timeout Change-Id: I45f8972425b8fcc50bcb6cb8e75620a56dbc6494 --- plugins/wakeup-manager/src/wakeup_manager.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 84327d0..1b89605 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -569,6 +569,10 @@ static Eina_Bool streaming_duration_expired(void *data) break; case STREAMING_MODE::FOLLOW_UP: audio_manager->stop_streaming_follow_up_data(); + audio_manager->stop_recording(true); + audio_manager->clear_audio_data(); + audio_manager->set_recording_session(RECORDING_SESSION_WAKE_WORD); + audio_manager->start_recording(true); break; } -- 2.7.4 From 237f1633d24938be459d452763eaec6b0290af8b Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 21 Nov 2019 19:21:52 +0900 Subject: [PATCH 14/16] Add support for wakeup engine command transmission Change-Id: I0588f24a03d888e15c952cd09bba60472f8e2074 --- inc/multi_assistant_main.h | 1 + inc/multi_assistant_service_plugin.h | 3 + inc/multi_wakeup_recognizer.h | 2 + plugins/wakeup-manager/inc/wakeup_engine_manager.h | 5 ++ plugins/wakeup-manager/inc/wakeup_manager.h | 2 + .../wakeup-manager/inc/wakeup_manager_wrapper.h | 4 ++ .../wakeup-manager/src/wakeup_engine_manager.cpp | 42 ++++++++++++++ plugins/wakeup-manager/src/wakeup_manager.cpp | 13 +++++ .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 35 ++++++++++++ src/multi_assistant_dbus.c | 65 ++++++++++++++++++++++ src/multi_assistant_service_plugin.c | 39 +++++++++++++ 11 files changed, 211 insertions(+) diff --git a/inc/multi_assistant_main.h b/inc/multi_assistant_main.h index 8acd240..3e2b2c1 100644 --- a/inc/multi_assistant_main.h +++ b/inc/multi_assistant_main.h @@ -81,6 +81,7 @@ #define MAS_METHOD_SEND_PREPROCESSING_INFORMATION "mas_method_send_preprocessing_information" #define MAS_METHOD_AUDIO_STREAMING_DATA_SECTION "mas_method_audio_streaming_data_section" #define MAS_METHOD_SEND_PREPROCESSING_RESULT "mas_method_send_preprocessing_result" +#define MAS_METHOD_SEND_WAKEUP_ENGINE_COMMAND "mas_method_send_wakeup_engine_command" #define MAS_UI_METHOD_SEND_ASR_RESULT "mas_ui_method_send_asr_result" #define MAS_UI_METHOD_SEND_RESULT "mas_ui_method_send_result" diff --git a/inc/multi_assistant_service_plugin.h b/inc/multi_assistant_service_plugin.h index 48a9233..b225b29 100644 --- a/inc/multi_assistant_service_plugin.h +++ b/inc/multi_assistant_service_plugin.h @@ -177,6 +177,8 @@ typedef int (*wakeup_manager_set_setting_changed_callback)(wakeup_service_settin typedef int (*wakeup_manager_set_error_callback)(wakeup_service_error_cb callback, void* user_data); #define MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK "wakeup_manager_set_streaming_section_changed_callback" typedef int (*wakeup_manager_set_streaming_section_changed_callback)(wakeup_service_streaming_section_changed_cb callback, void* user_data); +#define MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_ENGINE_COMMAND_CALLBACK "wakeup_manager_set_wakeup_engine_command_callback" +typedef int (*wakeup_manager_set_wakeup_engine_command_callback)(wakeup_service_wakeup_engine_command_cb callback, void* user_data); typedef struct { wakeup_manager_initialize initialize; @@ -213,6 +215,7 @@ typedef struct { wakeup_manager_set_setting_changed_callback set_setting_changed_callback; wakeup_manager_set_error_callback set_error_callback; wakeup_manager_set_streaming_section_changed_callback set_streaming_section_changed_callback; + wakeup_manager_set_wakeup_engine_command_callback set_wakeup_engine_command_callback; } wakeup_manager_interface; #ifdef __cplusplus diff --git a/inc/multi_wakeup_recognizer.h b/inc/multi_wakeup_recognizer.h index b0732f9..f928720 100644 --- a/inc/multi_wakeup_recognizer.h +++ b/inc/multi_wakeup_recognizer.h @@ -105,6 +105,8 @@ typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* us typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); +typedef void (*wakeup_service_wakeup_engine_command_cb)(const char* assistant_name, const char* command, void* user_data); + #ifdef __cplusplus } #endif diff --git a/plugins/wakeup-manager/inc/wakeup_engine_manager.h b/plugins/wakeup-manager/inc/wakeup_engine_manager.h index d9d478c..8789ee8 100644 --- a/plugins/wakeup-manager/inc/wakeup_engine_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_engine_manager.h @@ -84,6 +84,8 @@ typedef int (*wakeup_engine_set_speech_status_callback)(wakeup_service_speech_st typedef int (*wakeup_engine_set_error_callback)(wakeup_service_error_cb callback, void* user_data); #define MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_DATA_REQUIRE_STATUS_CALLBACK "wakeup_engine_set_audio_data_require_status_callback" typedef int (*wakeup_engine_set_audio_data_require_status_callback)(wakeup_service_audio_data_require_status_cb callback, void* user_data); +#define MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_ENGINE_COMMAND_CALLBACK "wakeup_engine_set_wakeup_engine_command_callback" +typedef int (*wakeup_engine_set_wakeup_engine_command_callback)(wakeup_service_engine_command_cb callback, void* user_data); /* Interfaces after version 1 */ #define MA_WAKEUP_ENGINE_FUNC_GET_VERSION "wakeup_engine_get_version" @@ -113,6 +115,7 @@ typedef struct { wakeup_engine_set_speech_status_callback set_speech_status_callback; wakeup_engine_set_error_callback set_error_callback; wakeup_engine_set_audio_data_require_status_callback set_audio_data_require_status_callback; + wakeup_engine_set_wakeup_engine_command_callback set_wakeup_engine_command_callback; } wakeup_engine_interface; class IEngineEventObserver @@ -127,6 +130,7 @@ public: virtual bool on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0; virtual bool on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0; + virtual bool on_wakeup_engine_command(string assistant_name, string command) = 0; }; class CWakeupEngineManager @@ -169,6 +173,7 @@ public: bool on_speech_status(string engine_name, wakeup_service_speech_status_e status); bool on_error(string engine_name, int error_code, string error_message); bool on_audio_data_require_status(string engine_name, bool require); + bool on_wakeup_engine_command(string engine_name, string assistant_name, string command); private: typedef struct { int version; diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index c6623a2..ef2574d 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -48,6 +48,7 @@ public: virtual void on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0; virtual void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0; + virtual void on_wakeup_engine_command(const char* assistant_name, const char* command) = 0; }; class ISettingValueObserver { @@ -138,6 +139,7 @@ private: bool on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override; bool on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override; + bool on_wakeup_engine_command(string assistant_name, string command) override; void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; } private: diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index d6d20a5..561d855 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -71,6 +71,8 @@ typedef void (*wakeup_service_audio_data_require_status_cb)(bool require, void* typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); +typedef void (*wakeup_service_engine_command_cb)(const char* assistant_name, const char* command, void* user_data); + typedef enum { MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0, MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, @@ -156,6 +158,8 @@ EXPORT_API int wakeup_manager_set_error_callback(wakeup_service_error_cb callbac EXPORT_API int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streaming_section_changed_cb callback, void* user_data); +EXPORT_API int wakeup_manager_set_wakeup_engine_command_callback(wakeup_service_engine_command_cb callback, void* user_data); + /* Internal API declarations for dependency modules */ int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len); diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index 896057f..d9f6d7e 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -99,6 +99,9 @@ void CWakeupEngineManager::deinitialize() if (info.interface.set_audio_data_require_status_callback) { info.interface.set_audio_data_require_status_callback(nullptr, nullptr); } + if (info.interface.set_wakeup_engine_command_callback) { + info.interface.set_wakeup_engine_command_callback(nullptr, nullptr); + } MWR_LOGI("Trying to deinitialize engine : %s", info.engine_name.c_str()); if (info.interface.deinitialize) { int ret = info.interface.deinitialize(); @@ -587,6 +590,30 @@ bool CWakeupEngineManager::on_audio_data_require_status(string engine_name, bool return true; } +bool CWakeupEngineManager::on_wakeup_engine_command(string engine_name, string assistant_name, string command) +{ + MWR_LOGD("[ENTER]"); + + for (const auto& observer : mObservers) { + if (observer) { + const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(), + [engine_name](const EngineInfo& info) { + return (0 == info.engine_name.compare(engine_name)); + }); + if (mEngineInfo.end() != iter) { + for (const auto& assistant : iter->assistant_list) { + if (0 == assistant_name.compare(assistant) || 0 == assistant_name.length()) + if (!observer->on_wakeup_engine_command(assistant, command)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + } + } + + return true; +} + void CWakeupEngineManager::add_engine(string name, string path) { MWR_LOGD("Name (%s), Filepath(%s)", name.c_str(), path.c_str()); @@ -667,6 +694,9 @@ void CWakeupEngineManager::add_engine(string name, string path) info.interface.set_audio_data_require_status_callback = (wakeup_engine_set_audio_data_require_status_callback)dlsym(info.engine_handle, MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_DATA_REQUIRE_STATUS_CALLBACK); + info.interface.set_wakeup_engine_command_callback = + (wakeup_engine_set_wakeup_engine_command_callback)dlsym(info.engine_handle, + MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_ENGINE_COMMAND_CALLBACK); /* Interfaces after version 1 */ info.interface.get_version = @@ -722,6 +752,18 @@ void CWakeupEngineManager::add_engine(string name, string path) }, &(callback_user_data.back())); } + if (info.interface.set_wakeup_engine_command_callback) { + info.interface.set_wakeup_engine_command_callback( + [](const char* assistant_name, const char* command, void* user_data) { + CallbackUserData* data = static_cast(user_data); + if (nullptr == data) return; + if (nullptr == data->manager) return; + if (nullptr == command) return; + data->manager->on_wakeup_engine_command( + data->engine_name, (assistant_name ? assistant_name : ""), command); + }, &(callback_user_data.back())); + } + if (info.interface.initialize) { info.interface.initialize(); } diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 1b89605..d0b0e05 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -909,6 +909,19 @@ bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section( return true; } +bool CWakeupManager::CEngineEventObserver::on_wakeup_engine_command( + string assistant_name, string command) +{ + if (nullptr == mWakeupManager) return false; + + vector observers = mWakeupManager->get_wakeup_observers(); + for (const auto& observer : observers) { + observer->on_wakeup_engine_command(assistant_name.c_str(), command.c_str()); + } + + return true; +} + void CWakeupManager::CPolicyEventObserver::on_wakeup(wakeup_event_info wakeup_info) { if (nullptr == mWakeupManager) return; diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 0d30121..5e041ee 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -45,12 +45,16 @@ static void* g_error_user_data; static wakeup_service_streaming_section_changed_cb g_streaming_section_changed_cb; static void* g_streaming_section_changed_user_data; +static wakeup_service_engine_command_cb g_wakeup_engine_command_cb; +static void* g_wakeup_engine_command_user_data; + class CWakeupEventObserver : public IWakeupEventObserver { void on_wakeup(wakeup_event_info wakeup_info) override; void on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override; void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override; + void on_wakeup_engine_command(const char* assistant_name, const char* command) override; }; class CSettingValueObserver : public ISettingValueObserver @@ -99,6 +103,12 @@ int wakeup_manager_initialize(void) g_error_cb = NULL; g_error_user_data = NULL; + g_streaming_section_changed_cb = NULL; + g_streaming_section_changed_user_data = NULL; + + g_wakeup_engine_command_cb = NULL; + g_wakeup_engine_command_user_data = NULL; + g_wakeup_manager->initialize(); MWR_LOGD("[END]"); @@ -617,6 +627,23 @@ int wakeup_manager_set_streaming_section_changed_callback(wakeup_service_streami return 0; } +int wakeup_manager_set_wakeup_engine_command_callback(wakeup_service_engine_command_cb callback, void* user_data) +{ + MWR_LOGD("[ENTER]"); + + if (NULL == callback) { + MWR_LOGE("[ERROR] Input parameter is NULL"); + return -1; + } + + g_wakeup_engine_command_cb = callback; + g_wakeup_engine_command_user_data = user_data; + + MWR_LOGD("[END]"); + return 0; +} + + int wakeup_manager_feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len) { if (nullptr == g_wakeup_manager) return -1; @@ -655,6 +682,14 @@ void CWakeupEventObserver::on_audio_streaming_data_section( } } +void CWakeupEventObserver::on_wakeup_engine_command( + const char* assistant_name, const char* command) +{ + if (g_wakeup_engine_command_cb) { + g_wakeup_engine_command_cb(assistant_name, command, g_wakeup_engine_command_user_data); + } +} + void CSettingValueObserver::on_value_changed() { if (g_setting_changed_cb) { diff --git a/src/multi_assistant_dbus.c b/src/multi_assistant_dbus.c index 53130d8..e1ee0a6 100644 --- a/src/multi_assistant_dbus.c +++ b/src/multi_assistant_dbus.c @@ -447,6 +447,71 @@ int masc_dbus_send_preprocessing_result(int pid, bool result) return 0; } +int masc_dbus_send_wakeup_engine_command(int pid, const char* command) +{ + if (0 != __dbus_check()) { + return -1; //MAS_ERROR_OPERATION_FAILED; + } + + DBusMessage* msg; + + DBusError err; + dbus_error_init(&err); + + char service_name[64]; + memset(service_name, '\0', 64); + snprintf(service_name, 64, "%s_%d", MA_CLIENT_SERVICE_NAME, pid); + + msg = dbus_message_new_method_call( + service_name, + MA_CLIENT_SERVICE_OBJECT_PATH, + MA_CLIENT_SERVICE_INTERFACE, + MAS_METHOD_SEND_WAKEUP_ENGINE_COMMAND); + + static int count = 0; + if (NULL == msg) { + MAS_LOGE(">>>> Request mas send wakeup engine command : Fail to make message"); + return -1; // MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD(">>>> Request mas send wakeup engine command : %s", service_name); + } + + char* temp_command = NULL; + if (!command) + temp_command = strdup("#NULL"); + else + temp_command = strdup(command); + + if (true != dbus_message_append_args(msg, + DBUS_TYPE_STRING, &temp_command, + DBUS_TYPE_INVALID)) { + dbus_message_unref(msg); + MAS_LOGE("[ERROR] Fail to append args"); + if (temp_command) + free(temp_command); + return -1; + } + + dbus_message_set_no_reply(msg, TRUE); + + if (1 != dbus_connection_send(g_conn_sender, msg, NULL)) { + MAS_LOGE("[Dbus ERROR] Fail to Send"); + if (temp_command) + free(temp_command); + return -1; // MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD("[Dbus DEBUG] Success to Send wakeup_engine_command : %s", command); + dbus_connection_flush(g_conn_sender); + } + + dbus_message_unref(msg); + + if (temp_command) + free(temp_command); + + return 0; +} + int masc_ui_dbus_send_hello(void) { if (0 != __dbus_check()) { diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index e7640f6..3af09f0 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -354,6 +354,19 @@ static void __streaming_section_changed_cb(ma_audio_streaming_data_section_e sec } } +static void __wakeup_engine_command_cb(const char* assistant_name, const char* command, void* user_data) +{ + MAS_LOGD( "[SUCCESS] __wakeup_engine_command_cb is called, command(%s)", command); + + int pid = mas_get_client_pid_by_appid(assistant_name); + if (-1 != pid) { + int ret = masc_dbus_send_wakeup_engine_command(pid, command); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to send wakeup engine command, ret(%d)", ret); + } + } +} + int multi_assistant_service_plugin_initialize(void) { MAS_LOGD( "[Enter]"); @@ -472,6 +485,9 @@ int multi_assistant_service_plugin_initialize(void) _wakeup_manager_interface.set_streaming_section_changed_callback = (wakeup_manager_set_streaming_section_changed_callback)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_SET_STREAMING_SECTION_CHANGED_CALLBACK); + _wakeup_manager_interface.set_wakeup_engine_command_callback = + (wakeup_manager_set_wakeup_engine_command_callback)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_ENGINE_COMMAND_CALLBACK); int ret = -1; if (NULL != g_handle) { @@ -1046,6 +1062,12 @@ int multi_assistant_service_plugin_set_callbacks(void) return ret; } + ret = multi_assistant_service_plugin_set_wakeup_engine_command_callback(__wakeup_engine_command_cb, NULL); + if (0 != ret) { + MAS_LOGE("Fail to set wakeup engine command cb"); + return ret; + } + return 0; } @@ -1192,3 +1214,20 @@ int multi_assistant_service_plugin_set_streaming_section_changed_callback(wakeup } return ret; } + +int multi_assistant_service_plugin_set_wakeup_engine_command_callback(wakeup_service_wakeup_engine_command_cb callback, void* user_data) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_set_wakeup_engine_command_callback func = _wakeup_manager_interface.set_wakeup_engine_command_callback; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SET_WAKEUP_ENGINE_COMMAND_CALLBACK); + } else { + ret = func(callback, user_data); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set error callback, ret(%d)", ret); + } + } + } + return ret; +} \ No newline at end of file -- 2.7.4 From 71d57e0658dba389847524a9a943b26b9a8db317 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 22 Nov 2019 14:18:09 +0900 Subject: [PATCH 15/16] Enable sending wakeup engine commands to the dependency module Change-Id: I27db2e1cc1da263428d800dc3b02dab6d5770df3 --- inc/multi_wakeup_recognizer.h | 8 +++++- .../dependency-default/inc/dependency_default.h | 1 + .../dependency-default/src/dependency_default.cpp | 6 ++++ plugins/wakeup-manager/inc/dependency_resolver.h | 4 +++ plugins/wakeup-manager/inc/wakeup_engine_manager.h | 6 ++-- plugins/wakeup-manager/inc/wakeup_manager.h | 6 ++-- .../wakeup-manager/inc/wakeup_manager_wrapper.h | 8 +++++- plugins/wakeup-manager/src/dependency_resolver.cpp | 28 ++++++++++++++++++ .../wakeup-manager/src/wakeup_engine_manager.cpp | 33 ++++++++++++++-------- plugins/wakeup-manager/src/wakeup_manager.cpp | 15 +++++++--- .../wakeup-manager/src/wakeup_manager_wrapper.cpp | 7 +++-- src/multi_assistant_service_plugin.c | 2 +- 12 files changed, 98 insertions(+), 26 deletions(-) diff --git a/inc/multi_wakeup_recognizer.h b/inc/multi_wakeup_recognizer.h index f928720..03e6685 100644 --- a/inc/multi_wakeup_recognizer.h +++ b/inc/multi_wakeup_recognizer.h @@ -91,6 +91,12 @@ typedef enum { MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD, /**< Wake word section started */ } ma_audio_streaming_data_section_e; +typedef enum { + WAKEUP_ENGINE_COMMAND_TARGET_ALL_ASSISTANTS = 0, /**< All assistants - name not required */ + WAKEUP_ENGINE_COMMAND_TARGET_SPECIFIC_ASSISTANTS, /**< Specific assistant - name required */ + WAKEUP_ENGINE_COMMAND_TARGET_DEPENDENCY_MODULE, /**< Dependency module - name not required */ +} wakeup_engine_command_target_e; + typedef void (*ma_audio_streaming_data_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); typedef void (*wakeup_service_wakeup_event_cb)(wakeup_event_info wakeup_info, const char* wakeup_word, void* user_data); @@ -105,7 +111,7 @@ typedef void (*wakeup_service_error_cb)(int error, const char* err_msg, void* us typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); -typedef void (*wakeup_service_wakeup_engine_command_cb)(const char* assistant_name, const char* command, void* user_data); +typedef void (*wakeup_service_wakeup_engine_command_cb)(wakeup_engine_command_target_e target, const char* assistant_name, const char* command, void* user_data); #ifdef __cplusplus } diff --git a/plugins/wakeup-manager/dependency-default/inc/dependency_default.h b/plugins/wakeup-manager/dependency-default/inc/dependency_default.h index 2598200..f0e3c29 100644 --- a/plugins/wakeup-manager/dependency-default/inc/dependency_default.h +++ b/plugins/wakeup-manager/dependency-default/inc/dependency_default.h @@ -68,6 +68,7 @@ EXPORT_API int mas_dependency_set_recording_session(unsigned int session); EXPORT_API int mas_dependency_set_background_volume(double ratio); EXPORT_API int mas_dependency_get_audio_format(int* rate, int* channel, int* audio_type); EXPORT_API int mas_dependency_get_audio_source_type(char** type); +EXPORT_API int mas_dependency_process_wakeup_engine_command(const char* engine_name, const char* command); #ifdef __cplusplus } diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp index edff076..e3d859d 100644 --- a/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp @@ -88,3 +88,9 @@ int mas_dependency_get_audio_source_type(char** type) dependency_default_audio_get_audio_source_type(type); return 0; } + +int mas_dependency_process_wakeup_engine_command(const char* engine_name, const char* command) +{ + LOGD("Wakeup Engine %s has sent a command : %s", engine_name, command); + return 0; +} \ No newline at end of file diff --git a/plugins/wakeup-manager/inc/dependency_resolver.h b/plugins/wakeup-manager/inc/dependency_resolver.h index b1ed068..2f7271a 100644 --- a/plugins/wakeup-manager/inc/dependency_resolver.h +++ b/plugins/wakeup-manager/inc/dependency_resolver.h @@ -43,6 +43,7 @@ int dependency_resolver_set_recording_session(unsigned int); int dependency_resolver_set_background_volume(double ratio); int dependency_resolver_get_audio_format(int* rate, int* channel, int* audio_type); int dependency_resolver_get_audio_source_type(char** type); +int dependency_resolver_process_wakeup_engine_command(const char* engine, const char* command); /************************************************************************************** *** Definitions for dependencies @@ -73,6 +74,8 @@ typedef int (*mas_dependency_set_background_volume)(double ratio); typedef int (*mas_dependency_get_audio_format)(int* rate, int* channel, int* audio_type); #define MAS_DEPENDENCY_FUNC_GET_AUDIO_SOURCE_TYPE "mas_dependency_get_audio_source_type" typedef int (*mas_dependency_get_audio_source_type)(char** type); +#define MAS_DEPENDENCY_FUNC_PROCESS_WAKEUP_ENGINE_COMMAND "mas_dependency_process_wakeup_engine_command" +typedef int (*mas_dependency_process_wakeup_engine_command)(const char* engine_name, const char* command); typedef struct { mas_dependency_initialize initialize; @@ -84,6 +87,7 @@ typedef struct { mas_dependency_set_background_volume set_background_volume; mas_dependency_get_audio_format get_audio_format; mas_dependency_get_audio_source_type get_audio_source_type; + mas_dependency_process_wakeup_engine_command process_wakeup_engine_command; } mas_dependency; #ifdef __cplusplus diff --git a/plugins/wakeup-manager/inc/wakeup_engine_manager.h b/plugins/wakeup-manager/inc/wakeup_engine_manager.h index 8789ee8..cdf6f69 100644 --- a/plugins/wakeup-manager/inc/wakeup_engine_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_engine_manager.h @@ -130,7 +130,8 @@ public: virtual bool on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0; virtual bool on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0; - virtual bool on_wakeup_engine_command(string assistant_name, string command) = 0; + virtual bool on_wakeup_engine_command( + wakeup_engine_command_target_e target, string engine_name, string assistant_name, string command) = 0; }; class CWakeupEngineManager @@ -173,7 +174,8 @@ public: bool on_speech_status(string engine_name, wakeup_service_speech_status_e status); bool on_error(string engine_name, int error_code, string error_message); bool on_audio_data_require_status(string engine_name, bool require); - bool on_wakeup_engine_command(string engine_name, string assistant_name, string command); + bool on_wakeup_engine_command(string engine_name, + wakeup_engine_command_target_e target, string assistant_name, string command); private: typedef struct { int version; diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index ef2574d..3c7ba95 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -48,7 +48,8 @@ public: virtual void on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0; virtual void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) = 0; - virtual void on_wakeup_engine_command(const char* assistant_name, const char* command) = 0; + virtual void on_wakeup_engine_command( + wakeup_engine_command_target_e target, const char* name, const char* command) = 0; }; class ISettingValueObserver { @@ -139,7 +140,8 @@ private: bool on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override; bool on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override; - bool on_wakeup_engine_command(string assistant_name, string command) override; + bool on_wakeup_engine_command( + wakeup_engine_command_target_e target, string engine, string name, string command) override; void set_wakeup_manager(CWakeupManager *manager) { mWakeupManager = manager; } private: diff --git a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h index 561d855..1e975b9 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h +++ b/plugins/wakeup-manager/inc/wakeup_manager_wrapper.h @@ -55,6 +55,12 @@ typedef enum { MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD, /**< Wake word section started */ } ma_audio_streaming_data_section_e; +typedef enum { + WAKEUP_ENGINE_COMMAND_TARGET_ALL_ASSISTANTS = 0, /**< All assistants - name not required */ + WAKEUP_ENGINE_COMMAND_TARGET_SPECIFIC_ASSISTANT, /**< Specific assistant - name required */ + WAKEUP_ENGINE_COMMAND_TARGET_DEPENDENCY_MODULE, /**< Dependency module - name not required */ +} wakeup_engine_command_target_e; + typedef void (*ma_audio_streaming_data_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); typedef void (*wakeup_service_wakeup_event_cb)(wakeup_event_info wakeup_info, void* user_data); @@ -71,7 +77,7 @@ typedef void (*wakeup_service_audio_data_require_status_cb)(bool require, void* typedef void (*wakeup_service_streaming_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); -typedef void (*wakeup_service_engine_command_cb)(const char* assistant_name, const char* command, void* user_data); +typedef void (*wakeup_service_engine_command_cb)(wakeup_engine_command_target_e target, const char* assistant_name, const char* command, void* user_data); typedef enum { MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0, diff --git a/plugins/wakeup-manager/src/dependency_resolver.cpp b/plugins/wakeup-manager/src/dependency_resolver.cpp index 43b080d..b15eef8 100644 --- a/plugins/wakeup-manager/src/dependency_resolver.cpp +++ b/plugins/wakeup-manager/src/dependency_resolver.cpp @@ -86,6 +86,9 @@ int dependency_resolver_initialize(mas_proxy_interface interface) g_mas_dependency.get_audio_source_type = (mas_dependency_get_audio_source_type)dlsym(g_handle, MAS_DEPENDENCY_FUNC_GET_AUDIO_SOURCE_TYPE); + g_mas_dependency.process_wakeup_engine_command = + (mas_dependency_process_wakeup_engine_command)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_PROCESS_WAKEUP_ENGINE_COMMAND); int ret = -1; int dependency_version = 0; @@ -316,3 +319,28 @@ int dependency_resolver_get_audio_source_type(char** type) return ret; } + +int dependency_resolver_process_wakeup_engine_command(const char* engine, const char* command) +{ + int ret = -1; + if (NULL != g_handle) { + mas_dependency_process_wakeup_engine_command func = g_mas_dependency.process_wakeup_engine_command; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_PROCESS_WAKEUP_ENGINE_COMMAND); + } else { + try { + ret = func(engine, command); + } catch (const std::exception& e) { + MAS_LOGE("[ERROR] %s of dependency module threw exception : %s", + MAS_DEPENDENCY_FUNC_PROCESS_WAKEUP_ENGINE_COMMAND, e.what()); + } + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to process wakeup engine command, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + + return ret; +} diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index d9f6d7e..f75f958 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -590,21 +590,29 @@ bool CWakeupEngineManager::on_audio_data_require_status(string engine_name, bool return true; } -bool CWakeupEngineManager::on_wakeup_engine_command(string engine_name, string assistant_name, string command) +bool CWakeupEngineManager::on_wakeup_engine_command(string engine_name, wakeup_engine_command_target_e target, string assistant_name, string command) { MWR_LOGD("[ENTER]"); for (const auto& observer : mObservers) { if (observer) { - const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(), - [engine_name](const EngineInfo& info) { - return (0 == info.engine_name.compare(engine_name)); - }); - if (mEngineInfo.end() != iter) { - for (const auto& assistant : iter->assistant_list) { - if (0 == assistant_name.compare(assistant) || 0 == assistant_name.length()) - if (!observer->on_wakeup_engine_command(assistant, command)) { - LOGE("[Recorder WARNING] One of the observer returned false"); + if (WAKEUP_ENGINE_COMMAND_TARGET_DEPENDENCY_MODULE == target) { + if (!observer->on_wakeup_engine_command(target, engine_name, assistant_name, command)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } else { + const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(), + [engine_name](const EngineInfo& info) { + return (0 == info.engine_name.compare(engine_name)); + }); + if (mEngineInfo.end() != iter) { + for (const auto& assistant : iter->assistant_list) { + if (0 == assistant_name.compare(assistant) || + WAKEUP_ENGINE_COMMAND_TARGET_ALL_ASSISTANTS == target) { + if (!observer->on_wakeup_engine_command(target, engine_name, assistant, command)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } } } } @@ -754,13 +762,14 @@ void CWakeupEngineManager::add_engine(string name, string path) if (info.interface.set_wakeup_engine_command_callback) { info.interface.set_wakeup_engine_command_callback( - [](const char* assistant_name, const char* command, void* user_data) { + [](wakeup_engine_command_target_e target, + const char* assistant_name, const char* command, void* user_data) { CallbackUserData* data = static_cast(user_data); if (nullptr == data) return; if (nullptr == data->manager) return; if (nullptr == command) return; data->manager->on_wakeup_engine_command( - data->engine_name, (assistant_name ? assistant_name : ""), command); + data->engine_name, target, (assistant_name ? assistant_name : ""), command); }, &(callback_user_data.back())); } diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index d0b0e05..d4e82fc 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -910,13 +910,20 @@ bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section( } bool CWakeupManager::CEngineEventObserver::on_wakeup_engine_command( - string assistant_name, string command) + wakeup_engine_command_target_e target, string engine_name, string assistant_name, string command) { if (nullptr == mWakeupManager) return false; - vector observers = mWakeupManager->get_wakeup_observers(); - for (const auto& observer : observers) { - observer->on_wakeup_engine_command(assistant_name.c_str(), command.c_str()); + if (WAKEUP_ENGINE_COMMAND_TARGET_DEPENDENCY_MODULE == target) { + dependency_resolver_process_wakeup_engine_command(engine_name.c_str(), command.c_str()); + } + + if (WAKEUP_ENGINE_COMMAND_TARGET_ALL_ASSISTANTS == target || + WAKEUP_ENGINE_COMMAND_TARGET_SPECIFIC_ASSISTANT == target) { + vector observers = mWakeupManager->get_wakeup_observers(); + for (const auto& observer : observers) { + observer->on_wakeup_engine_command(target, assistant_name.c_str(), command.c_str()); + } } return true; diff --git a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp index 5e041ee..1abfbdf 100644 --- a/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager_wrapper.cpp @@ -54,7 +54,8 @@ class CWakeupEventObserver : public IWakeupEventObserver void on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) override; void on_audio_streaming_data_section(ma_audio_streaming_data_section_e section) override; - void on_wakeup_engine_command(const char* assistant_name, const char* command) override; + void on_wakeup_engine_command( + wakeup_engine_command_target_e target, const char* assistant_name, const char* command) override; }; class CSettingValueObserver : public ISettingValueObserver @@ -683,10 +684,10 @@ void CWakeupEventObserver::on_audio_streaming_data_section( } void CWakeupEventObserver::on_wakeup_engine_command( - const char* assistant_name, const char* command) + wakeup_engine_command_target_e target, const char* assistant_name, const char* command) { if (g_wakeup_engine_command_cb) { - g_wakeup_engine_command_cb(assistant_name, command, g_wakeup_engine_command_user_data); + g_wakeup_engine_command_cb(target, assistant_name, command, g_wakeup_engine_command_user_data); } } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 3af09f0..892eed3 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -354,7 +354,7 @@ static void __streaming_section_changed_cb(ma_audio_streaming_data_section_e sec } } -static void __wakeup_engine_command_cb(const char* assistant_name, const char* command, void* user_data) +static void __wakeup_engine_command_cb(wakeup_engine_command_target_e target, const char* assistant_name, const char* command, void* user_data) { MAS_LOGD( "[SUCCESS] __wakeup_engine_command_cb is called, command(%s)", command); -- 2.7.4 From dee3e8c805ca0740d6352debebb5db487d39f396 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 22 Nov 2019 16:53:06 +0900 Subject: [PATCH 16/16] Bump version to 0.2.12 Change-Id: I00755dfa06ed68a8a67e0c0051582f0055ded072 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 0aa3e02..e91e3a3 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index a87311f..edf1ad9 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.11 +Version: 0.2.12 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4