From efb490080309c118c4d48eab95e8fb80b5ac6f3e Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 15 Jun 2020 11:53:18 +0900 Subject: [PATCH 01/16] Bump version to 0.3.3 Change-Id: Idf3fcc0402c1f5979071c8859c8c842fc2b450f6 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 378863b..2a18a99 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 8c201ec..10c9ac5 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.3.2 +Version: 0.3.3 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From f9124a7b3ddf502a0cdfd8f132ff3bf4109cdf22 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 8 Jul 2020 21:27:34 +0900 Subject: [PATCH 02/16] Fix streaming START/FINISH event pair mismatch Change-Id: I03680aa7848579ac5fd1235f264d5c74d7bc3876 --- plugins/wakeup-manager/inc/wakeup_manager.h | 6 ++ plugins/wakeup-manager/src/wakeup_manager.cpp | 101 ++++++++++++--------- .../wakeup-manager/src/wakeup_policy_default.cpp | 2 + 3 files changed, 66 insertions(+), 43 deletions(-) diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index de7cb11..437dc06 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -136,6 +136,12 @@ public: void feed_audio_data(mas_speech_streaming_event_e event, void* buffer, int len); void set_dependency_module_command(string engine_name, string command); + + void start_streaming_duration_timer(); + void stop_streaming_duration_timer(); + + void set_streaming_duration_timer(Ecore_Timer* timer); + Ecore_Timer* get_streaming_duration_timer(); private: bool change_voice_key_status(ma_voice_key_status_e status); diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index b0ddfd1..1df79ea 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -137,10 +137,7 @@ bool CWakeupManager::deinitialize() { MWR_LOGI("[ENTER]"); - if (mStreamingDurationTimer) { - ecore_timer_del(mStreamingDurationTimer); - mStreamingDurationTimer = nullptr; - } + stop_streaming_duration_timer(); dependency_resolver_deinitialize(); @@ -562,6 +559,7 @@ bool CWakeupManager::process_plugin_event(mas_plugin_event_e event, void* data, } } if (stop_recording) { + stop_streaming_duration_timer(); mAudioManager.finalize_audio_data(); if (STREAMING_MODE::UTTERANCE == mStreamingMode) { @@ -603,6 +601,8 @@ static Eina_Bool streaming_duration_expired(void *data) CWakeupManager *wakeup_manager = static_cast(data); if (nullptr == wakeup_manager) return ECORE_CALLBACK_CANCEL; + wakeup_manager->set_streaming_duration_timer(nullptr); + CAudioManager *audio_manager = wakeup_manager->get_audio_manager(); CWakeupEngineManager *engine_manager = wakeup_manager->get_engine_manager(); @@ -658,17 +658,10 @@ bool CWakeupManager::start_streaming_utterance_data() mAudioManager.start_streaming_current_utterance_data(mLastWakeupEventInfo.wakeup_end_time); } - ecore_thread_main_loop_begin(); - if (mStreamingDurationTimer) { - ecore_timer_del(mStreamingDurationTimer); - mStreamingDurationTimer = nullptr; - } + stop_streaming_duration_timer(); if (streaming_by_manager) { - mStreamingDurationTimer = ecore_timer_add( - mWakeupSettings.get_streaming_duration_max(), - streaming_duration_expired, this); + start_streaming_duration_timer(); } - ecore_thread_main_loop_end(); MWR_LOGD("[END]"); return true; @@ -683,10 +676,7 @@ bool CWakeupManager::stop_streaming_utterance_data() mWakeupEngineManager.stop_streaming_current_utterance_data(); if (mStreamingDurationTimer) { - ecore_thread_main_loop_begin(); - ecore_timer_del(mStreamingDurationTimer); - mStreamingDurationTimer = nullptr; - ecore_thread_main_loop_end(); + stop_streaming_duration_timer(); } if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING); @@ -717,15 +707,8 @@ bool CWakeupManager::start_streaming_follow_up_data() mAudioManager.start_streaming_follow_up_data(); - ecore_thread_main_loop_begin(); - if (mStreamingDurationTimer) { - ecore_timer_del(mStreamingDurationTimer); - mStreamingDurationTimer = nullptr; - } - mStreamingDurationTimer = ecore_timer_add( - mWakeupSettings.get_streaming_duration_max(), - streaming_duration_expired, this); - ecore_thread_main_loop_end(); + stop_streaming_duration_timer(); + start_streaming_duration_timer(); MWR_LOGD("[END]"); return true; @@ -740,10 +723,7 @@ bool CWakeupManager::stop_streaming_follow_up_data() mWakeupEngineManager.stop_streaming_current_utterance_data(); if (mStreamingDurationTimer) { - ecore_thread_main_loop_begin(); - ecore_timer_del(mStreamingDurationTimer); - mStreamingDurationTimer = nullptr; - ecore_thread_main_loop_end(); + stop_streaming_duration_timer(); } if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { @@ -771,15 +751,8 @@ bool CWakeupManager::start_streaming_previous_utterance_data() mStreamingMode = STREAMING_MODE::PREVIOUS_UTTERANCE; mAudioManager.start_streaming_previous_utterance_data(); - ecore_thread_main_loop_begin(); - if (mStreamingDurationTimer) { - ecore_timer_del(mStreamingDurationTimer); - mStreamingDurationTimer = nullptr; - } - mStreamingDurationTimer = ecore_timer_add( - mWakeupSettings.get_streaming_duration_max(), - streaming_duration_expired, this); - ecore_thread_main_loop_end(); + stop_streaming_duration_timer(); + start_streaming_duration_timer(); MWR_LOGD("[END]"); return true; @@ -794,10 +767,7 @@ bool CWakeupManager::stop_streaming_previous_utterance_data() mWakeupEngineManager.stop_streaming_current_utterance_data(); if (mStreamingDurationTimer) { - ecore_thread_main_loop_begin(); - ecore_timer_del(mStreamingDurationTimer); - mStreamingDurationTimer = nullptr; - ecore_thread_main_loop_end(); + stop_streaming_duration_timer(); } if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { @@ -906,6 +876,49 @@ void CWakeupManager::set_dependency_module_command(string engine_name, string co mWakeupEngineManager.engine_set_dependency_module_command(engine_name, command); } +void CWakeupManager::start_streaming_duration_timer() +{ + ecore_main_loop_thread_safe_call_async([](void* data) { + CWakeupManager* manager = static_cast(data); + if (!manager) return; + + CWakeupSettings *settings = manager->get_wakeup_settings(); + if (settings) { + Ecore_Timer* timer = ecore_timer_add( + settings->get_streaming_duration_max(), + streaming_duration_expired, manager); + manager->set_streaming_duration_timer(timer); + MWR_LOGI("DURATION_TIMER STARTED : %p", timer); + } + }, this); +} + +void CWakeupManager::stop_streaming_duration_timer() +{ + MWR_LOGI("DURATION_TIMER STOP"); + if (mStreamingDurationTimer) { + ecore_main_loop_thread_safe_call_async([](void* data) { + CWakeupManager* manager = static_cast(data); + if (!manager) return; + + Ecore_Timer* timer = manager->get_streaming_duration_timer(); + void* ret = ecore_timer_del(timer); + MWR_LOGI("DURATION_TIMER EXISTS : %p %p", timer, ret); + manager->set_streaming_duration_timer(nullptr); + }, this); + } +} + +void CWakeupManager::set_streaming_duration_timer(Ecore_Timer* timer) +{ + mStreamingDurationTimer = timer; +} + +Ecore_Timer* CWakeupManager::get_streaming_duration_timer() +{ + return mStreamingDurationTimer; +} + bool CWakeupManager::CEngineEventObserver::on_wakeup_event(string engine_name, mas_wakeup_event_info wakeup_info) { MWR_LOGD("[ENTER]"); @@ -977,6 +990,7 @@ bool CWakeupManager::CEngineEventObserver::on_streaming_audio_data( } if (MAS_SPEECH_STREAMING_EVENT_FINISH == event) { mWakeupManager->set_streaming_mode(STREAMING_MODE::NONE); + mWakeupManager->stop_streaming_duration_timer(); } return true; @@ -1066,6 +1080,7 @@ bool CWakeupManager::CAudioEventObserver::on_streaming_audio_data( } if (MAS_SPEECH_STREAMING_EVENT_FINISH == event) { mWakeupManager->set_streaming_mode(STREAMING_MODE::NONE); + mWakeupManager->stop_streaming_duration_timer(); } return true; diff --git a/plugins/wakeup-manager/src/wakeup_policy_default.cpp b/plugins/wakeup-manager/src/wakeup_policy_default.cpp index 7a471fd..e07374c 100644 --- a/plugins/wakeup-manager/src/wakeup_policy_default.cpp +++ b/plugins/wakeup-manager/src/wakeup_policy_default.cpp @@ -59,8 +59,10 @@ void CWakeupPolicyDefault::wakeup_candidate(mas_wakeup_event_info wakeup_info) { mWakeupInfos.push_back(wakeup_info); if (nullptr == mTimer) { + LOGD("Begin Ecore Thread Main Loop"); ecore_thread_main_loop_begin(); mTimer = ecore_timer_add(mDelaySeconds, timer_func, this); + LOGD("End Ecore Thread Main Loop"); ecore_thread_main_loop_end(); } } -- 2.7.4 From c6d3c1c29ba478e9ebe686ccafb2cdf83553b38d Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 16 Jul 2020 20:12:54 +0900 Subject: [PATCH 03/16] Ignore outdated wakeup timer events Change-Id: I0493c55d587c8d244a2db3f204592e259512172f --- src/multi_assistant_service_plugin.c | 86 ++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index 6dedaf4..f34d095 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -93,48 +93,78 @@ Eina_Bool __send_result(void *data) } #endif /* -TEST_CODE */ -Eina_Bool process_wakeup_event_by_appid_timer(char* appid) +static int g_last_wakeup_event_id = 0; +typedef struct { + int id; + char* data; +} wakeup_event; + +void process_wakeup_event_by_appid_timer(wakeup_event* event) { - MAS_LOGD("[ENTER] appid(%s)", appid); + if (!event) return; + MAS_LOGD("[ENTER] appid(%s), id(%d), g_last_wakeup_event_id(%d)", + event->data, event->id, g_last_wakeup_event_id); + + if (!event->data) { + free(event); + return; + } + + if (event->id < g_last_wakeup_event_id) { + free(event->data); + free(event); + return; + } int pid = -1; - if (!appid) return ECORE_CALLBACK_CANCEL; - bool use_custom_ui = mas_get_client_custom_ui_option_by_appid(appid); + bool use_custom_ui = mas_get_client_custom_ui_option_by_appid(event->data); bool ui_panel_enabled = is_ui_panel_enabled(); if (ui_panel_enabled) masc_ui_dbus_enable_common_ui(!use_custom_ui); - mas_set_current_client_by_appid(appid); - if (ui_panel_enabled) masc_ui_dbus_change_assistant(appid); - if ((pid = mas_get_client_pid_by_appid(appid)) != -1) { + mas_set_current_client_by_appid(event->data); + if (ui_panel_enabled) masc_ui_dbus_change_assistant(event->data); + if ((pid = mas_get_client_pid_by_appid(event->data)) != -1) { mas_client_send_preprocessing_information(pid); mas_client_activate(pid); mas_process_preprocessing_state_event(PREPROCESSING_STATE_EVENT_ASSISTANT_ACTIVATED); } else { // Appropriate MA Client not available, trying to launch new one - MAS_LOGD("MA Client with appid %s does not exist, launching client", appid); - mas_launch_client_by_appid(appid, CLIENT_LAUNCH_MODE_ACTIVATION); + MAS_LOGD("MA Client with appid %s does not exist, launching client", event->data); + mas_launch_client_by_appid(event->data, CLIENT_LAUNCH_MODE_ACTIVATION); } - if (appid) free(appid); + free(event->data); + free(event); MAS_LOGD("END"); - return ECORE_CALLBACK_CANCEL; + return; } -Eina_Bool process_wakeup_event_by_word_timer(char* wakeup_word) +void process_wakeup_event_by_word_timer(wakeup_event* event) { - MAS_LOGD("[ENTER]"); + if (!event) return; + MAS_LOGD("[ENTER] wakeword(%s), id(%d)", event->data, event->id); + + if (!event->data) { + free(event); + return; + } - if (!wakeup_word) return EINA_FALSE; + const char* appid = mas_get_client_appid_by_wakeup_word(event->data); - const char* appid = mas_get_client_appid_by_wakeup_word(wakeup_word); - process_wakeup_event_by_appid_timer(strdup(appid)); + wakeup_event* appid_timer_event = malloc(sizeof(wakeup_event)); + if (appid_timer_event) { + appid_timer_event->id = event->id; + appid_timer_event->data = strdup(appid ? appid : ""); + process_wakeup_event_by_appid_timer(event); + } - if (wakeup_word) free(wakeup_word); + free(event->data); + free(event); MAS_LOGD("END"); - return ECORE_CALLBACK_CANCEL; + return; } static void __wakeup_event_cb(mas_wakeup_event_info wakeup_info, void* user_data) @@ -206,13 +236,21 @@ static void __wakeup_event_cb(mas_wakeup_event_info wakeup_info, void* user_data } #endif /* - TEST_CODE */ if (wakeup_info.wakeup_appid) { - ecore_thread_main_loop_begin(); - ecore_timer_add(0.0f, process_wakeup_event_by_appid_timer, (void*)strdup(wakeup_info.wakeup_appid)); - ecore_thread_main_loop_end(); + wakeup_event* appid_timer_event = malloc(sizeof(wakeup_event)); + if (appid_timer_event) { + appid_timer_event->id = ++g_last_wakeup_event_id; + appid_timer_event->data = + strdup(wakeup_info.wakeup_appid ? wakeup_info.wakeup_appid : ""); + ecore_main_loop_thread_safe_call_async(process_wakeup_event_by_appid_timer, (void*)appid_timer_event); + } } else if (wakeup_info.wakeup_word) { - ecore_thread_main_loop_begin(); - ecore_timer_add(0.0f, process_wakeup_event_by_word_timer, (void*)strdup(wakeup_info.wakeup_word)); - ecore_thread_main_loop_end(); + wakeup_event* word_timer_event = malloc(sizeof(wakeup_event)); + if (word_timer_event) { + word_timer_event->id = ++g_last_wakeup_event_id; + word_timer_event->data = + strdup(wakeup_info.wakeup_word ? wakeup_info.wakeup_word : ""); + ecore_main_loop_thread_safe_call_async(process_wakeup_event_by_word_timer, (void*)word_timer_event); + } } } -- 2.7.4 From 9549b5c5b81aa3be186cac1e7caabd05e34c5504 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 17 Jul 2020 14:30:26 +0900 Subject: [PATCH 04/16] Use async version of aul_launch_app for avoiding deadlock Change-Id: I484ced8492df92262ee34e19b687ae62d4cba76f --- src/multi_assistant_service.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 9ed865e..3fc3b00 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -1184,9 +1184,9 @@ int mas_bring_client_to_foreground(const char* appid) return -1; } - int result = aul_launch_app(appid, b); + int result = aul_launch_app_async(appid, b); if (result < AUL_R_OK) { - MAS_LOGE("ERROR : aul_launch_app failed. app_id [%s] bundle[%p] result[%d : %s]", + MAS_LOGE("ERROR : aul_launch_app_async failed. app_id [%s] bundle[%p] result[%d : %s]", appid, b, result, get_error_message(result)); } -- 2.7.4 From ed6f6921882fee2577b25be38ef2a2230c39ecef Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 16 Jul 2020 20:09:12 +0900 Subject: [PATCH 05/16] Add logs for checking wakeup_engine_command delivery Change-Id: Ib114ff5f46c96850d344eb26c43135f371222b74 --- plugins/wakeup-manager/src/wakeup_engine_manager.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index d9181dc..40f6833 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -728,7 +728,8 @@ bool CWakeupEngineManager::on_audio_data_require_status(string engine_name, bool bool CWakeupEngineManager::on_wakeup_engine_command(string engine_name, mas_wakeup_engine_command_target_e target, string assistant_name, string command) { - MWR_LOGD("[ENTER]"); + MWR_LOGI("[ENTER] : %s %d %s %s", + engine_name.c_str(), target, assistant_name.c_str(), command.c_str()); for (const auto& observer : mObservers) { if (observer) { @@ -745,6 +746,7 @@ bool CWakeupEngineManager::on_wakeup_engine_command(string engine_name, mas_wake for (const auto& assistant : iter->assistant_list) { if (0 == assistant_name.compare(assistant) || MAS_WAKEUP_ENGINE_COMMAND_TARGET_ALL_ASSISTANTS == target) { + MWR_LOGI("Calling on_wakeup_engine_command for %s", assistant.c_str()); if (!observer->on_wakeup_engine_command(target, engine_name, assistant, command)) { LOGE("[Recorder WARNING] One of the observer returned false"); } -- 2.7.4 From 39ee9688098e5e7f79fe5d510373fed1c6446152 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 17 Jul 2020 11:42:04 +0900 Subject: [PATCH 06/16] Bump version to 0.2.35 Change-Id: Ib8b1ca66b5e49ef6266973cb960092c7839473a3 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index db96b45..c012fe7 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index f5ffa24..5288e81 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.34 +Version: 0.2.35 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From eb0c49d73f42ef9d2adbad128bf0744b8807ba17 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 17 Jul 2020 21:29:55 +0900 Subject: [PATCH 07/16] Replace ecore_thread_main_loop_* functions with alternatives Change-Id: Ied3b35f41bb7c9f8bc74bc24d42e5a92e3b4b551 --- plugins/wakeup-manager/inc/wakeup_policy_default.h | 8 ++++++- .../wakeup-manager/src/wakeup_policy_default.cpp | 28 ++++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/plugins/wakeup-manager/inc/wakeup_policy_default.h b/plugins/wakeup-manager/inc/wakeup_policy_default.h index d4b36b7..914c539 100644 --- a/plugins/wakeup-manager/inc/wakeup_policy_default.h +++ b/plugins/wakeup-manager/inc/wakeup_policy_default.h @@ -40,12 +40,18 @@ public: bool valid() override; void set_assistant_priority(string appid, int priority); - void set_delay(float seconds); void wakeup_candidate(mas_wakeup_event_info wakeup_info) override; void select_candidate(mas_wakeup_event_info wakeup_info) override; void timer_expired(); + + void set_delay(float seconds); + float get_delay(); + + void set_timer(Ecore_Timer* timer); + Ecore_Timer* get_timer(); + private: typedef struct { string appid; diff --git a/plugins/wakeup-manager/src/wakeup_policy_default.cpp b/plugins/wakeup-manager/src/wakeup_policy_default.cpp index e07374c..45b5caa 100644 --- a/plugins/wakeup-manager/src/wakeup_policy_default.cpp +++ b/plugins/wakeup-manager/src/wakeup_policy_default.cpp @@ -45,12 +45,28 @@ void CWakeupPolicyDefault::set_delay(float seconds) mDelaySeconds = seconds; } +float CWakeupPolicyDefault::get_delay() +{ + return mDelaySeconds; +} + +void CWakeupPolicyDefault::set_timer(Ecore_Timer* timer) +{ + mTimer = timer; +} + +Ecore_Timer* CWakeupPolicyDefault::get_timer() +{ + return mTimer; +} + static Eina_Bool timer_func(void *data) { LOGD("[ENTER]"); if (data) { CWakeupPolicyDefault *policy = static_cast(data); policy->timer_expired(); + policy->set_timer(nullptr); } return ECORE_CALLBACK_CANCEL; } @@ -59,11 +75,13 @@ void CWakeupPolicyDefault::wakeup_candidate(mas_wakeup_event_info wakeup_info) { mWakeupInfos.push_back(wakeup_info); if (nullptr == mTimer) { - LOGD("Begin Ecore Thread Main Loop"); - ecore_thread_main_loop_begin(); - mTimer = ecore_timer_add(mDelaySeconds, timer_func, this); - LOGD("End Ecore Thread Main Loop"); - ecore_thread_main_loop_end(); + ecore_main_loop_thread_safe_call_async([](void* data) { + CWakeupPolicyDefault* policy = static_cast(data); + if (!policy) return; + + Ecore_Timer* timer = ecore_timer_add(policy->get_delay(), timer_func, data); + policy->set_timer(timer); + }, this); } } -- 2.7.4 From e667ff3d7f76ec4daf1c2f8cdd7defe789ae9647 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 22 Jul 2020 16:49:01 +0900 Subject: [PATCH 08/16] Bump version to 0.3.4 Change-Id: I07a09e3abf3283bd5e737bbb7682b689c6a6d204 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 2a18a99..83153e1 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 10c9ac5..4781812 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.3.3 +Version: 0.3.4 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 894ba9867035e67eda19e35f17e105f36cef911b Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 22 Jul 2020 17:21:24 +0900 Subject: [PATCH 09/16] Bump version to 0.2.36 Change-Id: Idf77f083e501e14f1c5b00f802cc01c60a929754 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index c012fe7..13cc8d2 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 5288e81..495fec1 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.2.35 +Version: 0.2.36 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 111600c4d88fdc3ae398b1cd0bc2a840b32ffa7e Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 3 Aug 2020 16:36:48 +0900 Subject: [PATCH 10/16] Fix bug assigning a newly created variable's value to itself Change-Id: I4ed18c047de2d7747193bf7296984eb2d782f70b --- src/service_plugin.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/service_plugin.cpp b/src/service_plugin.cpp index fe05b22..ca4416c 100644 --- a/src/service_plugin.cpp +++ b/src/service_plugin.cpp @@ -157,6 +157,7 @@ static void process_wakeup_event_by_word_timer(void* data) char* wakeup_word = static_cast(param->data); const char* appid = nullptr; + int id = param->id; CServicePlugin* plugin = param->plugin; CServiceMain* service_main = nullptr; @@ -174,7 +175,7 @@ static void process_wakeup_event_by_word_timer(void* data) if (service_main && appid) { param = new(std::nothrow) AsyncParam; if (param) { - param->id = param->id; + param->id = id; param->data = static_cast(strdup(appid)); param->plugin = plugin; process_wakeup_event_by_appid_timer(static_cast(param)); -- 2.7.4 From 6b26929dd1f6dc252d3fe89e6532aab4724b4743 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 3 Aug 2020 16:37:48 +0900 Subject: [PATCH 11/16] Bump version to 0.3.5 Change-Id: Ic80e59952a36c16b2f0f0508d880a83de78419d0 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 83153e1..e7ec256 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 4781812..1aa7b5c 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.3.4 +Version: 0.3.5 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From fe16d7610ebe5d86ec24df523118ab1bfaf0cb96 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 7 Aug 2020 15:25:06 +0900 Subject: [PATCH 12/16] Fix bug NULL is passed on streaming failure event Change-Id: I5b2900a9110d10051bfa028deb0ab5adf5f24a4e --- src/service_plugin.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/service_plugin.cpp b/src/service_plugin.cpp index ca4416c..254fb8e 100644 --- a/src/service_plugin.cpp +++ b/src/service_plugin.cpp @@ -350,8 +350,15 @@ void handle_speech_streaming_event_failure(void* data) static void __audio_streaming_cb(mas_speech_streaming_event_e event, void* buffer, int len, void *user_data) { + CServicePlugin* plugin = static_cast(user_data); + if (event == MAS_SPEECH_STREAMING_EVENT_FAIL) { - ecore_main_loop_thread_safe_call_async(handle_speech_streaming_event_failure, NULL); + AsyncParam* param = new(std::nothrow) AsyncParam; + if (param) { + param->plugin = plugin; + ecore_main_loop_thread_safe_call_async( + handle_speech_streaming_event_failure, static_cast(param)); + } return; } static int count = 0; @@ -361,7 +368,6 @@ static void __audio_streaming_cb(mas_speech_streaming_event_e event, void* buffe } ++count; - CServicePlugin* plugin = static_cast(user_data); CServiceIpcDbus* service_ipc = nullptr; CServiceMain* service_main = nullptr; if (plugin) { -- 2.7.4 From acb7991ffb6361917e8923ac1fbadac98a3c9be1 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Fri, 7 Aug 2020 15:26:03 +0900 Subject: [PATCH 13/16] Bump version to 0.3.6 Change-Id: Ibe2072f0b0d6b81c8d110d45ec41e25a3e456cc2 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index e7ec256..7ac0e5a 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 1aa7b5c..6957ade 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.3.5 +Version: 0.3.6 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4 From 600725865c2dce036e6260c28a812fa18659aff0 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 13 Aug 2020 14:14:36 +0900 Subject: [PATCH 14/16] Ignore streaming requests from clients not currently woken up Change-Id: I875b53caf10a110c967f6039604fbd4761ea47b7 --- src/service_main.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/service_main.cpp b/src/service_main.cpp index 7830c3f..a741ae0 100644 --- a/src/service_main.cpp +++ b/src/service_main.cpp @@ -200,6 +200,17 @@ int CServiceMain::client_send_recognition_result(pid_t pid, int result) int CServiceMain::client_start_streaming_audio_data(pid_t pid, int type) { int ret = -1; + const char *current_maclient_appid = NULL; + if (mCurrentClientInfo >= 0 && mCurrentClientInfo < MAX_MACLIENT_INFO_NUM) { + current_maclient_appid = mClientInfo[mCurrentClientInfo].appid; + pid_t pid_by_appid = mClientManager.find_client_pid_by_appid( + std::string{current_maclient_appid}); + if (pid != pid_by_appid) { + MAS_LOGE("[ERROR] Current client: [%d], active client : [%d %s]", + pid, pid_by_appid, current_maclient_appid); + return ret; + } + } switch(type) { case MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE: ret = mServicePlugin.start_streaming_utterance_data(); -- 2.7.4 From 26affd08fabbb73a92e13e430b5f9a352b5d9d3a Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 13 Aug 2020 14:32:28 +0900 Subject: [PATCH 15/16] Add more logs for tracking detailed streaming process Change-Id: Ic252bfc9af80f8fb49a882d49cc34f94e409d016 --- plugins/wakeup-manager/src/wakeup_engine_manager.cpp | 4 +++- plugins/wakeup-manager/src/wakeup_manager.cpp | 5 +++++ src/client_manager.cpp | 5 +++++ src/service_main.cpp | 2 ++ src/service_plugin.cpp | 1 + 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp index 0706120..2765498 100644 --- a/plugins/wakeup-manager/src/wakeup_engine_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_engine_manager.cpp @@ -270,8 +270,10 @@ void CWakeupEngineManager::streaming_speech_data_thread_func() { MWR_LOGI("[ENTER]"); - if (nullptr == mSelectedEngine) + if (nullptr == mSelectedEngine) { + MWR_LOGE("No Engine Selected"); return; + } const wakeup_engine_interface *interface = &(mSelectedEngine->interface); diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index d1331cb..5b04280 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -888,7 +888,9 @@ void CWakeupManager::set_dependency_module_command(string engine_name, string co void CWakeupManager::start_streaming_duration_timer() { + MWR_LOGI("DURATION_TIMER START"); ecore_main_loop_thread_safe_call_async([](void* data) { + MWR_LOGI("DURATION_TIMER START - async"); CWakeupManager* manager = static_cast(data); if (!manager) return; @@ -907,7 +909,9 @@ void CWakeupManager::stop_streaming_duration_timer() { MWR_LOGI("DURATION_TIMER STOP"); if (mStreamingDurationTimer) { + MWR_LOGI("DURATION_TIMER STOP - has timer"); ecore_main_loop_thread_safe_call_async([](void* data) { + MWR_LOGI("DURATION_TIMER STOP - async"); CWakeupManager* manager = static_cast(data); if (!manager) return; @@ -921,6 +925,7 @@ void CWakeupManager::stop_streaming_duration_timer() void CWakeupManager::set_streaming_duration_timer(Ecore_Timer* timer) { + MWR_LOGI("DURATION_TIMER SET : %p", timer); mStreamingDurationTimer = timer; } diff --git a/src/client_manager.cpp b/src/client_manager.cpp index 01cd9d5..18339df 100644 --- a/src/client_manager.cpp +++ b/src/client_manager.cpp @@ -38,6 +38,7 @@ int CClientManager::create_client(pid_t pid, std::string appid) data->pid = pid; data->appid = appid; mClientList = g_slist_append(mClientList, data); + MAS_LOGI("Create client : %s %d", appid.c_str(), pid); } else { MAS_LOGE("[ERROR] data is NULL"); return -1;// MA_ERROR_OUT_OF_MEMORY; @@ -74,6 +75,7 @@ int CClientManager::destroy_client_by_appid(std::string appid) ma_client_s* CClientManager::find_client_by_appid(std::string appid) { + MAS_LOGI("Looking for client with appid : %s", appid.c_str()); ma_client_s *data = NULL; int count = g_slist_length(mClientList); @@ -84,6 +86,7 @@ ma_client_s* CClientManager::find_client_by_appid(std::string appid) if (NULL != data) { if (0 == appid.compare(data->appid)) { + MAS_LOGI("Found client : %s %d", data->appid.c_str(), data->pid); return data; } } @@ -138,6 +141,7 @@ pid_t CClientManager::find_client_pid_by_index(unsigned int index) pid_t CClientManager::find_client_pid_by_appid(std::string appid) { + MAS_LOGI("Checking PID of appid : %s", appid.c_str()); pid_t pid = -1; if (nullptr == mApplicationManager) { @@ -147,6 +151,7 @@ pid_t CClientManager::find_client_pid_by_appid(std::string appid) ma_client_s* client = find_client_by_appid(appid); if (client) { + MAS_LOGI("Found client : %s %d", client->appid.c_str(), client->pid); bool running = mApplicationManager->is_application_running(client->pid); if (false == running) { MAS_LOGE("The PID for %s was %d, but it seems to be terminated", diff --git a/src/service_main.cpp b/src/service_main.cpp index a741ae0..bb377a2 100644 --- a/src/service_main.cpp +++ b/src/service_main.cpp @@ -199,6 +199,7 @@ int CServiceMain::client_send_recognition_result(pid_t pid, int result) int CServiceMain::client_start_streaming_audio_data(pid_t pid, int type) { + MAS_LOGI("[ENTER"); int ret = -1; const char *current_maclient_appid = NULL; if (mCurrentClientInfo >= 0 && mCurrentClientInfo < MAX_MACLIENT_INFO_NUM) { @@ -748,6 +749,7 @@ pid_t CServiceMain::get_current_audio_processing_pid() pid_t CServiceMain::get_client_pid_by_appid(const char *appid) { + MAS_LOGI("Checking PID of appid : %s", appid); pid_t ret = -1; if (appid) { diff --git a/src/service_plugin.cpp b/src/service_plugin.cpp index 254fb8e..7f26d00 100644 --- a/src/service_plugin.cpp +++ b/src/service_plugin.cpp @@ -1020,6 +1020,7 @@ int CServicePlugin::process_event(int event, void *data, int len) int CServicePlugin::start_streaming_utterance_data(void) { + MAS_LOGI("[ENTER"); int ret = -1; if (NULL != mPluginHandle) { wakeup_manager_start_streaming_utterance_data func = mWakeupManagerInterface.start_streaming_utterance_data; -- 2.7.4 From d8d75be5cdb3ebe1b55d33426fbcb38bcc7d79a9 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 13 Aug 2020 14:34:31 +0900 Subject: [PATCH 16/16] Bump version to 0.3.7 Change-Id: I7d91386600cd93aab861f3b68644a607c0c978d5 --- org.tizen.multi-assistant-service.xml | 2 +- packaging/org.tizen.multi-assistant-service.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.tizen.multi-assistant-service.xml b/org.tizen.multi-assistant-service.xml index 7ac0e5a..2f8a2db 100644 --- a/org.tizen.multi-assistant-service.xml +++ b/org.tizen.multi-assistant-service.xml @@ -1,5 +1,5 @@ - + Won Nam Jang Sooyeon Kim diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index 6957ade..80dfbcb 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -1,6 +1,6 @@ Name: org.tizen.multi-assistant-service Summary: Multi assistant service -Version: 0.3.6 +Version: 0.3.7 Release: 1 Group: Graphics & UI Framework/Voice Framework License: Flora-1.1 -- 2.7.4