From: Xie Ligang Date: Fri, 5 Jul 2019 06:01:32 +0000 (+0800) Subject: Implement previous_utterance and follow_up steaming X-Git-Tag: submit/tizen/20190821.051714~3 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F54%2F209354%2F8;p=platform%2Fcore%2Fuifw%2Fwakeup-engine-default.git Implement previous_utterance and follow_up steaming Change-Id: Icd234173ca5f45aacaf14ed7255afb207851f4f2 Signed-off-by: Xie Ligang --- diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp index 75f21e6..e9f0ddb 100644 --- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp @@ -274,9 +274,12 @@ void CAudioManager::finalize_audio_data() void CAudioManager::clear_audio_data() { lock_guard lock(mMutex); - - for (const auto &data : mAudioData) { - if (data.data.buffer) free(data.data.buffer); + while(!mAudioData.empty()) { + const auto &front = mAudioData.front(); + if (front.data.buffer) { + vm_free_simple(front.data.buffer); + } + mAudioData.pop_front(); } mAudioData.clear(); } @@ -325,6 +328,7 @@ void CAudioManager::start_streaming_previous_utterance_data() MWR_LOGE("ERROR : mStreamingPreviousThread is joinable, will not start a new thread"); return; } + lock_guard lock(mMutex); mStreamingPreviousThread = thread(&CAudioManager::streaming_previous_audio_data_thread_func, this); } @@ -344,10 +348,22 @@ void CAudioManager::stop_streaming_previous_utterance_data() void CAudioManager::start_streaming_follow_up_data() { + if (mStreamingThread.joinable()) { + MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread"); + return; + } + + mStreamingThread = thread(&CAudioManager::streaming_audio_data_thread_func, this, 0); } void CAudioManager::stop_streaming_follow_up_data() { + if (mStreamingThread.joinable()) { + MWR_LOGD("mStreamingThread is joinable, trying join()"); + mStopStreamingThread.store(true); + mStreamingThread.join(); + } + mStopStreamingThread.store(false); } void CAudioManager::change_system_volume() diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index bd7205a..9ffe0fb 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -382,6 +382,8 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) // LOCK REQUIRED if (MA_PLUGIN_EVENT_VOICE_KEY_PRESSED == event) { if (mVoiceKeyPressed != true) { + mAudioManager.stop_recording(); + stop_streaming_utterance_data(); stop_streaming_previous_utterance_data(); stop_streaming_follow_up_data(); @@ -391,7 +393,7 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) mVoiceKeyPressed = true; - /* (Re)Start recorder thread using appropriate recorder */ + /* Start recorder thread using appropriate recording device */ mAudioManager.start_recording(); /* Wakeup default assistant */ @@ -418,7 +420,7 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) mAudioManager.finalize_audio_data(); if (mWakeupEngineManager.get_audio_data_required()) { - /* Restart recorder thread using standard mic */ + /* Restart recorder thread using appropriate recording device */ mAudioManager.stop_recording(); mAudioManager.start_recording(); } else { @@ -479,6 +481,9 @@ static Eina_Bool streaming_duration_expired(void *data) } wakeup_manager->set_streaming_mode(STREAMING_MODE::NONE); + if (WAKEUP_MANAGER_STATE_UTTERANCE == wakeup_manager->get_manager_state()) { + wakeup_manager->change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING); + } return ECORE_CALLBACK_CANCEL; } @@ -518,6 +523,7 @@ bool CWakeupManager::start_streaming_utterance_data() bool CWakeupManager::stop_streaming_utterance_data() { MWR_LOGD("[ENTER]"); + if (STREAMING_MODE::UTTERANCE != mStreamingMode) return false; if (mStreamingDurationTimer) { ecore_thread_main_loop_begin(); @@ -541,8 +547,29 @@ bool CWakeupManager::start_streaming_follow_up_data() { MWR_LOGD("[ENTER]"); + mAudioManager.stop_streaming_follow_up_data(); + mWakeupEngineManager.stop_streaming_current_utterance_data(); + mStreamingMode = STREAMING_MODE::FOLLOW_UP; + /* For the follow up streaming, audio data should be recorded from now on */ + mAudioManager.stop_recording(); + mAudioManager.clear_audio_data(); + change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE); + mAudioManager.start_recording(); + + mAudioManager.start_streaming_follow_up_data(); + + ecore_thread_main_loop_begin(); + if (mStreamingDurationTimer) { + ecore_timer_del(mStreamingDurationTimer); + mStreamingDurationTimer = nullptr; + } + mStreamingDurationTimer = ecore_timer_add( + mWakeupSettings.get_streaming_duration_max(), + streaming_duration_expired, this); + ecore_thread_main_loop_end(); + MWR_LOGD("[END]"); return true; } @@ -550,9 +577,26 @@ bool CWakeupManager::start_streaming_follow_up_data() bool CWakeupManager::stop_streaming_follow_up_data() { MWR_LOGD("[ENTER]"); + if (STREAMING_MODE::FOLLOW_UP != mStreamingMode) return false; + + if (mStreamingDurationTimer) { + ecore_thread_main_loop_begin(); + ecore_timer_del(mStreamingDurationTimer); + mStreamingDurationTimer = nullptr; + ecore_thread_main_loop_end(); + } + if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { + change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING); + } + mAudioManager.stop_streaming_follow_up_data(); + mWakeupEngineManager.stop_streaming_current_utterance_data(); mStreamingMode = STREAMING_MODE::NONE; + mAudioManager.stop_recording(); + mAudioManager.clear_audio_data(); + mAudioManager.start_recording(); + MWR_LOGD("[END]"); return true; } @@ -561,7 +605,21 @@ bool CWakeupManager::start_streaming_previous_utterance_data() { MWR_LOGD("[ENTER]"); + mAudioManager.stop_streaming_previous_utterance_data(); + mWakeupEngineManager.stop_streaming_current_utterance_data(); + mStreamingMode = STREAMING_MODE::PREVIOUS_UTTERANCE; + mAudioManager.start_streaming_previous_utterance_data(); + + ecore_thread_main_loop_begin(); + if (mStreamingDurationTimer) { + ecore_timer_del(mStreamingDurationTimer); + mStreamingDurationTimer = nullptr; + } + mStreamingDurationTimer = ecore_timer_add( + mWakeupSettings.get_streaming_duration_max(), + streaming_duration_expired, this); + ecore_thread_main_loop_end(); MWR_LOGD("[END]"); return true; @@ -570,6 +628,19 @@ bool CWakeupManager::start_streaming_previous_utterance_data() bool CWakeupManager::stop_streaming_previous_utterance_data() { MWR_LOGD("[ENTER]"); + if (STREAMING_MODE::PREVIOUS_UTTERANCE != mStreamingMode) return false; + + if (mStreamingDurationTimer) { + ecore_thread_main_loop_begin(); + ecore_timer_del(mStreamingDurationTimer); + mStreamingDurationTimer = nullptr; + ecore_thread_main_loop_end(); + } + if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) { + change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING); + } + mAudioManager.stop_streaming_previous_utterance_data(); + mWakeupEngineManager.stop_streaming_current_utterance_data(); mStreamingMode = STREAMING_MODE::NONE; diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index a1b1f54..c0e7e47 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -295,8 +295,7 @@ int mas_client_send_result(int pid, char* display_text, char* utterance_text, ch MAS_LOGE("[ERROR] Fail to send result, ret(%d)", ret); } - /* Should pass "RESULT_RECEIVED" event to wakeup manager */ - multi_assistant_service_plugin_update_result_state(NULL, 0); + multi_assistant_service_plugin_update_result_state(NULL, MA_RECOGNITION_RESULT_EVENT_SUCCESS); return ret; } @@ -309,7 +308,7 @@ int mas_client_send_recognition_result(int pid, int result) MAS_LOGE("[ERROR] Fail to send recognition result, ret(%d)", ret); } - multi_assistant_service_plugin_update_result_state(NULL, 0); + multi_assistant_service_plugin_update_result_state(NULL, result); return ret; } @@ -317,7 +316,6 @@ int mas_client_send_recognition_result(int pid, int result) int mas_client_start_streaming_audio_data(int pid, int type) { int ret = -1; - /* switch(type) { case MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE: ret = multi_assistant_service_plugin_start_streaming_utterance_data(); @@ -329,17 +327,12 @@ int mas_client_start_streaming_audio_data(int pid, int type) ret = multi_assistant_service_plugin_start_streaming_follow_up_data(); break; } - */ - //if (pid is activated assistant) { - ret = multi_assistant_service_plugin_start_streaming_utterance_data(); - //} return ret; } int mas_client_stop_streaming_audio_data(int pid, int type) { int ret = -1; - /* switch(type) { case MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE: ret = multi_assistant_service_plugin_stop_streaming_utterance_data(); @@ -351,8 +344,6 @@ int mas_client_stop_streaming_audio_data(int pid, int type) ret = multi_assistant_service_plugin_stop_streaming_follow_up_data(); break; } - */ - ret = multi_assistant_service_plugin_stop_streaming_utterance_data(); return ret; } @@ -409,9 +400,9 @@ int mas_ui_client_change_assistant(const char* appid) mas_set_current_client_by_appid(appid); int pid = mas_get_client_pid_by_appid(appid); if (pid != -1) { - masc_dbus_active_state_change(pid, MA_ACTIVE_STATE_ACTIVE); /* Bring MA client to foreground - is there a better way instead of launching? */ mas_launch_client_by_appid(appid); + masc_dbus_active_state_change(pid, MA_ACTIVE_STATE_ACTIVE); MAS_LOGD("MA Client with appid %s exists, requesting speech data", (appid ? appid : "NULL")); /* int ret = multi_assistant_service_plugin_start_streaming_utterance_data(); diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index cd2fa8d..1c4028c 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -763,6 +763,44 @@ int multi_assistant_service_plugin_stop_streaming_utterance_data(void) return ret; } +int multi_assistant_service_plugin_start_streaming_previous_utterance_data(void) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_start_streaming_previous_utterance_data func = _wakeup_manager_interface.start_streaming_previous_utterance_data; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_START_STREAMING_PREVIOUS_UTTERANCE_DATA); + } else { + ret = func(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to request previous speech data, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + return ret; +} + +int multi_assistant_service_plugin_stop_streaming_previous_utterance_data(void) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_stop_streaming_previous_utterance_data func = _wakeup_manager_interface.stop_streaming_previous_utterance_data; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_STOP_STREAMING_PREVIOUS_UTTERANCE_DATA); + } else { + ret = func(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to request previous speech data, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + return ret; +} + int multi_assistant_service_plugin_start_streaming_follow_up_data(void) { int ret = -1;