Implement previous_utterance and follow_up steaming

Change-Id: Icd234173ca5f45aacaf14ed7255afb207851f4f2
Signed-off-by: Xie Ligang <ligang0.xie@samsung.com>

diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp
index 75f21e6..e9f0ddb 100644 (file)
--- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp
+++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp
@@ -274,9 +274,12 @@ void CAudioManager::finalize_audio_data()
 void CAudioManager::clear_audio_data()
 {
        lock_guard<mutex> lock(mMutex);
-
-       for (const auto &data : mAudioData) {
-               if (data.data.buffer) free(data.data.buffer);
+       while(!mAudioData.empty()) {
+               const auto &front = mAudioData.front();
+               if (front.data.buffer) {
+                       vm_free_simple(front.data.buffer);
+               }
+               mAudioData.pop_front();
        }
        mAudioData.clear();
 }
@@ -325,6 +328,7 @@ void CAudioManager::start_streaming_previous_utterance_data()
                MWR_LOGE("ERROR : mStreamingPreviousThread is joinable, will not start a new thread");
                return;
        }
+       lock_guard<mutex> lock(mMutex);
        mStreamingPreviousThread = thread(&CAudioManager::streaming_previous_audio_data_thread_func, this);
 }
 
@@ -344,10 +348,22 @@ void CAudioManager::stop_streaming_previous_utterance_data()
 
 void CAudioManager::start_streaming_follow_up_data()
 {
+       if (mStreamingThread.joinable()) {
+               MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread");
+               return;
+       }
+
+       mStreamingThread = thread(&CAudioManager::streaming_audio_data_thread_func, this, 0);
 }
 
 void CAudioManager::stop_streaming_follow_up_data()
 {
+       if (mStreamingThread.joinable()) {
+               MWR_LOGD("mStreamingThread is joinable, trying join()");
+               mStopStreamingThread.store(true);
+               mStreamingThread.join();
+       }
+       mStopStreamingThread.store(false);
 }
 
 void CAudioManager::change_system_volume()

diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp
index bd7205a..9ffe0fb 100644 (file)
--- a/plugins/wakeup-manager/src/wakeup_manager.cpp
+++ b/plugins/wakeup-manager/src/wakeup_manager.cpp
@@ -382,6 +382,8 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len)
        // LOCK REQUIRED
        if (MA_PLUGIN_EVENT_VOICE_KEY_PRESSED == event) {
                if (mVoiceKeyPressed != true) {
+                       mAudioManager.stop_recording();
+
                        stop_streaming_utterance_data();
                        stop_streaming_previous_utterance_data();
                        stop_streaming_follow_up_data();
@@ -391,7 +393,7 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len)
 
                        mVoiceKeyPressed = true;
 
-                       /* (Re)Start recorder thread using appropriate recorder */
+                       /* Start recorder thread using appropriate recording device */
                        mAudioManager.start_recording();
 
                        /* Wakeup default assistant */
@@ -418,7 +420,7 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len)
                        mAudioManager.finalize_audio_data();
 
                        if (mWakeupEngineManager.get_audio_data_required()) {
-                               /* Restart recorder thread using standard mic */
+                               /* Restart recorder thread using appropriate recording device */
                                mAudioManager.stop_recording();
                                mAudioManager.start_recording();
                        } else {
@@ -479,6 +481,9 @@ static Eina_Bool streaming_duration_expired(void *data)
        }
        wakeup_manager->set_streaming_mode(STREAMING_MODE::NONE);
 
+       if (WAKEUP_MANAGER_STATE_UTTERANCE == wakeup_manager->get_manager_state()) {
+               wakeup_manager->change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
+       }
        return ECORE_CALLBACK_CANCEL;
 }
 
@@ -518,6 +523,7 @@ bool CWakeupManager::start_streaming_utterance_data()
 bool CWakeupManager::stop_streaming_utterance_data()
 {
        MWR_LOGD("[ENTER]");
+       if (STREAMING_MODE::UTTERANCE != mStreamingMode) return false;
 
        if (mStreamingDurationTimer) {
                ecore_thread_main_loop_begin();
@@ -541,8 +547,29 @@ bool CWakeupManager::start_streaming_follow_up_data()
 {
        MWR_LOGD("[ENTER]");
 
+       mAudioManager.stop_streaming_follow_up_data();
+       mWakeupEngineManager.stop_streaming_current_utterance_data();
+
        mStreamingMode = STREAMING_MODE::FOLLOW_UP;
 
+       /* For the follow up streaming, audio data should be recorded from now on */
+       mAudioManager.stop_recording();
+       mAudioManager.clear_audio_data();
+       change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE);
+       mAudioManager.start_recording();
+
+       mAudioManager.start_streaming_follow_up_data();
+
+       ecore_thread_main_loop_begin();
+       if (mStreamingDurationTimer) {
+               ecore_timer_del(mStreamingDurationTimer);
+               mStreamingDurationTimer = nullptr;
+       }
+       mStreamingDurationTimer = ecore_timer_add(
+               mWakeupSettings.get_streaming_duration_max(),
+               streaming_duration_expired, this);
+       ecore_thread_main_loop_end();
+
        MWR_LOGD("[END]");
        return true;
 }
@@ -550,9 +577,26 @@ bool CWakeupManager::start_streaming_follow_up_data()
 bool CWakeupManager::stop_streaming_follow_up_data()
 {
        MWR_LOGD("[ENTER]");
+       if (STREAMING_MODE::FOLLOW_UP != mStreamingMode) return false;
+
+       if (mStreamingDurationTimer) {
+               ecore_thread_main_loop_begin();
+               ecore_timer_del(mStreamingDurationTimer);
+               mStreamingDurationTimer = nullptr;
+               ecore_thread_main_loop_end();
+       }
+       if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
+               change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
+       }
+       mAudioManager.stop_streaming_follow_up_data();
+       mWakeupEngineManager.stop_streaming_current_utterance_data();
 
        mStreamingMode = STREAMING_MODE::NONE;
 
+       mAudioManager.stop_recording();
+       mAudioManager.clear_audio_data();
+       mAudioManager.start_recording();
+
        MWR_LOGD("[END]");
        return true;
 }
@@ -561,7 +605,21 @@ bool CWakeupManager::start_streaming_previous_utterance_data()
 {
        MWR_LOGD("[ENTER]");
 
+       mAudioManager.stop_streaming_previous_utterance_data();
+       mWakeupEngineManager.stop_streaming_current_utterance_data();
+
        mStreamingMode = STREAMING_MODE::PREVIOUS_UTTERANCE;
+       mAudioManager.start_streaming_previous_utterance_data();
+
+       ecore_thread_main_loop_begin();
+       if (mStreamingDurationTimer) {
+               ecore_timer_del(mStreamingDurationTimer);
+               mStreamingDurationTimer = nullptr;
+       }
+       mStreamingDurationTimer = ecore_timer_add(
+               mWakeupSettings.get_streaming_duration_max(),
+               streaming_duration_expired, this);
+       ecore_thread_main_loop_end();
 
        MWR_LOGD("[END]");
        return true;
@@ -570,6 +628,19 @@ bool CWakeupManager::start_streaming_previous_utterance_data()
 bool CWakeupManager::stop_streaming_previous_utterance_data()
 {
        MWR_LOGD("[ENTER]");
+       if (STREAMING_MODE::PREVIOUS_UTTERANCE != mStreamingMode) return false;
+
+       if (mStreamingDurationTimer) {
+               ecore_thread_main_loop_begin();
+               ecore_timer_del(mStreamingDurationTimer);
+               mStreamingDurationTimer = nullptr;
+               ecore_thread_main_loop_end();
+       }
+       if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
+               change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
+       }
+       mAudioManager.stop_streaming_previous_utterance_data();
+       mWakeupEngineManager.stop_streaming_current_utterance_data();
 
        mStreamingMode = STREAMING_MODE::NONE;
 

diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c
index a1b1f54..c0e7e47 100644 (file)
--- a/src/multi_assistant_service.c
+++ b/src/multi_assistant_service.c
@@ -295,8 +295,7 @@ int mas_client_send_result(int pid, char* display_text, char* utterance_text, ch
                MAS_LOGE("[ERROR] Fail to send result, ret(%d)", ret);
        }
 
-       /* Should pass "RESULT_RECEIVED" event to wakeup manager */
-       multi_assistant_service_plugin_update_result_state(NULL, 0);
+       multi_assistant_service_plugin_update_result_state(NULL, MA_RECOGNITION_RESULT_EVENT_SUCCESS);
 
        return ret;
 }
@@ -309,7 +308,7 @@ int mas_client_send_recognition_result(int pid, int result)
                MAS_LOGE("[ERROR] Fail to send recognition result, ret(%d)", ret);
        }
 
-       multi_assistant_service_plugin_update_result_state(NULL, 0);
+       multi_assistant_service_plugin_update_result_state(NULL, result);
 
        return ret;
 }
@@ -317,7 +316,6 @@ int mas_client_send_recognition_result(int pid, int result)
 int mas_client_start_streaming_audio_data(int pid, int type)
 {
        int ret = -1;
-       /*
        switch(type) {
                case MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE:
                        ret = multi_assistant_service_plugin_start_streaming_utterance_data();
@@ -329,17 +327,12 @@ int mas_client_start_streaming_audio_data(int pid, int type)
                        ret = multi_assistant_service_plugin_start_streaming_follow_up_data();
                        break;
        }
-       */
-       //if (pid is activated assistant) {
-               ret = multi_assistant_service_plugin_start_streaming_utterance_data();
-       //}
        return ret;
 }
 
 int mas_client_stop_streaming_audio_data(int pid, int type)
 {
        int ret = -1;
-       /*
        switch(type) {
                case MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE:
                        ret = multi_assistant_service_plugin_stop_streaming_utterance_data();
@@ -351,8 +344,6 @@ int mas_client_stop_streaming_audio_data(int pid, int type)
                        ret = multi_assistant_service_plugin_stop_streaming_follow_up_data();
                        break;
        }
-       */
-       ret = multi_assistant_service_plugin_stop_streaming_utterance_data();
        return ret;
 }
 
@@ -409,9 +400,9 @@ int mas_ui_client_change_assistant(const char* appid)
        mas_set_current_client_by_appid(appid);
        int pid = mas_get_client_pid_by_appid(appid);
        if (pid != -1) {
-               masc_dbus_active_state_change(pid, MA_ACTIVE_STATE_ACTIVE);
                /* Bring MA client to foreground - is there a better way instead of launching? */
                mas_launch_client_by_appid(appid);
+               masc_dbus_active_state_change(pid, MA_ACTIVE_STATE_ACTIVE);
                MAS_LOGD("MA Client with appid %s exists, requesting speech data", (appid ? appid : "NULL"));
                /*
                int ret = multi_assistant_service_plugin_start_streaming_utterance_data();

diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c
index cd2fa8d..1c4028c 100644 (file)
--- a/src/multi_assistant_service_plugin.c
+++ b/src/multi_assistant_service_plugin.c
@@ -763,6 +763,44 @@ int multi_assistant_service_plugin_stop_streaming_utterance_data(void)
        return ret;
 }
 
+int multi_assistant_service_plugin_start_streaming_previous_utterance_data(void)
+{
+       int ret = -1;
+       if (NULL != g_handle) {
+               wakeup_manager_start_streaming_previous_utterance_data func = _wakeup_manager_interface.start_streaming_previous_utterance_data;
+               if (NULL == func) {
+                       MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_START_STREAMING_PREVIOUS_UTTERANCE_DATA);
+               } else {
+                       ret = func();
+                       if (0 != ret) {
+                               MAS_LOGE("[ERROR] Fail to request previous speech data, ret(%d)", ret);
+                       }
+               }
+       } else {
+               MAS_LOGE("[ERROR] g_handle is not valid");
+       }
+       return ret;
+}
+
+int multi_assistant_service_plugin_stop_streaming_previous_utterance_data(void)
+{
+       int ret = -1;
+       if (NULL != g_handle) {
+               wakeup_manager_stop_streaming_previous_utterance_data func = _wakeup_manager_interface.stop_streaming_previous_utterance_data;
+               if (NULL == func) {
+                       MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_STOP_STREAMING_PREVIOUS_UTTERANCE_DATA);
+               } else {
+                       ret = func();
+                       if (0 != ret) {
+                               MAS_LOGE("[ERROR] Fail to request previous speech data, ret(%d)", ret);
+                       }
+               }
+       } else {
+               MAS_LOGE("[ERROR] g_handle is not valid");
+       }
+       return ret;
+}
+
 int multi_assistant_service_plugin_start_streaming_follow_up_data(void)
 {
        int ret = -1;