From 5de277e219a2f973311441822ac34fd02778e43c Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 19 Mar 2019 20:52:12 +0900 Subject: [PATCH] Store background audio data for alternate way of streaming Change-Id: If9d6dd8a63a98354cdddb89caadced7a30ab6c7f --- inc/multi_wakeup_recognizer.h | 1 + .../wakeup-manager/inc/wakeup_audio_manager.h | 32 ++- .../wakeup-manager/inc/wakeup_interfaces.h | 1 + .../src/wakeup_audio_manager.cpp | 194 +++++++++++++++--- plugins/wakeup-manager/src/wakeup_manager.cpp | 93 ++++++++- src/multi_assistant_service_plugin.c | 2 +- 6 files changed, 280 insertions(+), 43 deletions(-) diff --git a/inc/multi_wakeup_recognizer.h b/inc/multi_wakeup_recognizer.h index ba70612..df8df1a 100644 --- a/inc/multi_wakeup_recognizer.h +++ b/inc/multi_wakeup_recognizer.h @@ -34,6 +34,7 @@ typedef struct { long wakeup_start_time; long wakeup_end_time; + bool wakeup_time_valid; const void *extra_data; int extra_data_length; diff --git a/plugins/wakeup-manager/inc/wakeup_audio_manager.h b/plugins/wakeup-manager/inc/wakeup_audio_manager.h index 4e73a89..1aeb0ac 100644 --- a/plugins/wakeup-manager/inc/wakeup_audio_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_audio_manager.h @@ -21,6 +21,7 @@ #include "wakeup_interfaces.h" #include +#include #include #include @@ -59,19 +60,27 @@ public: void start_recording(); void stop_recording(); - void clear_audio_data(); - void finalize_audio_data(); + void add_speech_data(wakeup_speech_data& data); + void clear_speech_data(); + void finalize_speech_data(); - void start_streaming(); - void stop_streaming(); + void start_streaming_current_utterance_data(bool from_start_time = false, long start_time = 0); + void stop_streaming_current_utterance_data(); + + void start_streaming_previous_utterance_data(); + void stop_streaming_previous_utterance_data(); + + void start_streaming_follow_up_data(); + void stop_streaming_follow_up_data(); void voice_key_pressed_set(bool pressed); bool voice_key_pressed_get(); - - void add_custom_speech_data(wakeup_speech_data& data); private: void recorder_thread_func(void); - void streaming_thread_func(void); + void streaming_speech_data_thread_func(); + void streaming_background_data_thread_func(long start_time); + + long get_current_milliseconds_after_epoch(); std::vector mObservers; @@ -84,7 +93,14 @@ private: std::thread mStreamingThread; std::atomic_bool mStopStreamingThread{false}; - std::vector mSpeechData; + static constexpr long mBackgroundRecordingDurationMilliseconds = 10 * 1000; + typedef struct { + long time; + wakeup_speech_data data; + } wakeup_speech_data_with_time; + std::vector mSpeechData; + std::vector mPreviousSpeechData; + std::list mBackgroundData; bool mVoiceKeyPressed{false}; }; diff --git a/plugins/wakeup-manager/inc/wakeup_interfaces.h b/plugins/wakeup-manager/inc/wakeup_interfaces.h index a0da3b0..7303ca6 100644 --- a/plugins/wakeup-manager/inc/wakeup_interfaces.h +++ b/plugins/wakeup-manager/inc/wakeup_interfaces.h @@ -26,6 +26,7 @@ typedef struct { long wakeup_start_time; long wakeup_end_time; + bool wakeup_time_valid; const void *extra_data; int extra_data_length; diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp index ffea203..be7bd64 100644 --- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp @@ -52,7 +52,7 @@ static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void data.buffer = malloc(voice_data->length); if (data.buffer) { memcpy(data.buffer, voice_data->audio_buf, voice_data->length); - manager->add_custom_speech_data(data); + manager->add_speech_data(data); } } else { MWR_LOGE("[Recorder ERROR] voice key seems to be already released"); @@ -146,6 +146,7 @@ int CAudioManager::initialize(void) MWR_LOGD("[Recorder] Bluetooth is available"); } #endif + return 0; } int CAudioManager::deinitialize(void) @@ -153,7 +154,7 @@ int CAudioManager::deinitialize(void) MWR_LOGD("[ENTER]"); for (const auto &data : mSpeechData) { - free(data.buffer); + free(data.data.buffer); } mSpeechData.clear(); @@ -204,19 +205,9 @@ void CAudioManager::recorder_thread_func() while (!(mStopRecorderThread.load())) { unsigned char buffer[BUFFER_LENGTH]; - int ret; memset(buffer, '\0', BUFFER_LENGTH); - auto now = std::chrono::system_clock::now(); - auto now_ms = std::chrono::time_point_cast(now); - /* number of milliseconds since the epoch of system_clock */ - auto value = now_ms.time_since_epoch(); - - static long time = 0; - if (time == value.count()) { - LOGE("[Recorder WARNING] Time value duplicated : %lu", time); - } - time = value.count(); + long time = get_current_milliseconds_after_epoch(); int read_bytes = audio_in_read(mAudioIn, buffer, BUFFER_LENGTH); if (0 > read_bytes) { @@ -233,6 +224,33 @@ void CAudioManager::recorder_thread_func() } } + long delta = mBackgroundRecordingDurationMilliseconds; + if (mBackgroundData.size() > 0) { + while(mBackgroundData.size() > 0 && mBackgroundData.front().time < time - delta) { + const auto &front = mBackgroundData.front(); + if (front.data.buffer) { + free(front.data.buffer); + } + mBackgroundData.pop_front(); + if (0 == buffer_count % 100) { + LOGD("[Recorder] list pop_front (%zu)", mBackgroundData.size()); + } + } + } + + wakeup_speech_data_with_time data; + data.data.buffer = malloc(read_bytes); + if (data.data.buffer) { + data.time = time; + data.data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE; + data.data.len = read_bytes; + memcpy(data.data.buffer, buffer, read_bytes); + mBackgroundData.push_back(data); + if (0 == buffer_count % 100) { + LOGD("[Recorder] list push_back (%zu), %ld %ld %ld", mBackgroundData.size(), mBackgroundData.front().time, time, time - delta); + } + } + // UNLOCK REQUIRED /* Audio read log */ if (0 == buffer_count % 100) { @@ -299,13 +317,12 @@ void CAudioManager::start_recording() } -void CAudioManager::streaming_thread_func(void) +void CAudioManager::streaming_speech_data_thread_func() { MWR_LOGD("[ENTER]"); - MWR_LOGD("data_count : %zu", mSpeechData.size()); - int index = 0; + MWR_LOGD("data_count : %zu", mSpeechData.size()); while (!(mStopStreamingThread.load())) { int ret = -1; @@ -343,7 +360,7 @@ void CAudioManager::streaming_thread_func(void) continue; } - wakeup_speech_data &speech_data = mSpeechData.at(index); + wakeup_speech_data& speech_data = mSpeechData.at(index).data; for (const auto& observer : mObservers) { if (observer) { if (!observer->on_streaming_audio_data( @@ -355,6 +372,8 @@ void CAudioManager::streaming_thread_func(void) if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) { MWR_LOGI("[INFO] Finish to get and send speech data"); + /* Now move all the speech data to previous speech data for later use */ + mPreviousSpeechData = std::move(mSpeechData); break; } @@ -362,15 +381,104 @@ void CAudioManager::streaming_thread_func(void) } } -void CAudioManager::clear_audio_data() +void CAudioManager::streaming_background_data_thread_func(long start_time) +{ + MWR_LOGD("[ENTER]"); + + auto lead = mBackgroundData.begin(); + auto iter = lead; + while (lead != mBackgroundData.end() && lead->time < start_time) { + iter = lead; + std::advance(lead, 1); + } + + MWR_LOGD("data_count : %zu", mBackgroundData.size()); + + while (!(mStopStreamingThread.load())) { + int ret = -1; + int cnt = 0; + + /* get feedback data */ + if (lead == mBackgroundData.end()) { + /* empty queue */ + MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret); + + /* waiting */ + while (1) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + if (iter == mBackgroundData.end()) { + iter = mBackgroundData.begin(); + } + lead = iter; + if (lead != mBackgroundData.end()) { + std::advance(lead, 1); + } + if (lead != mBackgroundData.end()) { + MWR_LOGI("[INFO] Resume thread"); + break; + } + if (200 < cnt) { + MWR_LOGE("[ERROR] Wrong request, there's no pcm data"); + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_streaming_audio_data( + WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + return; + } + cnt++; + } + MWR_LOGI("[INFO] Finish to wait for new feedback data come"); + + /* resume feedback thread */ + continue; + } + + iter = lead; + /* Extracted background data will be used as previous utterance data*/ + mSpeechData.push_back(*iter); + + wakeup_speech_data& speech_data = iter->data; + for (const auto& observer : mObservers) { + if (observer) { + if (!observer->on_streaming_audio_data( + speech_data.event, speech_data.buffer, speech_data.len)) { + LOGE("[Recorder WARNING] One of the observer returned false"); + } + } + } + + if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) { + MWR_LOGI("[INFO] Finish to get and send speech data"); + /* Now move all the speech data to previous speech data for later use */ + mPreviousSpeechData = std::move(mSpeechData); + break; + } + + std::advance(lead, 1); + } +} + +void CAudioManager::add_speech_data(wakeup_speech_data& speech_data) +{ + wakeup_speech_data_with_time data; + data.data = speech_data; + data.time = get_current_milliseconds_after_epoch(); + mSpeechData.push_back(data); +} + +void CAudioManager::clear_speech_data() { - for (const auto &speech_data : mSpeechData) { - if (speech_data.buffer) free(speech_data.buffer); + for (const auto &data : mSpeechData) { + if (data.data.buffer) free(data.data.buffer); } mSpeechData.clear(); } -void CAudioManager::finalize_audio_data() +void CAudioManager::finalize_speech_data() { unsigned char final_buffer[2] = {'\0', }; wakeup_speech_data speech_data; @@ -379,16 +487,24 @@ void CAudioManager::finalize_audio_data() speech_data.buffer = malloc(speech_data.len); if (speech_data.buffer) { memcpy(speech_data.buffer, final_buffer, speech_data.len); - mSpeechData.push_back(speech_data); + wakeup_speech_data_with_time data; + data.data = speech_data; + data.time = get_current_milliseconds_after_epoch(); + mSpeechData.push_back(data); } } -void CAudioManager::start_streaming() +void CAudioManager::start_streaming_current_utterance_data(bool from_start_time, long start_time) { - mStreamingThread = std::thread(&CAudioManager::streaming_thread_func, this); + if (from_start_time) { + mSpeechData.clear(); + mStreamingThread = std::thread(&CAudioManager::streaming_background_data_thread_func, this, start_time); + } else { + mStreamingThread = std::thread(&CAudioManager::streaming_speech_data_thread_func, this); + } } -void CAudioManager::stop_streaming() +void CAudioManager::stop_streaming_current_utterance_data() { if (mStreamingThread.joinable()) { MWR_LOGD("mStreamingThread is joinable, trying join()"); @@ -396,6 +512,25 @@ void CAudioManager::stop_streaming() mStreamingThread.join(); } mStopStreamingThread.store(false); + + /* Now move all the speech data to previous speech data for later use */ + mPreviousSpeechData = std::move(mSpeechData); +} + +void CAudioManager::start_streaming_previous_utterance_data() +{ +} + +void CAudioManager::stop_streaming_previous_utterance_data() +{ +} + +void CAudioManager::start_streaming_follow_up_data() +{ +} + +void CAudioManager::stop_streaming_follow_up_data() +{ } void CAudioManager::voice_key_pressed_set(bool pressed) @@ -413,9 +548,14 @@ bool CAudioManager::voice_key_pressed_get() return mVoiceKeyPressed; } -void CAudioManager::add_custom_speech_data(wakeup_speech_data& data) +long CAudioManager::get_current_milliseconds_after_epoch() { - mSpeechData.push_back(data); + auto now = std::chrono::system_clock::now(); + auto now_ms = std::chrono::time_point_cast(now); + /* number of milliseconds since the epoch of system_clock */ + auto value = now_ms.time_since_epoch(); + + return value.count(); } } // wakeup diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 90e68e7..fd2b196 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -88,6 +88,8 @@ static wakeup_engine_info g_wakeup_engine_info[MAX_WAKEUP_ENGINE_NUM]; static char* g_current_language = NULL; static wakeup_manager_state_e g_wakeup_manager_state; +static wakeup_event_info g_last_wakeup_event_info; + #define DEFAULT_ASSISTANT_APPID "com.samsung.bixby-voice" #define WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID "db/multi-assistant/default_assistant_appid" @@ -97,6 +99,7 @@ static wakeup_manager_state_e g_wakeup_manager_state; #define WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS "db/multi-assistant/enabled_assistants" #define WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_DELAY "db/multi-assistant/wakeup_policy_delay" #define WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_PRIORITY "db/multi-assistant/wakeup_policy_priority" +#define WAKEUP_SETTINGS_KEY_STREAMING_DURATION_MAX "db/multi-assistant/streaming_duration_max" typedef struct { std::string default_assistant_appid{DEFAULT_ASSISTANT_APPID}; @@ -106,10 +109,21 @@ typedef struct { std::vector enabled_assistants{DEFAULT_ASSISTANT_APPID}; float wakeup_policy_delay{0.1}; std::vector wakeup_policy_priority; // No priority by default + float streaming_duration_max{10.0}; } wakeup_settings; static wakeup_settings g_wakeup_settings; +enum STREAMING_MODE { + STREAMING_MODE_NONE, + STREAMING_MODE_UTTERANCE, + STREAMING_MODE_PREVIOUS_UTTERANCE, + STREAMING_MODE_FOLLOW_UP, +}; + +static STREAMING_MODE g_streaming_mode{STREAMING_MODE_NONE}; +static Ecore_Timer* g_streaming_duration_timer; + class CWakeupEventObserver : public multiassistant::wakeup::IWakeupEventObserver { public: @@ -437,6 +451,7 @@ static int wakeup_engine_info_initialize() /* We'll need to check vconf for enabled wakeup engines */ g_wakeup_engine_info[loop].enabled = true; g_wakeup_engine_info[loop].audio_data_require_status = false; + MWR_LOGD("Initializing wakeup engine : %s %p", g_wakeup_engine_info[loop].engine_path, g_wakeup_engine_info[loop].interface.initialize); if (g_wakeup_engine_info[loop].interface.initialize) { g_wakeup_engine_info[loop].interface.initialize(); @@ -464,6 +479,7 @@ static int wakeup_engine_info_initialize() void CWakeupEventObserver::on_wakeup(wakeup_event_info info) { if (NULL != g_wakeup_event_cb) { + g_last_wakeup_event_info = info; g_wakeup_event_cb(info, g_wakeup_event_user_data); } } @@ -567,6 +583,11 @@ int wakeup_manager_initialize(void) free(vconf_str); vconf_str = nullptr; } + vconf_ret = vconf_get_dbl(WAKEUP_SETTINGS_KEY_STREAMING_DURATION_MAX, &vconf_double); + if (0 == vconf_ret) { + g_wakeup_settings.streaming_duration_max = vconf_double; + MWR_LOGD("streaming_duration_max : %f", g_wakeup_settings.streaming_duration_max); + } wakeup_policy_initialize(); @@ -730,6 +751,7 @@ int wakeup_manager_change_state(wakeup_manager_state_e state) g_wakeup_engine_info[loop].interface.update_manager_state(state); } } + return 0; } int wakeup_manager_activate(void) @@ -846,7 +868,7 @@ int wakeup_manager_process_event(int event, void* data, int len) if (event == MA_PLUGIN_EVENT_VOICE_KEY_PRESSED) { if (g_voice_key_pressed != true) { /* Clear all existing data */ - g_audio_manager.clear_audio_data(); + g_audio_manager.clear_speech_data(); g_voice_key_pressed = true; /* (Re)Start recorder thread using bt hid */ @@ -864,7 +886,7 @@ int wakeup_manager_process_event(int event, void* data, int len) } else if (event == MA_PLUGIN_EVENT_VOICE_KEY_RELEASED) { if (g_voice_key_pressed != false) { g_voice_key_pressed = false; - g_audio_manager.finalize_audio_data(); + g_audio_manager.finalize_speech_data(); if (g_audio_data_required == true) { /* Restart recorder thread using standard mic */ g_audio_manager.start_recording(); @@ -975,19 +997,58 @@ void join_engine_data_thread() g_engine_data_thread_should_stop.store(false); } +static Eina_Bool streaming_duration_expired(void *data) +{ + MWR_LOGD("[ENTER]"); + switch(g_streaming_mode) { + case STREAMING_MODE_UTTERANCE: + g_audio_manager.stop_streaming_current_utterance_data(); + join_engine_data_thread(); + break; + case STREAMING_MODE_PREVIOUS_UTTERANCE: + g_audio_manager.stop_streaming_previous_utterance_data(); + break; + case STREAMING_MODE_FOLLOW_UP: + g_audio_manager.stop_streaming_follow_up_data(); + break; + } + + unsigned char final_buffer[2] = {'\0', }; + __wakeup_service_streaming_cb( + WAKEUP_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer)); + + g_streaming_mode = STREAMING_MODE_NONE; + return ECORE_CALLBACK_CANCEL; +} + int wakeup_manager_start_streaming_utterance_data(void) { MWR_LOGD("[ENTER]"); - g_audio_manager.stop_streaming(); + g_streaming_mode = STREAMING_MODE_UTTERANCE; + + g_audio_manager.stop_streaming_current_utterance_data(); join_engine_data_thread(); + /* What if the user pressed voice key but then again immediately releases? */ if (g_voice_key_pressed) { - g_audio_manager.start_streaming(); + g_audio_manager.start_streaming_current_utterance_data(); } else { - start_engine_data_thread(); + if(g_last_wakeup_event_info.wakeup_time_valid) { + g_audio_manager.start_streaming_current_utterance_data(true, g_last_wakeup_event_info.wakeup_end_time); + } else { + start_engine_data_thread(); + } } + ecore_thread_main_loop_begin(); + if (g_streaming_duration_timer) { + ecore_timer_del(g_streaming_duration_timer); + } + g_streaming_duration_timer = ecore_timer_add(g_wakeup_settings.streaming_duration_max, + streaming_duration_expired, nullptr); + ecore_thread_main_loop_end(); + MWR_LOGD("[END]"); return 0; } @@ -995,11 +1056,17 @@ int wakeup_manager_start_streaming_utterance_data(void) int wakeup_manager_stop_streaming_utterance_data(void) { MWR_LOGD("[ENTER]"); + if (g_streaming_duration_timer) { + ecore_thread_main_loop_begin(); + ecore_timer_del(g_streaming_duration_timer); + ecore_thread_main_loop_end(); + } if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_UTTERANCE) { wakeup_manager_change_state(WAKEUP_MANAGER_STATE_PROCESSING); - g_audio_manager.stop_streaming(); - join_engine_data_thread(); } + g_audio_manager.stop_streaming_current_utterance_data(); + join_engine_data_thread(); + g_streaming_mode = STREAMING_MODE_NONE; MWR_LOGD("[END]"); return 0; } @@ -1008,6 +1075,8 @@ int wakeup_manager_start_streaming_follow_up_data(void) { MWR_LOGD("[ENTER]"); + g_streaming_mode = STREAMING_MODE_FOLLOW_UP; + MWR_LOGD("[END]"); return 0; } @@ -1016,6 +1085,8 @@ int wakeup_manager_stop_streaming_follow_up_data(void) { MWR_LOGD("[ENTER]"); + g_streaming_mode = STREAMING_MODE_NONE; + MWR_LOGD("[END]"); return 0; } @@ -1024,6 +1095,8 @@ int wakeup_manager_start_streaming_previous_utterance_data(void) { MWR_LOGD("[ENTER]"); + g_streaming_mode = STREAMING_MODE_PREVIOUS_UTTERANCE; + MWR_LOGD("[END]"); return 0; } @@ -1032,6 +1105,8 @@ int wakeup_manager_stop_streaming_previous_utterance_data(void) { MWR_LOGD("[ENTER]"); + g_streaming_mode = STREAMING_MODE_NONE; + MWR_LOGD("[END]"); return 0; } @@ -1169,4 +1244,8 @@ bool CAudioDataObserver::on_streaming_audio_data( wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) { __wakeup_service_streaming_cb(event, buffer, len); + if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == event) { + g_streaming_mode = STREAMING_MODE_NONE; + } + return true; } diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index a74099d..0b55e8f 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -548,7 +548,7 @@ int multi_assistant_service_plugin_send_assistant_specific_command(const char* a { int ret = -1; if (NULL != g_handle) { - wakeup_manager_send_assistant_specific_command func = _wakeup_manager_interface.send_assistant_specific_command; + wakeup_manager_send_assistant_specific_command func = _wakeup_manager_interface.send_assistant_specific_command; if (NULL == func) { MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SEND_ASSISTANT_SPECIFIC_COMMAND); } else { -- 2.34.1