From a2d6238596af0301a87b57e1d5b07cf8c3517ccc Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Tue, 20 Aug 2019 20:19:06 +0900 Subject: [PATCH] Provide session information before start recording Change-Id: Ifd314ffdf7aa608074f2ca36b749da21f089030b --- .../inc/dependency_default.h | 12 ++++++++++ .../inc/dependency_default_audio.h | 2 ++ .../src/dependency_default.cpp | 6 +++++ .../src/dependency_default_audio.cpp | 12 +++++++++- .../dependency-tv/inc/dependency_tv.h | 8 +++++++ .../dependency-tv/inc/dependency_tv_audio.h | 2 ++ .../dependency-tv/src/dependency_tv.cpp | 6 +++++ .../dependency-tv/src/dependency_tv_audio.cpp | 9 +++++++ .../wakeup-manager/inc/dependency_resolver.h | 4 ++++ .../wakeup-manager/inc/wakeup_audio_manager.h | 9 +++++++ .../src/dependency_resolver.cpp | 24 +++++++++++++++++++ .../src/wakeup_audio_manager.cpp | 5 ++++ plugins/wakeup-manager/src/wakeup_manager.cpp | 7 ++++++ 13 files changed, 105 insertions(+), 1 deletion(-) diff --git a/plugins/wakeup-manager/dependency-default/inc/dependency_default.h b/plugins/wakeup-manager/dependency-default/inc/dependency_default.h index 3484fd5..0c536d9 100644 --- a/plugins/wakeup-manager/dependency-default/inc/dependency_default.h +++ b/plugins/wakeup-manager/dependency-default/inc/dependency_default.h @@ -20,6 +20,10 @@ #include +#ifndef LOG_TAG +#define LOG_TAG "dependency_default" +#endif + #ifdef __cplusplus extern "C" { #endif @@ -38,6 +42,13 @@ typedef enum { MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_TAP, } ma_plugin_event_e; +typedef enum +{ + RECORDING_SESSION_WAKE_WORD, + RECORDING_SESSION_UTTERANCE, + RECORDING_SESSION_FOLLOW_UP, +} recording_session; + typedef void (*mas_dependency_error_cb)(int error, const char* err_msg, void* user_data); typedef int (*mas_proxy_process_event)(int event, void* data, int len); @@ -53,6 +64,7 @@ EXPORT_API int mas_dependency_deinitialize(void); EXPORT_API int mas_dependency_set_error_callback(mas_dependency_error_cb callback, void* user_data); EXPORT_API int mas_dependency_start_recording(void); EXPORT_API int mas_dependency_stop_recording(void); +EXPORT_API int mas_dependency_set_recording_session(unsigned int session); EXPORT_API int mas_dependency_change_system_volume(void); EXPORT_API int mas_dependency_recover_system_volume(void); EXPORT_API int mas_dependency_get_audio_format(int* rate, int* channel, int* audio_type); diff --git a/plugins/wakeup-manager/dependency-default/inc/dependency_default_audio.h b/plugins/wakeup-manager/dependency-default/inc/dependency_default_audio.h index 9ba14a0..2a40379 100644 --- a/plugins/wakeup-manager/dependency-default/inc/dependency_default_audio.h +++ b/plugins/wakeup-manager/dependency-default/inc/dependency_default_audio.h @@ -23,6 +23,8 @@ void dependency_default_audio_deinitialize(); void dependency_default_audio_start_recording(); void dependency_default_audio_stop_recording(); +void dependency_default_audio_set_recording_session(unsigned int session); + void dependency_default_audio_voice_key_pressed_set(bool pressed); void dependency_default_audio_change_system_volume(); diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp index d705b00..5456e84 100644 --- a/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default.cpp @@ -45,6 +45,12 @@ int mas_dependency_stop_recording(void) return 0; } +int mas_dependency_set_recording_session(unsigned int session) +{ + dependency_default_audio_set_recording_session(session); + return 0; +} + int mas_dependency_change_system_volume(void) { dependency_default_audio_change_system_volume(); diff --git a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp index afb3e99..b2c231f 100644 --- a/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp +++ b/plugins/wakeup-manager/dependency-default/src/dependency_default_audio.cpp @@ -284,6 +284,16 @@ void dependency_default_audio_stop_recording() } } +void dependency_default_audio_set_recording_session(unsigned int session) +{ + LOGD("Current recording session : [%s]", + (RECORDING_SESSION_WAKE_WORD == session) ? "wake word detecting" : + (RECORDING_SESSION_UTTERANCE == session) ? "utterance recording" : + (RECORDING_SESSION_FOLLOW_UP == session) ? "follow-up speech recording" : + "ERROR - Invalid recording session information" + ); +} + void dependency_default_audio_voice_key_pressed_set(bool pressed) { g_voice_key_pressed = pressed; @@ -370,4 +380,4 @@ void dependency_default_audio_get_audio_source_type(char** type) return; } *type = source_type; -} \ No newline at end of file +} diff --git a/plugins/wakeup-manager/dependency-tv/inc/dependency_tv.h b/plugins/wakeup-manager/dependency-tv/inc/dependency_tv.h index ef725eb..fddc4e6 100644 --- a/plugins/wakeup-manager/dependency-tv/inc/dependency_tv.h +++ b/plugins/wakeup-manager/dependency-tv/inc/dependency_tv.h @@ -40,6 +40,13 @@ typedef enum { MA_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_TAP, } ma_plugin_event_e; +typedef enum +{ + RECORDING_SESSION_WAKE_WORD, + RECORDING_SESSION_UTTERANCE, + RECORDING_SESSION_FOLLOW_UP, +} recording_session; + typedef void (*mas_dependency_error_cb)(int error, const char* err_msg, void* user_data); typedef int (*mas_proxy_process_event)(int event, void* data, int len); @@ -82,6 +89,7 @@ EXPORT_API int mas_dependency_deinitialize(void); EXPORT_API int mas_dependency_set_error_callback(mas_dependency_error_cb callback, void* user_data); EXPORT_API int mas_dependency_start_recording(void); EXPORT_API int mas_dependency_stop_recording(void); +EXPORT_API int mas_dependency_set_recording_session(unsigned int session); EXPORT_API int mas_dependency_change_system_volume(void); EXPORT_API int mas_dependency_recover_system_volume(void); EXPORT_API int mas_dependency_get_audio_format(int* rate, int* channel, int* audio_type); diff --git a/plugins/wakeup-manager/dependency-tv/inc/dependency_tv_audio.h b/plugins/wakeup-manager/dependency-tv/inc/dependency_tv_audio.h index fb5437e..a0b9d24 100644 --- a/plugins/wakeup-manager/dependency-tv/inc/dependency_tv_audio.h +++ b/plugins/wakeup-manager/dependency-tv/inc/dependency_tv_audio.h @@ -23,6 +23,8 @@ void dependency_tv_audio_deinitialize(); void dependency_tv_audio_start_recording(); void dependency_tv_audio_stop_recording(); +void dependency_tv_audio_set_recording_session(unsigned int session); + void dependency_tv_audio_change_system_volume(); void dependency_tv_audio_recover_system_volume(); diff --git a/plugins/wakeup-manager/dependency-tv/src/dependency_tv.cpp b/plugins/wakeup-manager/dependency-tv/src/dependency_tv.cpp index fc75aae..aec2a52 100644 --- a/plugins/wakeup-manager/dependency-tv/src/dependency_tv.cpp +++ b/plugins/wakeup-manager/dependency-tv/src/dependency_tv.cpp @@ -44,6 +44,12 @@ int mas_dependency_stop_recording(void) return 0; } +int mas_dependency_set_recording_session(unsigned int session) +{ + dependency_tv_audio_set_recording_session(session); + return 0; +} + int mas_dependency_change_system_volume(void) { dependency_tv_audio_change_system_volume(); diff --git a/plugins/wakeup-manager/dependency-tv/src/dependency_tv_audio.cpp b/plugins/wakeup-manager/dependency-tv/src/dependency_tv_audio.cpp index c648d33..7b01ee8 100644 --- a/plugins/wakeup-manager/dependency-tv/src/dependency_tv_audio.cpp +++ b/plugins/wakeup-manager/dependency-tv/src/dependency_tv_audio.cpp @@ -210,6 +210,15 @@ void dependency_tv_audio_stop_recording() { } +void dependency_tv_audio_set_recording_session(unsigned int session) +{ + LOGD("Current recording session : [%s]", + (RECORDING_SESSION_WAKE_WORD == session) ? "wake word detecting" : + (RECORDING_SESSION_UTTERANCE == session) ? "utterance recording" : + (RECORDING_SESSION_FOLLOW_UP == session) ? "follow-up speech recording" : + "ERROR - Invalid recording session information" + ); +} void dependency_tv_audio_change_system_volume() { diff --git a/plugins/wakeup-manager/inc/dependency_resolver.h b/plugins/wakeup-manager/inc/dependency_resolver.h index 38e1202..3d24f98 100644 --- a/plugins/wakeup-manager/inc/dependency_resolver.h +++ b/plugins/wakeup-manager/inc/dependency_resolver.h @@ -39,6 +39,7 @@ int dependency_resolver_deinitialize(void); int dependency_resolver_set_error_callback(mas_dependency_error_cb callback, void* user_data); int dependency_resolver_start_recording(); int dependency_resolver_stop_recording(); +int dependency_resolver_set_recording_session(unsigned int); int dependency_resolver_change_system_volume(); int dependency_resolver_recover_system_volume(); int dependency_resolver_get_audio_format(int* rate, int* channel, int* audio_type); @@ -65,6 +66,8 @@ typedef int (*mas_dependency_set_error_callback)(mas_dependency_error_cb callbac typedef int (*mas_dependency_start_recording)(void); #define MAS_DEPENDENCY_FUNC_STOP_RECORDING "mas_dependency_stop_recording" typedef int (*mas_dependency_stop_recording)(void); +#define MAS_DEPENDENCY_FUNC_SET_RECORDING_SESSION "mas_dependency_set_recording_session" +typedef int (*mas_dependency_set_recording_session)(unsigned int); #define MAS_DEPENDENCY_FUNC_CHANGE_SYSTEM_VOLUME "mas_dependency_change_system_volume" typedef int (*mas_dependency_change_system_volume)(void); #define MAS_DEPENDENCY_FUNC_RECOVER_SYSTEM_VOLUME "mas_dependency_recover_system_volume" @@ -80,6 +83,7 @@ typedef struct { mas_dependency_set_error_callback set_error_callback; mas_dependency_start_recording start_recording; mas_dependency_stop_recording stop_recording; + mas_dependency_set_recording_session set_recording_session; mas_dependency_change_system_volume change_system_volume; mas_dependency_recover_system_volume recover_system_volume; mas_dependency_get_audio_format get_audio_format; diff --git a/plugins/wakeup-manager/inc/wakeup_audio_manager.h b/plugins/wakeup-manager/inc/wakeup_audio_manager.h index d23a1f5..fdff003 100644 --- a/plugins/wakeup-manager/inc/wakeup_audio_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_audio_manager.h @@ -45,6 +45,13 @@ public: wakeup_speech_streaming_event_e event, void* buffer, unsigned int len) = 0; }; +typedef enum +{ + RECORDING_SESSION_WAKE_WORD, + RECORDING_SESSION_UTTERANCE, + RECORDING_SESSION_FOLLOW_UP, +} recording_session; + class CAudioManager { public: @@ -64,6 +71,8 @@ public: void start_recording(); void stop_recording(); + void set_recording_session(recording_session session); + void feed_audio_data(wakeup_speech_streaming_event_e event, void* buffer, int len); void finalize_audio_data(); void clear_audio_data(); diff --git a/plugins/wakeup-manager/src/dependency_resolver.cpp b/plugins/wakeup-manager/src/dependency_resolver.cpp index 7b541a6..7a1f35a 100644 --- a/plugins/wakeup-manager/src/dependency_resolver.cpp +++ b/plugins/wakeup-manager/src/dependency_resolver.cpp @@ -72,6 +72,9 @@ int dependency_resolver_initialize(mas_proxy_interface interface) g_mas_dependency.stop_recording = (mas_dependency_stop_recording)dlsym(g_handle, MAS_DEPENDENCY_FUNC_STOP_RECORDING); + g_mas_dependency.set_recording_session = + (mas_dependency_set_recording_session)dlsym(g_handle, + MAS_DEPENDENCY_FUNC_SET_RECORDING_SESSION); g_mas_dependency.change_system_volume = (mas_dependency_change_system_volume)dlsym(g_handle, MAS_DEPENDENCY_FUNC_CHANGE_SYSTEM_VOLUME); @@ -189,6 +192,27 @@ int dependency_resolver_stop_recording(void) return ret; } +int dependency_resolver_set_recording_session(unsigned int session) +{ + int ret = -1; + MAS_LOGD("g_handle : %p", g_handle); + if (NULL != g_handle) { + mas_dependency_set_recording_session func = g_mas_dependency.set_recording_session; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MAS_DEPENDENCY_FUNC_SET_RECORDING_SESSION); + } else { + ret = func(session); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to set recording session, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + + return ret; +} + int dependency_resolver_change_system_volume() { int ret = -1; diff --git a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp index e9f0ddb..cba9f16 100644 --- a/plugins/wakeup-manager/src/wakeup_audio_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_audio_manager.cpp @@ -72,6 +72,11 @@ void CAudioManager::start_recording() dependency_resolver_start_recording(); } +void CAudioManager::set_recording_session(recording_session session) +{ + dependency_resolver_set_recording_session((unsigned int)session); +} + /* Need to consider adapting conventional producer-consumer model */ void CAudioManager::streaming_previous_audio_data_thread_func() { diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 9ffe0fb..ee68f22 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -162,6 +162,7 @@ bool CWakeupManager::activate(void) change_manager_state(WAKEUP_MANAGER_STATE_LISTENING); if (mWakeupEngineManager.get_audio_data_required()) { + mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD); mAudioManager.start_recording(); } @@ -394,6 +395,7 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) mVoiceKeyPressed = true; /* Start recorder thread using appropriate recording device */ + mAudioManager.set_recording_session(RECORDING_SESSION_UTTERANCE); mAudioManager.start_recording(); /* Wakeup default assistant */ @@ -422,6 +424,7 @@ bool CWakeupManager::process_event(ma_plugin_event_e event, void* data, int len) if (mWakeupEngineManager.get_audio_data_required()) { /* Restart recorder thread using appropriate recording device */ mAudioManager.stop_recording(); + mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD); mAudioManager.start_recording(); } else { mAudioManager.stop_recording(); @@ -484,6 +487,7 @@ static Eina_Bool streaming_duration_expired(void *data) if (WAKEUP_MANAGER_STATE_UTTERANCE == wakeup_manager->get_manager_state()) { wakeup_manager->change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING); } + return ECORE_CALLBACK_CANCEL; } @@ -556,6 +560,7 @@ bool CWakeupManager::start_streaming_follow_up_data() mAudioManager.stop_recording(); mAudioManager.clear_audio_data(); change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE); + mAudioManager.set_recording_session(RECORDING_SESSION_FOLLOW_UP); mAudioManager.start_recording(); mAudioManager.start_streaming_follow_up_data(); @@ -595,6 +600,7 @@ bool CWakeupManager::stop_streaming_follow_up_data() mAudioManager.stop_recording(); mAudioManager.clear_audio_data(); + mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD); mAudioManager.start_recording(); MWR_LOGD("[END]"); @@ -743,6 +749,7 @@ bool CWakeupManager::CEngineEventObserver::on_audio_data_require_status(string e if (audio_manager && engine_manager) { if (engine_manager->get_audio_data_required()) { if (mWakeupManager->get_voice_key_pressed() != true) { + audio_manager->set_recording_session(RECORDING_SESSION_WAKE_WORD); audio_manager->start_recording(); } } else { -- 2.34.1