From 11833941fe11849ec5b4c1944affce7445fe3fb5 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 4 Mar 2019 19:36:45 +0900 Subject: [PATCH] Add function for handling voice button event Change-Id: I112c3319312dcdaac2c746defea6885f177bdcd5 --- inc/multi_assistant_main.h | 1 + inc/multi_assistant_service.h | 4 + inc/multi_assistant_service_plugin.h | 21 +- .../org.tizen.multi-assistant-service.spec | 11 + plugins/wakeup-manager/CMakeLists.txt | 24 +- plugins/wakeup-manager/inc/wakeup_manager.h | 16 +- plugins/wakeup-manager/src/wakeup_manager.cpp | 394 +++++++++++++++++- src/multi_assistant_dbus.c | 2 + src/multi_assistant_dbus_server.c | 31 ++ src/multi_assistant_dbus_server.h | 2 + src/multi_assistant_service.c | 16 + src/multi_assistant_service_plugin.c | 27 +- 12 files changed, 525 insertions(+), 24 deletions(-) diff --git a/inc/multi_assistant_main.h b/inc/multi_assistant_main.h index cfe85b4..e134529 100644 --- a/inc/multi_assistant_main.h +++ b/inc/multi_assistant_main.h @@ -60,6 +60,7 @@ #define MA_METHOD_START_STREAMING_AUDIO_DATA "ma_method_start_streaming_audio_data" #define MA_METHOD_STOP_STREAMING_AUDIO_DATA "ma_method_stop_streaming_audio_data" #define MA_METHOD_UPDATE_VOICE_FEEDBACK_STATE "ma_method_update_voice_feedback_state" +#define MA_METHOD_SEND_ASSISTANT_SPECIFIC_COMMAND "ma_method_send_assistant_specific_command" #define MA_METHOD_ERROR "ma_method_error" #define MA_UI_METHOD_INITIALIZE "ma_ui_method_initialize" diff --git a/inc/multi_assistant_service.h b/inc/multi_assistant_service.h index ff4fe3f..adeff4a 100644 --- a/inc/multi_assistant_service.h +++ b/inc/multi_assistant_service.h @@ -47,6 +47,8 @@ int mas_client_stop_streaming_audio_data(int pid, int type); int mas_client_update_voice_feedback_state(int pid, int state); +int mas_client_send_assistant_specific_command(int pid, const char *command); + int mas_client_update_result_state(int pid, int state); int mas_ui_client_initialize(int pid); @@ -71,6 +73,8 @@ int mas_launch_client_by_wakeup_word(const char *wakeup_word); int mas_launch_client_by_appid(const char *appid); +int mas_process_voice_key_event(bool pressed); + #ifdef __cplusplus } #endif diff --git a/inc/multi_assistant_service_plugin.h b/inc/multi_assistant_service_plugin.h index 0989be8..1984547 100644 --- a/inc/multi_assistant_service_plugin.h +++ b/inc/multi_assistant_service_plugin.h @@ -27,6 +27,10 @@ extern "C" { #endif +typedef enum { + MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0, + MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, +} ma_plugin_event_e; int multi_assistant_service_plugin_initialize(void); @@ -40,11 +44,13 @@ int multi_assistant_service_plugin_activate(void); int multi_assistant_service_plugin_deactivate(void); -int multi_assistant_service_plugin_update_voice_feedback_state(const char *appid, int state); +int multi_assistant_service_plugin_update_voice_feedback_state(const char* appid, int state); -int multi_assistant_service_plugin_update_result_state(const char *appid, int state); +int multi_assistant_service_plugin_send_assistant_specific_command(const char *appid, const char* command); -int multi_assistant_service_plugin_process_event(int event, void *data, int len); +int multi_assistant_service_plugin_update_result_state(const char* appid, int state); + +int multi_assistant_service_plugin_process_event(int event, void* data, int len); int multi_assistant_service_plugin_start_streaming_utterance_data(void); @@ -58,7 +64,7 @@ int multi_assistant_service_plugin_start_streaming_follow_up_data(void); int multi_assistant_service_plugin_stop_streaming_follow_up_data(void); -int multi_assistant_service_plugin_get_recording_audio_format(int *rate, int *channel, int *audio_type); +int multi_assistant_service_plugin_get_recording_audio_format(int* rate, int* channel, int* audio_type); int multi_assistant_service_plugin_set_callbacks(void); @@ -92,9 +98,11 @@ typedef int (*wakeup_manager_activate)(void); #define MA_WAKEUP_MANAGER_FUNC_DEACTIVATE "wakeup_manager_deactivate" typedef int (*wakeup_manager_deactivate)(void); #define MA_WAKEUP_MANAGER_FUNC_UPDATE_VOICE_FEEDBACK_STATE "wakeup_manager_update_voice_feedback_state" -typedef int (*wakeup_manager_update_voice_feedback_state)(const char *appid, int state); +typedef int (*wakeup_manager_update_voice_feedback_state)(const char* appid, int state); +#define MA_WAKEUP_MANAGER_FUNC_SEND_ASSISTANT_SPECIFIC_COMMAND "wakeup_manager_send_assistant_specific_command" +typedef int (*wakeup_manager_send_assistant_specific_command)(const char* appid, const char* command); #define MA_WAKEUP_MANAGER_FUNC_UPDATE_RESULT_STATE "wakeup_manager_update_result_state" -typedef int (*wakeup_manager_update_result_state)(const char *appid, int state); +typedef int (*wakeup_manager_update_result_state)(const char* appid, int state); #define MA_WAKEUP_MANAGER_FUNC_PROCESS_EVENT "wakeup_manager_process_event" typedef int (*wakeup_manager_process_event)(int event, void* data, int len); #define MA_WAKEUP_MANAGER_FUNC_START_STREAMING_UTTERANCE_DATA "wakeup_manager_start_streaming_utterance_data" @@ -132,6 +140,7 @@ typedef struct { wakeup_manager_activate activate; wakeup_manager_deactivate deactivate; wakeup_manager_update_voice_feedback_state update_voice_feedback_state; + wakeup_manager_send_assistant_specific_command send_assistant_specific_command; wakeup_manager_update_result_state update_result_state; wakeup_manager_process_event process_event; wakeup_manager_start_streaming_utterance_data start_streaming_utterance_data; diff --git a/packaging/org.tizen.multi-assistant-service.spec b/packaging/org.tizen.multi-assistant-service.spec index efa5231..fb765ac 100644 --- a/packaging/org.tizen.multi-assistant-service.spec +++ b/packaging/org.tizen.multi-assistant-service.spec @@ -22,6 +22,14 @@ BuildRequires: pkgconfig(dbus-1) BuildRequires: pkgconfig(libtzplatform-config) BuildRequires: pkgconfig(libxml-2.0) BuildRequires: pkgconfig(multi-assistant) +%if "%{tizen_profile_name}" == "tv" +BuildRequires: pkgconfig(vd-win-util) +BuildRequires: pkgconfig(capi-network-bluetooth) +BuildRequires: pkgconfig(capi-network-bluetooth-tv) +#BuildRequires: pkgconfig(msfapi) +#BuildRequires: pkgconfig(farfield-voice-api) +%endif + BuildRequires: boost-system BuildRequires: boost-thread @@ -51,6 +59,9 @@ LDFLAGS="$LDFLAGS -Wl,-z -Wl,nodelete" export LDFLAGS cmake \ +%if "%{tizen_profile_name}" == "tv" + -D_TV_PRODUCT=TRUE \ +%endif -DCMAKE_INSTALL_PREFIX=%{_appdir} \ -DTZ_SYS_RO_SHARE=%TZ_SYS_RO_SHARE \ diff --git a/plugins/wakeup-manager/CMakeLists.txt b/plugins/wakeup-manager/CMakeLists.txt index 231e8b6..eb8ed08 100644 --- a/plugins/wakeup-manager/CMakeLists.txt +++ b/plugins/wakeup-manager/CMakeLists.txt @@ -11,10 +11,10 @@ INCLUDE_DIRECTORIES( ./inc ) -INCLUDE(FindPkgConfig) -pkg_check_modules(pkgs REQUIRED +SET(WMPKG_CHECK_MODULES appcore-agent ecore + ecore-wl2 dlog capi-appfw-app-manager capi-network-connection @@ -23,6 +23,18 @@ pkg_check_modules(pkgs REQUIRED eina ) +IF("${_TV_PRODUCT}" STREQUAL "TRUE") + SET(WMPKG_CHECK_MODULES ${WMPKG_CHECK_MODULES} + capi-network-bluetooth + capi-network-bluetooth-tv + vd-win-util + ) +ENDIF() + +INCLUDE(FindPkgConfig) +pkg_check_modules(wmpkgs REQUIRED ${WMPKG_CHECK_MODULES}) +MESSAGE("Modules : ${WMPKG_CHECK_MODULES}") + IF("${CMAKE_BUILD_TYPE}" STREQUAL "") SET(CMAKE_BUILD_TYPE "Release") ENDIF("${CMAKE_BUILD_TYPE}" STREQUAL "") @@ -35,7 +47,7 @@ SET(SRCS src/wakeup_manager.cpp ) -FOREACH(flag ${pkgs_CFLAGS}) +FOREACH(flag ${wmpkgs_CFLAGS}) SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} ${flag}") ENDFOREACH(flag) @@ -43,11 +55,15 @@ ENDFOREACH(flag) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS} -fPIC -std=c++11 -fvisibility=hidden") SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -fPIC -Wall" ) +IF("${_TV_PRODUCT}" STREQUAL "TRUE") + SET(EXTRA_CXXFLAGS "${EXTRA_CXXFLAGS} -DTV_PRODUCT") +ENDIF() SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS}") SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") ADD_LIBRARY(${PROJECT_NAME} SHARED ${SRCS} ) # Install libraries -TARGET_LINK_LIBRARIES(${PROJECT_NAME} ${pkgs_LDFLAGS} -ldl ${EXTRA_LDFLAGS}) +TARGET_LINK_LIBRARIES(${PROJECT_NAME} ${wmpkgs_LDFLAGS} -ldl ${EXTRA_LDFLAGS}) +MESSAGE("LDFLAG : ${wmpkgs_LDFLAGS}") INSTALL(FILES ${CMAKE_SOURCE_DIR}/plugins/wakeup-manager/libma-wakeup-manager.so DESTINATION ${TZ_SYS_RO_SHARE}/multiassistant/) diff --git a/plugins/wakeup-manager/inc/wakeup_manager.h b/plugins/wakeup-manager/inc/wakeup_manager.h index 6fce65d..a79a335 100644 --- a/plugins/wakeup-manager/inc/wakeup_manager.h +++ b/plugins/wakeup-manager/inc/wakeup_manager.h @@ -102,6 +102,8 @@ DLL_PUBLIC int wakeup_manager_deactivate(void); DLL_PUBLIC int wakeup_manager_update_voice_feedback_state(const char *appid, int state); +DLL_PUBLIC int wakeup_manager_send_assistant_specific_command(const char* appid, const char* command); + DLL_PUBLIC int wakeup_manager_update_result_state(const char *appid, int state); DLL_PUBLIC int wakeup_manager_process_event(int event, void* data, int len); @@ -143,6 +145,11 @@ typedef struct { int len; } wakeup_engine_speech_data; +typedef enum { + MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0, + MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, +} ma_plugin_event_e; + #define MA_WAKEUP_ENGINE_FUNC_INITIALIZE "wakeup_engine_initialize" typedef int (*wakeup_engine_initialize)(void); #define MA_WAKEUP_ENGINE_FUNC_DEINITIALIZE "wakeup_engine_deinitialize" @@ -167,10 +174,8 @@ typedef int (*wakeup_engine_feed_audio_data)(long time, void* data, int len); typedef int (*wakeup_engine_get_utterance_data_count)(void); #define MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA "wakeup_engine_get_utterance_data" typedef int (*wakeup_engine_get_utterance_data)(int index, wakeup_engine_speech_data *data); -#define MA_WAKEUP_ENGINE_FUNC_GET_PREVIOUS_UTTERANCE_DATA_COUNT "wakeup_engine_get_previous_utterance_data_count" -typedef int (*wakeup_engine_get_previous_utterance_data_count)(void); -#define MA_WAKEUP_ENGINE_FUNC_GET_PREVIOUS_UTTERANCE_DATA "wakeup_engine_get_previous_utterance_data" -typedef int (*wakeup_engine_get_previous_utterance_data)(int index, wakeup_engine_speech_data *data); +#define MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND "wakeup_engine_set_assistant_specific_command" +typedef int (*wakeup_engine_set_assistant_specific_command)(const char* appid, const char* command); #define MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK "wakeup_engine_set_wakeup_event_callback" typedef int (*wakeup_engine_set_wakeup_event_callback)(wakeup_service_wakeup_event_cb callback, void* user_data); #define MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK "wakeup_engine_set_speech_status_callback" @@ -193,8 +198,7 @@ typedef struct { wakeup_engine_feed_audio_data feed_audio_data; wakeup_engine_get_utterance_data_count get_utterance_data_count; wakeup_engine_get_utterance_data get_utterance_data; - wakeup_engine_get_previous_utterance_data_count get_previous_utterance_data_count; - wakeup_engine_get_previous_utterance_data get_previous_utterance_data; + wakeup_engine_set_assistant_specific_command set_assistant_specific_command; wakeup_engine_set_wakeup_event_callback set_wakeup_event_callback; wakeup_engine_set_speech_status_callback set_speech_status_callback; wakeup_engine_set_error_callback set_error_callback; diff --git a/plugins/wakeup-manager/src/wakeup_manager.cpp b/plugins/wakeup-manager/src/wakeup_manager.cpp index 2d234c8..e9ec931 100644 --- a/plugins/wakeup-manager/src/wakeup_manager.cpp +++ b/plugins/wakeup-manager/src/wakeup_manager.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -53,6 +54,11 @@ static void* g_error_user_data; #define MAX_WAKEUP_ENGINE_NUM 10 static int g_engine_count = 0; +#ifdef TV_PRODUCT +#define TV_BT_MODE +#include +#endif + typedef struct { bool active; bool enabled; @@ -79,6 +85,160 @@ static std::atomic_bool g_recorder_thread_should_stop; static std::thread g_speech_data_thread; static std::atomic_bool g_speech_data_thread_should_stop; +static bool g_voice_key_pressed = false; +static std::vector g_speech_data; + +#ifdef TV_BT_MODE + +#define EFL_BETA_API_SUPPORT + +#include +#include +#include + +static int g_bt_extend_count; + +#define SMART_CONTROL_EXTEND_CMD 0x03 +#define SMART_CONTROL_START_CMD 0x04 + +Ecore_Event_Handler* _key_down_handler = NULL; +Ecore_Event_Handler* _key_up_handler = NULL; + +Eina_Bool _key_down_cb(void* data, int type, void* event) +{ + Ecore_Event_Key *ev = (Ecore_Event_Key *) event; + if (ev) { + MWR_LOGD("KEY[%s], typep[%d]", ev->keyname, type); + + if (ev->keyname && strncmp(ev->keyname, KEY_BT_VOICE, strlen(KEY_BT_VOICE)) == 0 ) { + wakeup_manager_send_assistant_specific_command(0, "voice_key_pressed"); + } + } + + return ECORE_CALLBACK_DONE; +} + +Eina_Bool _key_up_cb(void* data, int type, void* event) +{ + Ecore_Event_Key *ev = (Ecore_Event_Key *) event; + if (ev) { + MWR_LOGD("KEY[%s], typep[%d]", ev->keyname, type); + + if (ev->keyname && strncmp(ev->keyname, KEY_BT_VOICE, strlen(KEY_BT_VOICE)) == 0) { + bt_hid_rc_stop_sending_voice(NULL); + wakeup_manager_send_assistant_specific_command(0, "voice_key_released"); + } + } + return ECORE_CALLBACK_DONE; +} + +bool _grab_voice_key(void) +{ + Eina_Bool bRet = true; + bRet = ecore_wl2_window_keygrab_set(NULL, KEY_BT_VOICE, 0, 0, 0, ECORE_WL2_WINDOW_KEYGRAB_SHARED); + MWR_LOGD("ecore_wl2_window_keygrab_set ret[%d] [%s]", bRet, KEY_BT_VOICE); + return bRet; +} + +bool _ungrab_voice_key(void) +{ + Eina_Bool bRet = true; + bRet = ecore_wl2_window_keygrab_unset(NULL, KEY_BT_VOICE, 0, 0); + MWR_LOGD("ecore_wl2_window_keygrab_unset ret[%d] [%s]", bRet, KEY_BT_VOICE); + return bRet; +} + +bool _add_key_cb() +{ + if (_key_down_handler == NULL) + { + MWR_LOGE("_key_down_handler"); + _key_down_handler = ecore_event_handler_add(ECORE_EVENT_KEY_DOWN, _key_down_cb, NULL); + if(_key_down_handler == NULL) + { + MWR_LOGE("_key_down_handler == NULL "); + } + } + + if (_key_up_handler == NULL) + { + MWR_LOGE("_key_down_handler"); + _key_up_handler = ecore_event_handler_add(ECORE_EVENT_KEY_UP, _key_up_cb, NULL); + if(_key_up_handler == NULL) + { + MWR_LOGE("_key_up_handler == NULL "); + } + } + return true; +} + +bool _delete_key_cb(void) +{ + MWR_LOGE("start"); + if (_key_down_handler != NULL) + { + ecore_event_handler_del(_key_down_handler); + _key_down_handler = NULL; + } + + if (_key_up_handler != NULL) + { + ecore_event_handler_del(_key_up_handler); + _key_up_handler = NULL; + } + MWR_LOGE("end"); + return true; +} + +static void _bt_cb_hid_state_changed(int result, bool connected, const char *remote_address, void *user_data) +{ + MWR_LOGD("[Recorder] Bluetooth Event [%d] Received address [%s]", result, remote_address); + return; +} + +static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void *user_data) +{ + static int g_buffer_count = 0; + if (nullptr == voice_data) return; + + if (g_voice_key_pressed) { + wakeup_engine_speech_data data; + data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE; + data.len = voice_data->length; + data.buffer = malloc(voice_data->length); + if (data.buffer) { + memcpy(data.buffer, voice_data->audio_buf, voice_data->length); + g_speech_data.push_back(data); + } + } else { + MWR_LOGE("[Recorder ERROR] voice key seems to be already released"); + return; + } + + if (0 == g_buffer_count || 0 == g_buffer_count % 50) { + MWR_LOGD("[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, voice_data->length); + + if (0 == g_bt_extend_count % 5 && 0 != g_buffer_count) { + const unsigned char input_data[2] = {SMART_CONTROL_EXTEND_CMD, 0x10 }; + if (BT_ERROR_NONE != bt_hid_send_rc_command(NULL, input_data, sizeof(input_data))) { + MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command"); + } else { + MWR_LOGD("[Recorder] Extend bt audio recorder"); + } + } + g_bt_extend_count++; + + if (100000 == g_buffer_count) { + g_buffer_count = 0; + } + } + + g_buffer_count++; + + return; +} +#endif + static int recorder_initialize() { const int rate = 16000; @@ -119,12 +279,47 @@ static int recorder_initialize() audio_in_destroy(g_audio_h); return -1; } + +#ifdef TV_BT_MODE + bool is_bt_failed = false; + + if (false == is_bt_failed && BT_ERROR_NONE != bt_product_init()) { + MWR_LOGE("[Recorder ERROR] Fail to init bt"); + is_bt_failed = true; + } + + if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, NULL)) { + MWR_LOGE("[Recorder ERROR] Fail bt_hid_host_initialize()"); + is_bt_failed = true; + } + + if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, NULL)) { + MWR_LOGE("[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()"); + is_bt_failed = true; + } + + if (false == is_bt_failed) { + MWR_LOGD("[Recorder] Bluetooth is available"); + } + + _grab_voice_key(); + _add_key_cb(); +#endif } static int recorder_deinitialize(void) { MWR_LOGD("[ENTER]"); +#ifdef TV_BT_MODE + _delete_key_cb(); + _ungrab_voice_key(); + + bt_hid_unset_audio_data_receive_cb(); + bt_hid_host_deinitialize(); + bt_product_deinit(); +#endif + int ret = 0; ret = audio_in_unprepare(g_audio_h); if (AUDIO_IO_ERROR_NONE != ret) { @@ -251,8 +446,41 @@ static void join_recorder_thread() static void start_recorder_thread() { join_recorder_thread(); - g_recorder_thread_should_stop.store(false); - g_recorder_thread = std::thread(recorder_thread_func); + +#ifdef TV_BT_MODE + /* Do not start normal recorder thread if TV_BT_MODE and g_voice_key_pressed, + just send bt_hid start message */ + if (g_voice_key_pressed) + { + const unsigned char input_data[2] = {SMART_CONTROL_START_CMD, 0x00}; + int bt_retry = 0; + const int max_retry = 5; + while (max_retry > bt_retry) { + int ret = bt_hid_send_rc_command(NULL, input_data, sizeof(input_data)); + if (BT_ERROR_NONE == ret) { + MWR_LOGD("[Recorder] Start bt audio recorder"); + break; + } else if (BT_ERROR_NOW_IN_PROGRESS == ret) { + MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command : %d", ret); + usleep(50000); + bt_retry++; + } else { + break; + } + } + if (max_retry == bt_retry) { + MWR_LOGE("[Recorder ERROR] Fail to start bt audio"); + return; + } + + g_bt_extend_count = 0; + } + else +#endif + { + g_recorder_thread_should_stop.store(false); + g_recorder_thread = std::thread(recorder_thread_func); + } } static void wakeup_engine_audio_data_require_status_cb(bool require, void* user_data) @@ -276,12 +504,14 @@ static void wakeup_engine_audio_data_require_status_cb(bool require, void* user_ } if (audio_data_require_count > 0) { g_audio_data_required = true; - if (g_audio_data_required != prev_audio_data_required) { + if (g_audio_data_required != prev_audio_data_required && + g_voice_key_pressed != true) { start_recorder_thread(); } } else { g_audio_data_required = false; - if (g_audio_data_required != prev_audio_data_required) { + if (g_audio_data_required != prev_audio_data_required && + g_voice_key_pressed != true) { join_recorder_thread(); } } @@ -365,6 +595,9 @@ static void wakeup_engine_add_directory(const char* name, const char* path) g_wakeup_engine_info[g_engine_count].interface.get_utterance_data = (wakeup_engine_get_utterance_data)dlsym(handle, MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA); + g_wakeup_engine_info[g_engine_count].interface.set_assistant_specific_command = + (wakeup_engine_set_assistant_specific_command)dlsym(handle, + MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND); g_wakeup_engine_info[g_engine_count].interface.set_wakeup_event_callback = (wakeup_engine_set_wakeup_event_callback)dlsym(handle, MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK); @@ -485,6 +718,11 @@ int wakeup_manager_deinitialize(void) recorder_deinitialize(); + for (const auto &data : g_speech_data) { + free(data.buffer); + } + g_speech_data.clear(); + if (g_current_language) { free(g_current_language); g_current_language = NULL; @@ -600,6 +838,12 @@ int wakeup_manager_update_voice_feedback_state(const char* appid, int state) { MWR_LOGD("[ENTER]"); + if (NULL == appid) { + MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)", + (appid ? appid : "NULL")); + return -1; + } + if (state) { if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_LISTENING || g_wakeup_manager_state == WAKEUP_MANAGER_STATE_PROCESSING) { @@ -615,9 +859,49 @@ int wakeup_manager_update_voice_feedback_state(const char* appid, int state) return 0; } +int wakeup_manager_send_assistant_specific_command(const char* appid, const char* command) +{ + MWR_LOGD("[ENTER]"); + + if (NULL == appid || NULL == command) { + MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), command(%s)", + (appid ? appid : "NULL"), (command ? command : "NULL")); + return -1; + } + + const char* voice_key_pressed = "voice_key_pressed"; + const char* voice_key_released = "voice_key_released"; + + if (command) { + if (0 == strncmp(command, voice_key_pressed, strlen(voice_key_pressed))) { + wakeup_manager_process_event(MA_PLUGIN_EVENT_VOICE_KEY_PRESSED, NULL, 0); + } else if (0 == strncmp(command, voice_key_released, strlen(voice_key_released))) { + wakeup_manager_process_event(MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, NULL, 0); + } + } + + for (int loop = 0;loop < g_engine_count;loop++) { + /* Need to find the appropriate engine for this assistant, + for now assuming 0 is the one */ + if(loop == 0) { + if (g_wakeup_engine_info[loop].interface.set_assistant_specific_command) { + g_wakeup_engine_info[loop].interface.set_assistant_specific_command(appid, command); + } + } + } + + MWR_LOGD("[END]"); + return 0; +} + int wakeup_manager_update_result_state(const char* appid, int state) { MWR_LOGD("[ENTER]"); + if (NULL == appid) { + MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)", + (appid ? appid : "NULL")); + return -1; + } if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_PROCESSING) { wakeup_manager_change_state(WAKEUP_MANAGER_STATE_LISTENING); } @@ -629,6 +913,49 @@ int wakeup_manager_process_event(int event, void* data, int len) { MWR_LOGD("[ENTER]"); + // LOCK REQUIRED + if (event == MA_PLUGIN_EVENT_VOICE_KEY_PRESSED) { + if (g_voice_key_pressed != true) { + /* Clear all existing data */ + for (const auto &speech_data : g_speech_data) { + if (speech_data.buffer) free(speech_data.buffer); + } + g_speech_data.clear(); + + g_voice_key_pressed = true; + /* (Re)Start recorder thread using bt hid */ + start_recorder_thread(); + for (int loop = 0;loop < g_engine_count;loop++) { + /* Need to find the default assistant, for now assuming 0 is the one */ + if(loop == 0) { + g_wakeup_event_cb(WAKEUP_EVENT_SUCCESS, + "hi bixby", g_wakeup_event_user_data); + } + } + } + } else if (event == MA_PLUGIN_EVENT_VOICE_KEY_RELEASED) { + if (g_voice_key_pressed != false) { + unsigned char final_buffer[2] = {'\0', }; + + g_voice_key_pressed = false; + if (g_audio_data_required == true) { + /* Restart recorder thread using standard mic */ + start_recorder_thread(); + } else { + join_recorder_thread(); + } + wakeup_engine_speech_data speech_data; + speech_data.event = WAKEUP_SPEECH_STREAMING_EVENT_FINISH; + speech_data.len = sizeof(final_buffer); + speech_data.buffer = malloc(speech_data.len); + if (speech_data.buffer) { + memcpy(speech_data.buffer, final_buffer, speech_data.len); + g_speech_data.push_back(speech_data); + } + } + } + // UNLOCK REQUIRED + MWR_LOGD("[END]"); return 0; } @@ -648,7 +975,58 @@ void __wakeup_service_streaming_cb(wakeup_service_speech_streaming_event_e event } } -static void speech_data_thread_func(void) +static void manager_data_thread_func(void) +{ + MWR_LOGD("[ENTER]"); + + MWR_LOGD("data_count : %zu", g_speech_data.size()); + + int index = 0; + + while (1) { + int ret = -1; + int cnt = 0; + + /* get feedback data */ + if (index >= g_speech_data.size()) { + /* empty queue */ + MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret); + + /* waiting */ + while (1) { + usleep(10000); + if (index < g_speech_data.size()) { + MWR_LOGI("[INFO] Resume thread"); + break; + } + if (200 < cnt) { + MWR_LOGE("[ERROR] Wrong request, there's no pcm data"); + __wakeup_service_streaming_cb( + WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0); + return; + } + cnt++; + } + MWR_LOGI("[INFO] Finish to wait for new feedback data come"); + + /* resume feedback thread */ + continue; + } + + wakeup_engine_speech_data &speech_data = g_speech_data.at(index); + __wakeup_service_streaming_cb( + speech_data.event, speech_data.buffer, speech_data.len); + + if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) { + MWR_LOGI("[INFO] Finish to get and send speech data"); + break; + } + + index++; + } +} + +static void engine_data_thread_func(void) { MWR_LOGD("[ENTER]"); @@ -724,7 +1102,11 @@ int wakeup_manager_start_streaming_utterance_data(void) g_speech_data_thread.join(); } g_speech_data_thread_should_stop.store(false); - g_speech_data_thread = std::thread(speech_data_thread_func); + if (g_voice_key_pressed) { + g_speech_data_thread = std::thread(manager_data_thread_func); + } else { + g_speech_data_thread = std::thread(engine_data_thread_func); + } MWR_LOGD("[END]"); return 0; diff --git a/src/multi_assistant_dbus.c b/src/multi_assistant_dbus.c index 42ecaf1..75c57b1 100644 --- a/src/multi_assistant_dbus.c +++ b/src/multi_assistant_dbus.c @@ -646,6 +646,8 @@ static Eina_Bool listener_event_callback(void* data, Ecore_Fd_Handler *fd_handle } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_UPDATE_VOICE_FEEDBACK_STATE)) { ma_service_dbus_update_voice_feedback_state(g_conn_listener, msg); + } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_SEND_ASSISTANT_SPECIFIC_COMMAND)) { + ma_service_dbus_send_assistant_specific_command(g_conn_listener, msg); } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_UI_METHOD_INITIALIZE)) { ma_service_ui_dbus_initialize(g_conn_listener, msg); diff --git a/src/multi_assistant_dbus_server.c b/src/multi_assistant_dbus_server.c index 3e6fd55..0d0712c 100644 --- a/src/multi_assistant_dbus_server.c +++ b/src/multi_assistant_dbus_server.c @@ -438,6 +438,37 @@ int ma_service_dbus_update_voice_feedback_state(DBusConnection* conn, DBusMessag return 0; } +int ma_service_dbus_send_assistant_specific_command(DBusConnection* conn, DBusMessage* msg) +{ + DBusError err; + dbus_error_init(&err); + + int pid; + int ret = 0; + char* command; + + dbus_message_get_args(msg, &err, + DBUS_TYPE_INT32, &pid, + DBUS_TYPE_STRING, &command, + DBUS_TYPE_INVALID); + + MAS_LOGD("[DEBUG] MAS SEND ASSISTANT SPECIFIC COMMAND"); + + if (dbus_error_is_set(&err)) { + MAS_LOGE("[IN ERROR] mas send assistant specific command : Get arguments error (%s)", err.message); + dbus_error_free(&err); + ret = -1; //MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD("[IN] mas send assistant specific command : pid(%d), command(%s)", pid, command); + ret = mas_client_send_assistant_specific_command(pid, command); + } + + MAS_LOGD("<<<<<"); + MAS_LOGD(" "); + + return 0; +} + int ma_service_ui_dbus_initialize(DBusConnection* conn, DBusMessage* msg) { DBusError err; diff --git a/src/multi_assistant_dbus_server.h b/src/multi_assistant_dbus_server.h index eff7f19..83703d0 100644 --- a/src/multi_assistant_dbus_server.h +++ b/src/multi_assistant_dbus_server.h @@ -43,6 +43,8 @@ int ma_service_dbus_stop_streaming_audio_data(DBusConnection* conn, DBusMessage* int ma_service_dbus_update_voice_feedback_state(DBusConnection* conn, DBusMessage* msg); +int ma_service_dbus_send_assistant_specific_command(DBusConnection* conn, DBusMessage* msg); + int ma_service_ui_dbus_initialize(DBusConnection* conn, DBusMessage* msg); int ma_service_ui_dbus_deinitialize(DBusConnection* conn, DBusMessage* msg); diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 7bc14ac..f4af1f6 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -340,6 +340,12 @@ int mas_client_update_voice_feedback_state(int pid, int state) return 0; } +int mas_client_send_assistant_specific_command(int pid, const char *command) +{ + multi_assistant_service_plugin_send_assistant_specific_command(NULL, command); + return 0; +} + int mas_client_update_result_state(int pid, int state) { multi_assistant_service_plugin_update_result_state(NULL, state); @@ -739,6 +745,16 @@ int mas_launch_client_by_wakeup_word(const char *wakeup_word) return mas_launch_client_by_appid(appid); } + +int mas_process_voice_key_event(bool pressed) +{ + if (pressed) { + multi_assistant_service_plugin_process_event(MA_PLUGIN_EVENT_VOICE_KEY_PRESSED, NULL, 0); + } else { + multi_assistant_service_plugin_process_event(MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, NULL, 0); + } +} + bool service_app_create(void *data) { // Todo: add your code here. diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index c5fd251..1762c84 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -287,6 +287,9 @@ int multi_assistant_service_plugin_initialize(void) _wakeup_manager_interface.update_voice_feedback_state = (wakeup_manager_update_voice_feedback_state)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_UPDATE_VOICE_FEEDBACK_STATE); + _wakeup_manager_interface.send_assistant_specific_command = + (wakeup_manager_send_assistant_specific_command)dlsym(g_handle, + MA_WAKEUP_MANAGER_FUNC_SEND_ASSISTANT_SPECIFIC_COMMAND); _wakeup_manager_interface.update_result_state = (wakeup_manager_update_result_state)dlsym(g_handle, MA_WAKEUP_MANAGER_FUNC_UPDATE_RESULT_STATE); @@ -457,7 +460,8 @@ int multi_assistant_service_plugin_deactivate(void) } return ret; } -int multi_assistant_service_plugin_update_voice_feedback_state(const char *appid, int state) + +int multi_assistant_service_plugin_update_voice_feedback_state(const char* appid, int state) { int ret = -1; if (NULL != g_handle) { @@ -476,7 +480,26 @@ int multi_assistant_service_plugin_update_voice_feedback_state(const char *appid return ret; } -int multi_assistant_service_plugin_update_result_state(const char *appid, int state) +int multi_assistant_service_plugin_send_assistant_specific_command(const char* appid, const char* command) +{ + int ret = -1; + if (NULL != g_handle) { + wakeup_manager_update_voice_feedback_state func = _wakeup_manager_interface.send_assistant_specific_command; + if (NULL == func) { + MAS_LOGE("[ERROR] symbol lookup failed : %s", MA_WAKEUP_MANAGER_FUNC_SEND_ASSISTANT_SPECIFIC_COMMAND); + } else { + ret = func(appid, command); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to stop recording, ret(%d)", ret); + } + } + } else { + MAS_LOGE("[ERROR] g_handle is not valid"); + } + return ret; +} + +int multi_assistant_service_plugin_update_result_state(const char* appid, int state) { int ret = -1; if (NULL != g_handle) { -- 2.34.1