From: Ji-hoon Lee Date: Wed, 25 Sep 2019 01:56:21 +0000 (+0900) Subject: Add wake word audio data embedding related APIs X-Git-Tag: submit/tizen/20191002.080047~4 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F96%2F214696%2F2;p=platform%2Fcore%2Fuifw%2Fmulti-assistant.git Add wake word audio data embedding related APIs Change-Id: Ia435a7e32a1f6093690c4865fbc822dc006e9052 --- diff --git a/client/ma.c b/client/ma.c index 217ab1a..458974a 100644 --- a/client/ma.c +++ b/client/ma.c @@ -668,6 +668,25 @@ int __ma_cb_preprocessing_information_changed(const char* app_id) return 0; } +int __ma_cb_audio_streaming_data_section_changed(ma_audio_streaming_data_section_e section) +{ + ma_audio_streaming_data_section_changed_cb callback = NULL; + void* user_data; + + ma_client_get_audio_streaming_data_section_changed_cb(g_ma, &callback, &user_data); + + if (NULL != callback) { + ma_client_use_callback(g_ma); + callback(section, user_data); + ma_client_not_use_callback(g_ma); + SLOG(LOG_DEBUG, TAG_MAC, "[DEBUG] Audio streaming data section changed callback is called, (%d)", section); + } else { + SLOG(LOG_DEBUG, TAG_MAC, "[WARNING] Audio streaming data section changed callback is NULL"); + } + + return 0; +} + int ma_get_state(ma_state_e* state) { if (0 != __ma_get_feature_enabled()) { @@ -1696,3 +1715,93 @@ int ma_send_preprocessing_result(bool is_success) } return ret; } + +int ma_set_wake_word_audio_require_flag(bool require) +{ + SLOG(LOG_DEBUG, TAG_MAC, "[Manager] Set wake word audio require flag"); + + if (0 != __ma_get_feature_enabled()) { + SLOG(LOG_DEBUG, TAG_MAC, "@@@ [Manager] not supported"); + return MA_ERROR_NOT_SUPPORTED; + } + + ma_state_e state; + if (0 != ma_client_get_client_state(g_ma, &state)) { + SLOG(LOG_ERROR, TAG_MAC, "[ERROR] A handle is not available"); + SLOG(LOG_DEBUG, TAG_MAC, "@@@"); + return MA_ERROR_INVALID_STATE; + } + + if (state != MA_STATE_READY) { + SLOG(LOG_ERROR, TAG_MAC, "[ERROR] Invalid State: Current state is not 'READY', state(%d)", state); + SLOG(LOG_DEBUG, TAG_MAC, "@@@"); + return MA_ERROR_INVALID_STATE; + } + + /* Set wake word audio require flag */ + int pid = getpid(); + int ret = ma_dbus_set_wake_word_audio_require_flag(pid, require); + if (0 != ret) { + SLOG(LOG_WARN, TAG_MAC, "[WARNING] retry to set wake word audio require flag"); + } else { + SLOG(LOG_DEBUG, TAG_MAC, "[DEBUG] Success to set"); + } + return ret; +} + +int ma_set_audio_streaming_data_section_changed_cb(ma_audio_streaming_data_section_changed_cb callback, void* user_data) +{ + if (0 != __ma_get_feature_enabled()) { + return MA_ERROR_NOT_SUPPORTED; + } + + SLOG(LOG_DEBUG, TAG_MAC, "[Client DEBUG] Set Multi-assistant audio streaming data section changed cb"); + + if (NULL == callback) { + SLOG(LOG_ERROR, TAG_MAC, "[ERROR] Invalid parameter"); //LCOV_EXCL_LINE + return MA_ERROR_INVALID_PARAMETER; + } + + ma_state_e state; + + if (0 != ma_client_get_client_state(g_ma, &state)) { + SLOG(LOG_ERROR, TAG_MAC, "[ERROR] A handle is not available"); //LCOV_EXCL_LINE + return MA_ERROR_INVALID_STATE; + } + + /* check state */ + if (state != MA_STATE_INITIALIZED) { + SLOG(LOG_ERROR, TAG_MAC, "[ERROR] Invalid State: Current state is not 'Initialized' (%d)", state); //LCOV_EXCL_LINE + return MA_ERROR_INVALID_STATE; + } + + ma_client_set_audio_streaming_data_section_changed_cb(g_ma, callback, user_data); + + return MA_ERROR_NONE; +} + +int ma_unset_audio_streaming_data_section_changed_cb(void) +{ + if (0 != __ma_get_feature_enabled()) { + return MA_ERROR_NOT_SUPPORTED; + } + + SLOG(LOG_DEBUG, TAG_MAC, "[Client DEBUG] Unset Multi-assistant audio streaming data section changed cb"); + + ma_state_e state; + + if (0 != ma_client_get_client_state(g_ma, &state)) { + SLOG(LOG_ERROR, TAG_MAC, "[ERROR] A handle is not available"); //LCOV_EXCL_LINE + return MA_ERROR_INVALID_STATE; + } + + /* check state */ + if (state != MA_STATE_INITIALIZED) { + SLOG(LOG_ERROR, TAG_MAC, "[ERROR] Invalid State: Current state is not 'Initialized'"); //LCOV_EXCL_LINE + return MA_ERROR_INVALID_STATE; + } + + ma_client_set_audio_streaming_data_section_changed_cb(g_ma, NULL, NULL); + + return MA_ERROR_NONE; +} diff --git a/client/ma_client.c b/client/ma_client.c index 34fc466..8f4e572 100644 --- a/client/ma_client.c +++ b/client/ma_client.c @@ -38,6 +38,8 @@ typedef struct { void* wakeup_engine_command_user_data; ma_preprocessing_information_changed_cb preprocessing_information_changed_cb; void* preprocessing_information_changed_user_data; + ma_audio_streaming_data_section_changed_cb audio_streaming_data_section_changed_cb; + void* audio_streaming_data_section_changed_user_data; /* state */ ma_state_e previous_state; @@ -510,3 +512,29 @@ int ma_client_get_preprocessing_information_changed_cb(ma_h ma, ma_preprocessing return MA_ERROR_NONE; } + +int ma_client_set_audio_streaming_data_section_changed_cb(ma_h ma, ma_audio_streaming_data_section_changed_cb callback, void* user_data) +{ + ma_client_s* client = __client_get(ma); + + if (NULL == client) + return MA_ERROR_INVALID_PARAMETER; + + client->audio_streaming_data_section_changed_cb = callback; + client->audio_streaming_data_section_changed_user_data = user_data; + + return MA_ERROR_NONE; +} + +int ma_client_get_audio_streaming_data_section_changed_cb(ma_h ma, ma_audio_streaming_data_section_changed_cb* callback, void** user_data) +{ + ma_client_s* client = __client_get(ma); + + if (NULL == client) + return MA_ERROR_INVALID_PARAMETER; + + *callback = client->audio_streaming_data_section_changed_cb; + *user_data = client->audio_streaming_data_section_changed_user_data; + + return MA_ERROR_NONE; +} diff --git a/client/ma_client.h b/client/ma_client.h index 280a0c3..5cbe21a 100644 --- a/client/ma_client.h +++ b/client/ma_client.h @@ -88,6 +88,10 @@ int ma_client_set_preprocessing_information_changed_cb(ma_h ma, ma_preprocessing int ma_client_get_preprocessing_information_changed_cb(ma_h ma, ma_preprocessing_information_changed_cb* callback, void** user_data); +int ma_client_set_audio_streaming_data_section_changed_cb(ma_h ma, ma_audio_streaming_data_section_changed_cb callback, void* user_data); + +int ma_client_get_audio_streaming_data_section_changed_cb(ma_h ma, ma_audio_streaming_data_section_changed_cb* callback, void** user_data); + #ifdef __cplusplus } #endif diff --git a/client/ma_dbus.c b/client/ma_dbus.c index 611050d..9e88d96 100644 --- a/client/ma_dbus.c +++ b/client/ma_dbus.c @@ -33,6 +33,7 @@ extern int __ma_cb_audio_streaming(int event, char* buffer, int len); extern int __ma_cb_active_state_changed(int state); extern int __ma_cb_wakeup_engine_command(const char *command); extern int __ma_cb_preprocessing_information_changed(const char* app_id); +extern int __ma_cb_audio_streaming_data_section_changed(ma_audio_streaming_data_section_e section); static Eina_Bool listener_event_callback(void* data, Ecore_Fd_Handler* fd_handler) { @@ -185,6 +186,24 @@ static Eina_Bool listener_event_callback(void* data, Ecore_Fd_Handler* fd_handle SLOG(LOG_INFO, TAG_MAC, "@@@"); } /* MAS_METHOD_SEND_PREPROCESSING_INFORMATION */ + else if (dbus_message_is_method_call(msg, if_name, MAS_METHOD_AUDIO_STREAMING_DATA_SECTION)) { + SLOG(LOG_INFO, TAG_MAC, "@@@ Activate"); + int section; + + dbus_message_get_args(msg, &err, + DBUS_TYPE_INT32, §ion, + DBUS_TYPE_INVALID); + + if (dbus_error_is_set(&err)) { + SLOG(LOG_ERROR, TAG_MAC, "@@ Get arguments error (%s)", err.message); + dbus_error_free(&err); + } else { + __ma_cb_audio_streaming_data_section_changed(section); + } + + SLOG(LOG_INFO, TAG_MAC, "@@@"); + } /* MAS_METHOD_AUDIO_STREAMING_DATA_SECTION */ + else if (dbus_message_is_signal(msg, if_name, MAS_METHOD_ERROR)) { SLOG(LOG_DEBUG, TAG_MAC, "[DEBUG] Get Error"); int reason; @@ -1301,3 +1320,52 @@ int ma_dbus_send_preprocessing_result(int pid, bool result) return 0; } + +int ma_dbus_set_wake_word_audio_require_flag(int pid, bool require) +{ + if (0 != __dbus_check()) { + return MA_ERROR_OPERATION_FAILED; + } + + DBusMessage* msg; + + msg = dbus_message_new_method_call( + MA_SERVER_SERVICE_NAME, + MA_SERVER_SERVICE_OBJECT_PATH, + MA_SERVER_SERVICE_INTERFACE, + MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG); + + if (NULL == msg) { + SLOG(LOG_ERROR, TAG_MAC, "@@ Request multi-assistant set wake word audio require flag : Fail to make message"); //LCOV_EXCL_LINE + return MA_ERROR_OPERATION_FAILED; + } else { + SLOG(LOG_DEBUG, TAG_MAC, "[DEBUG] multi-assistant set wake word audio require flag"); + } + + int tmp_require = (require ? 1 : 0); + dbus_message_append_args(msg, + DBUS_TYPE_INT32, &pid, + DBUS_TYPE_INT32, &tmp_require, + DBUS_TYPE_INVALID); + + dbus_message_set_no_reply(msg, TRUE); + + DBusError error; + dbus_error_init (&error); + if (!dbus_connection_send(g_conn_sender, msg, NULL)) { + if (dbus_error_is_set (&error)) { + SLOG(LOG_ERROR, TAG_MAC, "[Dbus ERROR] Fail to Send : %s", error.message); //LCOV_EXCL_LINE + dbus_error_free (&error); + } else { + SLOG(LOG_ERROR, TAG_MAC, "[Dbus ERROR] Fail to Send"); //LCOV_EXCL_LINE + } + return MA_ERROR_OPERATION_FAILED; + } else { + SLOG(LOG_DEBUG, TAG_MAC, "[Dbus DEBUG] Success to Send"); + dbus_connection_flush(g_conn_sender); + } + + dbus_message_unref(msg); + + return 0; +} diff --git a/client/ma_dbus.h b/client/ma_dbus.h index 73cb77b..2e737d1 100644 --- a/client/ma_dbus.h +++ b/client/ma_dbus.h @@ -61,6 +61,8 @@ int ma_dbus_set_preprocessing_allow_mode(int pid, ma_preprocessing_allow_mode_e int ma_dbus_send_preprocessing_result(int pid, bool result); +int ma_dbus_set_wake_word_audio_require_flag(int pid, bool require); + #ifdef __cplusplus } #endif diff --git a/common/ma_defs.h b/common/ma_defs.h index 5d00bbf..3ffe16b 100644 --- a/common/ma_defs.h +++ b/common/ma_defs.h @@ -80,6 +80,7 @@ extern "C" #define MA_METHOD_SET_BACKGROUND_VOLUME "ma_method_set_background_volume" #define MA_METHOD_SET_PREPROCESSING_ALLOW_MODE "ma_method_set_preprocessing_allow_mode" #define MA_METHOD_SEND_PREPROCESSING_RESULT "ma_method_send_preprocessing_result" +#define MA_METHOD_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG "ma_method_set_wake_word_audio_require_flag" #define MA_METHOD_ERROR "ma_method_error" #define MA_UI_METHOD_INITIALIZE "ma_ui_method_initialize" @@ -92,6 +93,7 @@ extern "C" #define MAS_METHOD_WAKEUP_ENGINE_COMMAND "mas_method_wakeup_engine_command" #define MAS_METHOD_ERROR "mas_method_error" #define MAS_METHOD_SEND_PREPROCESSING_INFORMATION "mas_method_send_preprocessing_information" +#define MAS_METHOD_AUDIO_STREAMING_DATA_SECTION "mas_method_audio_streaming_data_section" #define MAS_UI_METHOD_HELLO "mas_ui_method_hello" #define MAS_UI_METHOD_SEND_ASR_RESULT "mas_ui_method_send_asr_result" diff --git a/include/multi_assistant.h b/include/multi_assistant.h index 9d1103d..ff7b35b 100644 --- a/include/multi_assistant.h +++ b/include/multi_assistant.h @@ -641,6 +641,57 @@ int ma_unset_preprocessing_information_changed_cb(void); */ int ma_send_preprocessing_result(bool is_success); +/** + * @brief Sets the wake word audio require flag. + * @details If set to true, wake word audio data will be included in audio streaming data. + * @since_tizen 5.5 + * + * @param[in] require The require value to be set + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MA_ERROR_NONE Successful + * @retval #MA_ERROR_NOT_SUPPORTED Not supported + * @retval #MA_ERROR_INVALID_STATE Invalid state + * @retval #MA_ERROR_OPERATION_FAILED Operation failed + * + * @pre The state should be #MA_STATE_READY. + */ +int ma_set_wake_word_audio_require_flag(bool require); + +/** + * @brief Sets the section changed callback for audio streaming data. + * @since_tizen 5.5 + * + * @param[in] callback The callback + * @param[in] user_data The user data passed to the callback function + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MA_ERROR_NONE Successful + * @retval #MA_ERROR_NOT_SUPPORTED Not supported + * @retval #MA_ERROR_INVALID_PARAMETER Invalid parameter + * @retval #MA_ERROR_INVALID_STATE Invalid state + * + * @pre The state should be #MA_STATE_INITIALIZED. + * @see ma_audio_streaming_data_section_changed_cb() + * @see ma_unset_audio_streaming_data_section_changed_cb() + */ +int ma_set_audio_streaming_data_section_changed_cb(ma_audio_streaming_data_section_changed_cb callback, void* user_data); + +/** + * @brief Unsets the section changed callback for audio streaming data. + * @since_tizen 5.5 + * + * @return @c 0 on success, otherwise a negative error value + * @retval #MA_ERROR_NONE Successful + * @retval #MA_ERROR_NOT_SUPPORTED Not supported + * @retval #MA_ERROR_INVALID_STATE Invalid state + * + * @pre The state should be #MA_STATE_INITIALIZED. + * @see ma_audio_streaming_data_section_changed_cb() + * @see ma_set_audio_streaming_data_section_changed_cb() + */ +int ma_unset_audio_streaming_data_section_changed_cb(void); + #ifdef __cplusplus } #endif diff --git a/include/multi_assistant_common.h b/include/multi_assistant_common.h index caa7553..d020b24 100644 --- a/include/multi_assistant_common.h +++ b/include/multi_assistant_common.h @@ -152,6 +152,15 @@ typedef enum { MA_PREPROCESSING_ALLOW_ALL, /**< Preprocessing allowed for all audio */ } ma_preprocessing_allow_mode_e; +/** + * @brief Enumerations for section information of audio streaming data. + * @since_tizen 5.5 + */ +typedef enum { + MA_AUDIO_STREAMING_DATA_SECTION_UTTERANCE = 0, /**< Utterance section started */ + MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD, /**< Wake word section started */ +} ma_audio_streaming_data_section_e; + /** * @brief Called when the client state is changed. * @since_tizen 5.0 @@ -265,6 +274,15 @@ typedef int (*ma_assistant_info_list_cb)(ma_assistant_info_h handle, void* user_ */ typedef void (*ma_preprocessing_information_changed_cb)(const char* app_id, void* user_data); +/** + * @brief Called when the section information of audio streaming data is changed. + * @since_tizen 5.5 + * + * @param[in] section The current section information of audio streaming data + * @param[in] user_data The user data passed from the callback registration function + */ +typedef void (*ma_audio_streaming_data_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); + #ifdef __cplusplus } #endif