From 0ace34918f2025741fb18aae86b8d65627d21a71 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Wed, 16 Oct 2019 18:27:12 +0900 Subject: [PATCH] Make preprocessing executed in sequential manner Change-Id: Id07bc66c866af8011ea06848483765f85fc68465 --- client/ma.c | 119 +++++++++++++++++++++++++++++ client/ma_client.c | 125 ++++++++++++++++++++++++++++++- client/ma_client.h | 30 ++++++++ include/multi_assistant.h | 34 --------- include/multi_assistant_common.h | 9 --- 5 files changed, 273 insertions(+), 44 deletions(-) diff --git a/client/ma.c b/client/ma.c index c9e9c90..c5b0cf4 100644 --- a/client/ma.c +++ b/client/ma.c @@ -291,6 +291,19 @@ int ma_deinitialize(void) break; } +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE + /* Release all existing preprocessing audio data */ + speech_data* data = NULL; + do { + ma_client_pop_preprocessing_audio_data(g_ma, &data); + if (data) { + SLOG(LOG_DEBUG, TAG_MAC, "[DEBUG] Audio streaming callback is called, event(%d)", data->event); + free(data->buffer); + free(data); + } + } while (data); +#endif + SLOG(LOG_DEBUG, TAG_MAC, "Success: destroy"); if (0 != ma_dbus_close_connection()) { @@ -540,6 +553,43 @@ int __ma_cb_audio_streaming(int event, char* buffer, int len) ma_audio_streaming_cb callback = NULL; void* user_data; +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE + bool preprocessing_result_received = false; + ma_preprocessing_allow_mode_e preprocessing_allow_mode; + ma_audio_streaming_data_type_e audio_streaming_data_type; + ma_client_get_preprocessing_result_received(g_ma, &preprocessing_result_received); + ma_client_get_preprocessing_allow_mode(g_ma, &preprocessing_allow_mode); + ma_client_get_audio_streaming_data_type(g_ma, &audio_streaming_data_type); + + if (MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE == audio_streaming_data_type) { + if (MA_PREPROCESSING_ALLOW_UTTERANCE == preprocessing_allow_mode || + MA_PREPROCESSING_ALLOW_ALL == preprocessing_allow_mode) { + if (!preprocessing_result_received) { + // Save audio data + speech_data data; + data.event = event; + data.buffer = buffer; + data.len = len; + ma_client_push_preprocessing_audio_data(g_ma, &data); + return 0; + } + } + } else if (MA_AUDIO_STREAMING_DATA_TYPE_FOLLOW_UP_SPEECH == audio_streaming_data_type) { + if (MA_PREPROCESSING_ALLOW_FOLLOW_UP == preprocessing_allow_mode || + MA_PREPROCESSING_ALLOW_ALL == preprocessing_allow_mode) { + if (!preprocessing_result_received) { + // Save audio data + speech_data data; + data.event = event; + data.buffer = buffer; + data.len = len; + ma_client_push_preprocessing_audio_data(g_ma, &data); + return 0; + } + } + } +#endif + ma_client_get_audio_streaming_cb(g_ma, &callback, &user_data); if (NULL != callback) { @@ -687,6 +737,40 @@ int __ma_cb_audio_streaming_data_section_changed(ma_audio_streaming_data_section return 0; } +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE +int __flush_preprocessing_audio_data() +{ + ma_audio_streaming_cb callback = NULL; + void* user_data; + + ma_client_get_audio_streaming_cb(g_ma, &callback, &user_data); + + if (NULL != callback) { + int event_filter = -1; + SLOG(LOG_DEBUG, TAG_MAC, "Flush audio data"); + ma_client_use_callback(g_ma); + // Flush audio data + speech_data* data = NULL; + do { + ma_client_pop_preprocessing_audio_data(g_ma, &data); + if (data) { + callback((ma_audio_streaming_event_e)data->event, data->buffer, data->len, user_data); + if (event_filter != data->event) { + SLOG(LOG_DEBUG, TAG_MAC, "[DEBUG] Audio streaming callback is called, event(%d)", data->event); + event_filter = data->event; + } + free(data->buffer); + free(data); + } + } while (data); + ma_client_not_use_callback(g_ma); + } else { + SLOG(LOG_DEBUG, TAG_MAC, "[WARNING] Audio streaming callback is NULL"); + } + return 0; +} +#endif + int __ma_cb_preprocessing_result_received(bool result) { ma_preprocessing_result_received_cb callback = NULL; @@ -703,6 +787,32 @@ int __ma_cb_preprocessing_result_received(bool result) SLOG(LOG_DEBUG, TAG_MAC, "[WARNING] Preprocessing result received callback is NULL"); } +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE + ma_client_set_preprocessing_result_received(g_ma, true); + + ma_preprocessing_allow_mode_e preprocessing_allow_mode; + ma_audio_streaming_data_type_e audio_streaming_data_type; + ma_client_get_preprocessing_allow_mode(g_ma, &preprocessing_allow_mode); + ma_client_get_audio_streaming_data_type(g_ma, &audio_streaming_data_type); + if (MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE == audio_streaming_data_type) { + if (MA_PREPROCESSING_ALLOW_UTTERANCE == preprocessing_allow_mode || + MA_PREPROCESSING_ALLOW_ALL == preprocessing_allow_mode) { + if (!result) { + __flush_preprocessing_audio_data(); + return 0; + } + } + } else if (MA_AUDIO_STREAMING_DATA_TYPE_FOLLOW_UP_SPEECH == audio_streaming_data_type) { + if (MA_PREPROCESSING_ALLOW_FOLLOW_UP == preprocessing_allow_mode || + MA_PREPROCESSING_ALLOW_ALL == preprocessing_allow_mode) { + if (!result) { + __flush_preprocessing_audio_data(); + return 0; + } + } + } +#endif + return 0; } @@ -1254,6 +1364,11 @@ int ma_start_receiving_audio_streaming_data(ma_audio_streaming_data_type_e type) SLOG(LOG_DEBUG, TAG_MAC, "[Client DEBUG] Send streaming request to the Multi-assistant"); +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE + ma_client_set_audio_streaming_data_type(g_ma, type); + ma_client_set_preprocessing_result_received(g_ma, false); +#endif + int ret = -1; int pid = getpid(); ret = ma_dbus_start_streaming_audio_data(pid, type); @@ -1648,6 +1763,10 @@ int ma_set_preprocessing_allow_mode(ma_preprocessing_allow_mode_e mode, const ch return MA_ERROR_INVALID_STATE; } +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE + ma_client_set_preprocessing_allow_mode(g_ma, mode); +#endif + /* change system volume */ int pid = getpid(); int ret = ma_dbus_set_preprocessing_allow_mode(pid, mode, app_id); diff --git a/client/ma_client.c b/client/ma_client.c index 72c2f47..1c00ea5 100644 --- a/client/ma_client.c +++ b/client/ma_client.c @@ -51,6 +51,12 @@ typedef struct { ma_active_state_e previous_active_state; ma_active_state_e current_active_state; +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE + ma_preprocessing_allow_mode_e preprocessing_allow_mode; + ma_audio_streaming_data_type_e audio_streaming_data_type; + bool preprocessing_result_received; +#endif + /* mutex */ int cb_ref_count; @@ -63,7 +69,10 @@ typedef struct { /* client list */ static GSList* g_client_list = NULL; - +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE +/* Audio data buffer when in preprocessing mode */ +static GSList* g_audio_data_list = NULL; +#endif static ma_client_s* __client_get(ma_h ma) { @@ -138,6 +147,12 @@ int ma_client_create(ma_h* ma) client->previous_active_state = MA_ACTIVE_STATE_INACTIVE; client->current_active_state = MA_ACTIVE_STATE_INACTIVE; +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE + client->preprocessing_allow_mode = MA_PREPROCESSING_ALLOW_NONE; + client->audio_streaming_data_type = MA_AUDIO_STREAMING_DATA_TYPE_CURRENT_UTTERANCE; + client->preprocessing_result_received = false; +#endif + client->cb_ref_count = 0; g_client_list = g_slist_append(g_client_list, client); @@ -572,3 +587,111 @@ int ma_client_get_preprocessing_result_received_cb(ma_h ma, ma_preprocessing_res return MA_ERROR_NONE; } + +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE +int ma_client_set_preprocessing_allow_mode(ma_h ma, ma_preprocessing_allow_mode_e mode) +{ + ma_client_s* client = __client_get(ma); + + if (NULL == client) + return MA_ERROR_INVALID_PARAMETER; + + client->preprocessing_allow_mode = mode; + + return MA_ERROR_NONE; +} + +int ma_client_get_preprocessing_allow_mode(ma_h ma, ma_preprocessing_allow_mode_e* mode) +{ + ma_client_s* client = __client_get(ma); + + if (NULL == client) + return MA_ERROR_INVALID_PARAMETER; + + *mode = client->preprocessing_allow_mode; + + return MA_ERROR_NONE; +} + +int ma_client_set_audio_streaming_data_type(ma_h ma, ma_audio_streaming_data_type_e type) +{ + ma_client_s* client = __client_get(ma); + + if (NULL == client) + return MA_ERROR_INVALID_PARAMETER; + + client->audio_streaming_data_type = type; + + return MA_ERROR_NONE; +} + +int ma_client_get_audio_streaming_data_type(ma_h ma, ma_audio_streaming_data_type_e* type) +{ + ma_client_s* client = __client_get(ma); + + if (NULL == client) + return MA_ERROR_INVALID_PARAMETER; + + *type = client->audio_streaming_data_type; + + return MA_ERROR_NONE; +} + +int ma_client_set_preprocessing_result_received(ma_h ma, bool received) +{ + ma_client_s* client = __client_get(ma); + + if (NULL == client) + return MA_ERROR_INVALID_PARAMETER; + + client->preprocessing_result_received = received; + + return MA_ERROR_NONE; +} + +int ma_client_get_preprocessing_result_received(ma_h ma, bool* received) +{ + ma_client_s* client = __client_get(ma); + + if (NULL == client) + return MA_ERROR_INVALID_PARAMETER; + + *received = client->preprocessing_result_received; + + return MA_ERROR_NONE; +} + +int ma_client_push_preprocessing_audio_data(ma_h ma, speech_data* data) +{ + speech_data *new_data = (speech_data*)calloc(1, sizeof(speech_data)); + if (NULL == new_data) { + SLOG(LOG_ERROR, TAG_MAC, "[ERROR] Fail to allocate memory"); //LCOV_EXCL_LINE + return MA_ERROR_OUT_OF_MEMORY; + } + new_data->event = data->event; + new_data->len = data->len; + new_data->buffer = (void*)calloc(data->len, sizeof(unsigned char)); + if (NULL == new_data->buffer) { + SLOG(LOG_ERROR, TAG_MAC, "[ERROR] Fail to allocate memory"); //LCOV_EXCL_LINE + free(new_data); + return MA_ERROR_OUT_OF_MEMORY; + } + memcpy(new_data->buffer, data->buffer, data->len); + + g_audio_data_list = g_slist_append(g_audio_data_list, new_data); + + return MA_ERROR_NONE; +} + +int ma_client_pop_preprocessing_audio_data(ma_h ma, speech_data** data) +{ + if (g_slist_length(g_audio_data_list) > 0) { + *data = g_slist_nth_data(g_audio_data_list, 0); + g_audio_data_list = g_slist_remove(g_audio_data_list, *data); + } else { + *data = NULL; + } + + return MA_ERROR_NONE; +} +#endif \ No newline at end of file diff --git a/client/ma_client.h b/client/ma_client.h index 72c6fff..579e2fe 100644 --- a/client/ma_client.h +++ b/client/ma_client.h @@ -27,6 +27,8 @@ extern "C" { #endif +#define MA_PREPROCESSING_SEQUENTIAL_MODE + int ma_client_create(ma_h* ma); int ma_client_destroy(ma_h ma); @@ -92,10 +94,38 @@ int ma_client_set_audio_streaming_data_section_changed_cb(ma_h ma, ma_audio_stre int ma_client_get_audio_streaming_data_section_changed_cb(ma_h ma, ma_audio_streaming_data_section_changed_cb* callback, void** user_data); +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE +typedef void (*ma_preprocessing_result_received_cb)(bool is_success, void* user_data); +#endif + int ma_client_set_preprocessing_result_received_cb(ma_h ma, ma_preprocessing_result_received_cb callback, void* user_data); int ma_client_get_preprocessing_result_received_cb(ma_h ma, ma_preprocessing_result_received_cb* callback, void** user_data); +#ifdef MA_PREPROCESSING_SEQUENTIAL_MODE +int ma_client_set_preprocessing_allow_mode(ma_h ma, ma_preprocessing_allow_mode_e mode); + +int ma_client_get_preprocessing_allow_mode(ma_h ma, ma_preprocessing_allow_mode_e* mode); + +int ma_client_set_audio_streaming_data_type(ma_h ma, ma_audio_streaming_data_type_e type); + +int ma_client_get_audio_streaming_data_type(ma_h ma, ma_audio_streaming_data_type_e* type); + +int ma_client_set_preprocessing_result_received(ma_h ma, bool received); + +int ma_client_get_preprocessing_result_received(ma_h ma, bool* received); + +typedef struct { + int event; + void* buffer; + int len; +} speech_data; + +int ma_client_push_preprocessing_audio_data(ma_h ma, speech_data* data); + +int ma_client_pop_preprocessing_audio_data(ma_h ma, speech_data** data); +#endif + #ifdef __cplusplus } #endif diff --git a/include/multi_assistant.h b/include/multi_assistant.h index 4319f29..c8661db 100644 --- a/include/multi_assistant.h +++ b/include/multi_assistant.h @@ -693,40 +693,6 @@ int ma_set_audio_streaming_data_section_changed_cb(ma_audio_streaming_data_secti */ int ma_unset_audio_streaming_data_section_changed_cb(void); -/** - * @brief Sets the preprocessing result received callback. - * @since_tizen 5.5 - * - * @param[in] callback The callback - * @param[in] user_data The user data passed to the callback function - * - * @return @c 0 on success, otherwise a negative error value - * @retval #MA_ERROR_NONE Successful - * @retval #MA_ERROR_NOT_SUPPORTED Not supported - * @retval #MA_ERROR_INVALID_PARAMETER Invalid parameter - * @retval #MA_ERROR_INVALID_STATE Invalid state - * - * @pre The state should be #MA_STATE_INITIALIZED. - * @see ma_preprocessing_result_received_cb() - * @see ma_unset_preprocessing_result_received_cb() - */ -int ma_set_preprocessing_result_received_cb(ma_preprocessing_result_received_cb callback, void* user_data); - -/** - * @brief Unsets the preprocessing information changed callback. - * @since_tizen 5.5 - * - * @return @c 0 on success, otherwise a negative error value - * @retval #MA_ERROR_NONE Successful - * @retval #MA_ERROR_NOT_SUPPORTED Not supported - * @retval #MA_ERROR_INVALID_STATE Invalid state - * - * @pre The state should be #MA_STATE_INITIALIZED. - * @see ma_preprocessing_result_received_cb() - * @see ma_set_preprocessing_result_received_cb() - */ -int ma_unset_preprocessing_result_received_cb(void); - #ifdef __cplusplus } #endif diff --git a/include/multi_assistant_common.h b/include/multi_assistant_common.h index d4922f1..033f347 100644 --- a/include/multi_assistant_common.h +++ b/include/multi_assistant_common.h @@ -283,15 +283,6 @@ typedef void (*ma_preprocessing_information_changed_cb)(const char* app_id, void */ typedef void (*ma_audio_streaming_data_section_changed_cb)(ma_audio_streaming_data_section_e section, void* user_data); -/** - * @brief Called when the preprocessing result is received. - * @since_tizen 5.5 - * - * @param[in] is_success The value indicating whether the preprocessing succeeded or not - * @param[in] user_data The user data passed from the callback registration function - */ -typedef void (*ma_preprocessing_result_received_cb)(bool is_success, void* user_data); - #ifdef __cplusplus } #endif -- 2.34.1