From 6276d315f8604deb6692ecdaf26540f545ddf402 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Mon, 8 Oct 2018 10:24:22 +0900 Subject: [PATCH] Launch MA Client dynamically when required Change-Id: I6caea5858fa335905a8844d79149bb71bcb83895 --- inc/multi_assistant_main.h | 1 + inc/multi_assistant_service.h | 13 +- src/multi_assistant_dbus.c | 3 + src/multi_assistant_dbus_server.c | 51 +++++- src/multi_assistant_dbus_server.h | 2 + src/multi_assistant_service.c | 239 +++++++++++++++++++++++++-- src/multi_assistant_service_plugin.c | 35 ++-- 7 files changed, 310 insertions(+), 34 deletions(-) diff --git a/inc/multi_assistant_main.h b/inc/multi_assistant_main.h index 65dc3d7..5f29664 100644 --- a/inc/multi_assistant_main.h +++ b/inc/multi_assistant_main.h @@ -53,6 +53,7 @@ #define MA_METHOD_INITIALIZE "ma_method_initialize" #define MA_METHOD_DEINITIALIZE "ma_method_deinitialize" +#define MA_METHOD_REQUEST_SPEECH_DATA "ma_method_request_speech_data" #define MA_METHOD_GET_RECORDING_AUDIO_FORMAT "ma_method_get_recording_audio_format" #define MA_METHOD_SEND_ASR_RESULT "ma_method_send_asr_result" #define MA_METHOD_SEND_RESULT "ma_method_send_result" diff --git a/inc/multi_assistant_service.h b/inc/multi_assistant_service.h index 30aee06..ca981d9 100644 --- a/inc/multi_assistant_service.h +++ b/inc/multi_assistant_service.h @@ -25,6 +25,15 @@ extern "C" { #endif +#define MAX_APPID_LEN 255 +typedef struct { + int pid; + char appid[MAX_APPID_LEN]; +} ma_client_s; + +int mas_check_client_available(); + +int mas_launch_client(); int mas_client_initialize(int pid); @@ -32,6 +41,8 @@ int mas_client_deinitialize(int pid); int mas_client_get_audio_format(int pid, int* rate, int* channel, int* audio_type); +int mas_client_request_speech_data(int pid); + int mas_client_send_asr_result(int pid, int event, char* asr_result); int mas_client_send_result(int pid, char* display_text, char* utterance_text, char* result_json); @@ -41,7 +52,7 @@ int mas_ui_client_initialize(int pid); int mas_ui_client_deinitialize(int pid); int mas_ui_client_change_assistant(char* app_id); - + #ifdef __cplusplus } diff --git a/src/multi_assistant_dbus.c b/src/multi_assistant_dbus.c index 76bf0de..e0e9d3d 100644 --- a/src/multi_assistant_dbus.c +++ b/src/multi_assistant_dbus.c @@ -523,6 +523,9 @@ static Eina_Bool listener_event_callback(void* data, Ecore_Fd_Handler *fd_handle } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_DEINITIALIZE)) { ma_service_dbus_deinitialize(g_conn_listener, msg); + } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_REQUEST_SPEECH_DATA)) { + ma_service_dbus_request_speech_data(g_conn_listener, msg); + } else if (dbus_message_is_method_call(msg, MA_SERVER_SERVICE_INTERFACE, MA_METHOD_GET_RECORDING_AUDIO_FORMAT)) { ma_service_dbus_get_audio_format(g_conn_listener, msg); diff --git a/src/multi_assistant_dbus_server.c b/src/multi_assistant_dbus_server.c index b7d7a79..377c805 100644 --- a/src/multi_assistant_dbus_server.c +++ b/src/multi_assistant_dbus_server.c @@ -19,7 +19,6 @@ #include "multi_assistant_dbus_server.h" #include "multi_assistant_service.h" - /* * Dbus Client-Daemon Server */ @@ -146,6 +145,56 @@ int ma_service_dbus_deinitialize(DBusConnection* conn, DBusMessage* msg) return 0; } +int ma_service_dbus_request_speech_data(DBusConnection* conn, DBusMessage* msg) +{ + DBusError err; + dbus_error_init(&err); + + int pid = -1; + int ret; + + dbus_message_get_args(msg, &err, + DBUS_TYPE_INT32, &pid, + DBUS_TYPE_INVALID); + + MAS_LOGD("[DEBUG] MAS REQUEST SPEECH DATA"); + + if (dbus_error_is_set(&err)) { + MAS_LOGE("[IN ERROR] mas request speech data : Get arguments error (%s)", err.message); + dbus_error_free(&err); + ret = -1; //MAS_ERROR_OPERATION_FAILED; + } else { + MAS_LOGD("[IN] mas request speech data"); + ret = mas_client_request_speech_data(pid); + } + + DBusMessage* reply; + reply = dbus_message_new_method_return(msg); + + if (NULL != reply) { + /* Append result and voice */ + dbus_message_append_args(reply, + DBUS_TYPE_INT32, &ret, + DBUS_TYPE_INVALID); + if (0 == ret) { + MAS_LOGD("[OUT] mas request speech data : result(%d)", ret); + } else { + MAS_LOGE("[OUT ERROR] mas request speech data : result(%d)", ret); + } + + if (!dbus_connection_send(conn, reply, NULL)) { + MAS_LOGE("[OUT ERROR] mas request speech data : Out Of Memory!"); + } + + dbus_connection_flush(conn); + dbus_message_unref(reply); + } else { + MAS_LOGE("[OUT ERROR] request speech data : Fail to create reply message!!"); + } + + return 0; +} + int ma_service_dbus_get_audio_format(DBusConnection* conn, DBusMessage* msg) { DBusError err; diff --git a/src/multi_assistant_dbus_server.h b/src/multi_assistant_dbus_server.h index 1b75d8a..87347b9 100644 --- a/src/multi_assistant_dbus_server.h +++ b/src/multi_assistant_dbus_server.h @@ -29,6 +29,8 @@ int ma_service_dbus_initialize(DBusConnection* conn, DBusMessage* msg); int ma_service_dbus_deinitialize(DBusConnection* conn, DBusMessage* msg); +int ma_service_dbus_request_speech_data(DBusConnection* conn, DBusMessage* msg); + int ma_service_dbus_get_audio_format(DBusConnection* conn, DBusMessage* msg); int ma_service_dbus_send_asr_result(DBusConnection* conn, DBusMessage* msg); diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index b935d84..078a4da 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -35,17 +36,212 @@ static const char *g_current_lang = "en_US"; static int g_local_port_id = -1; +static char g_current_maclient_appid[MAX_APPID_LEN] = {"com.samsung.bixby-voice"}; +static ma_client_s *g_current_maclient = NULL; +/* The following variable would not be needed when ma_request_speech_data() gets implemented */ +static char g_launching_maclient_appid[MAX_APPID_LEN] = {""}; + +/* client list */ +static GSList* g_client_list = NULL; + +int mas_check_client_available() +{ + if (g_current_maclient) { + if (g_current_maclient->appid && 0 == strncmp(g_current_maclient_appid, g_current_maclient->appid, MAX_APPID_LEN)) { + MAS_LOGD("MA client with appid %s currently seems to be available : %d", + g_current_maclient_appid, g_current_maclient->pid); + return 1; + } + } + return 0; +} + +int mas_launch_client() +{ + app_control_h app_control; + int ret = 0; + + MAS_LOGD("Current MA Client appid : %s", g_current_maclient_appid); + + ret = app_control_create(&app_control); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW("app_control_create returned %08x", ret); + return -1; + } + + ret = app_control_set_operation(app_control, APP_CONTROL_OPERATION_DEFAULT); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW("app_control_set_operation returned %08x", ret); + app_control_destroy(app_control); + return -1; + } + + /* These extra data key/value should be defined somewhere in multi-assistant framework */ + ret = app_control_add_extra_data(app_control, "request_speech_data", "true"); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW("app_control_add_extra_data returned %08x", ret); + app_control_destroy(app_control); + return -1; + } + + ret = app_control_set_app_id(app_control, g_current_maclient_appid); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW("app_control_set_app_id returned %08x", ret); + app_control_destroy(app_control); + return -1; + } + + ret = app_control_send_launch_request(app_control, NULL, NULL); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW ("app_control_send_launch_request returned %08x, app_id=%s", ret, g_current_maclient_appid); + } + app_control_destroy (app_control); + + if (APP_CONTROL_ERROR_NONE == ret) { + strncpy(g_launching_maclient_appid, g_current_maclient_appid, MAX_APPID_LEN); + g_launching_maclient_appid[MAX_APPID_LEN - 1] = '\0'; + } + + return ret; +} + +int ma_client_create(ma_client_s info) +{ + ma_client_s* data = NULL; + + data = (ma_client_s*)calloc(1, sizeof(ma_client_s)); + if (NULL == data) { + MAS_LOGE("[ERROR] Fail to allocate memory"); //LCOV_EXCL_LINE + return -1;// MA_ERROR_OUT_OF_MEMORY; + } + + *data = info; + + g_client_list = g_slist_append(g_client_list, data); + + return 0; +} + +int ma_client_destroy(ma_client_s *client) +{ + if (NULL == client) { + MAS_LOGE("Input parameter is NULL"); //LCOV_EXCL_LINE + return -1;// MA_ERROR_OPERATION_FAILED; + } + + g_client_list = g_slist_remove(g_client_list, client); + + free(client); + client = NULL; + + return 0; +} + +ma_client_s* ma_client_find_by_appid(const char *appid) +{ + if (NULL == appid) { + MAS_LOGE("Input parameter is NULL"); //LCOV_EXCL_LINE + return -1;// MA_ERROR_OPERATION_FAILED; + } + + ma_client_s *data = NULL; + + int count = g_slist_length(g_client_list); + int i; + + for (i = 0; i < count; i++) { + data = g_slist_nth_data(g_client_list, i); + + if (NULL != data) { + if (0 == strncmp(data->appid, appid, MAX_APPID_LEN)) { + return data; + } + } + } + + MAS_LOGE("[ERROR] client Not found"); + + return NULL; +} + +ma_client_s* ma_client_find_by_pid(int pid) +{ + ma_client_s *data = NULL; + + int count = g_slist_length(g_client_list); + int i; + + for (i = 0; i < count; i++) { + data = g_slist_nth_data(g_client_list, i); + + if (NULL != data) { + if (data->pid == pid) { + return data; + } + } + } + + MAS_LOGE("[ERROR] client Not found"); + + return NULL; +} + int mas_client_initialize(int pid) { MAS_LOGD("[Enter] pid(%d)", pid); + char appid[MAX_APPID_LEN] = {'\0',}; + if (AUL_R_OK == aul_app_get_appid_bypid(pid, appid, sizeof(appid))) { + appid[MAX_APPID_LEN - 1] = '\0'; + + MAS_LOGD("appid for pid %d is : %s", pid, (appid ? appid : "Unknown")); + + /* Remove existing client that has same appid, if there's any */ + ma_client_s *old_client = NULL; + old_client = ma_client_find_by_appid(appid); + if (old_client) { + ma_client_destroy(old_client); + old_client = NULL; + } + + /* And remove a client that has same pid also */ + old_client = ma_client_find_by_pid(pid); + if (old_client) { + ma_client_destroy(old_client); + old_client = NULL; + } + + ma_client_s new_client; + new_client.pid = pid; + strncpy(new_client.appid, appid, MAX_APPID_LEN); + new_client.appid[MAX_APPID_LEN - 1] = '\0'; + ma_client_create(new_client); + + if (0 == strncmp(g_current_maclient_appid, appid, MAX_APPID_LEN)) { + g_current_maclient = &new_client; + MAS_LOGD("MA client with current maclient appid connected!"); + + /* Since the ma_request_speech_data() is not available, we will request speech data here. + Need to remove below code when ma_request_speech_data() get implemented. + */ + if (0 == strncmp(g_launching_maclient_appid, appid, MAX_APPID_LEN)) { + g_launching_maclient_appid[0] = '\0'; + mas_client_request_speech_data(pid); + } + } + } + return 0; } int mas_client_deinitialize(int pid) { MAS_LOGD("[Enter] pid(%d)", pid); - + ma_client_s *client = ma_client_find_by_pid(pid); + if (client) { + ma_client_destroy(client); + client = NULL; + } return 0; } @@ -61,6 +257,29 @@ int mas_client_get_audio_format(int pid, int* rate, int* channel, int* audio_typ return ret; } +int mas_client_request_speech_data(int pid) +{ + int ret = -1; + MAS_LOGD("[Enter] pid(%d)", pid); + + char appid[MAX_APPID_LEN] = {'\0',}; + if (AUL_R_OK == aul_app_get_appid_bypid(pid, appid, sizeof(appid))) { + appid[MAX_APPID_LEN - 1] = '\0'; + + if (0 == strncmp(g_current_maclient_appid, appid, MAX_APPID_LEN)) { + MAS_LOGD("appid %s matches with current MA Client, requesting speech data", appid); + ret = wakeup_service_request_speech_data(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to request speech data(%d)", ret); + } + } else { + MAS_LOGE("appid %s does not match with current MA Client", appid); + } + } + + return ret; +} + int mas_client_send_asr_result(int pid, int event, char* asr_result) { MAS_LOGD("[Enter] pid(%d), event(%d), asr_result(%s)", pid, event, asr_result); @@ -69,7 +288,7 @@ int mas_client_send_asr_result(int pid, int event, char* asr_result) MAS_LOGE("[ERROR] Fail to send asr result, ret(%d)", ret); } - // if final event is , launch assistant app which is invoked with wakeup word. + // if final event is , launch assistant app which is invoked with wakeup word. /* TO_DO */ return ret; } @@ -157,7 +376,7 @@ static int init_wakeup(void) { MAS_LOGD("[Enter] init_wakeup "); - int ret = mas_dbus_open_connection(); + int ret = mas_dbus_open_connection(); if (0 != ret) { MAS_LOGE("[ERROR] Fail to open connection"); } @@ -213,7 +432,7 @@ static void deinit_wakeup(void) */ deinit_message_port(); - int ret = mas_dbus_close_connection(); + int ret = mas_dbus_close_connection(); if (0 != ret) { MAS_LOGE("[ERROR] Fail to close connection"); } @@ -226,7 +445,7 @@ static void deinit_wakeup(void) bool service_app_create(void *data) { - // Todo: add your code here. + // Todo: add your code here. MAS_LOGD("[Enter] Service app create"); @@ -240,15 +459,15 @@ bool service_app_create(void *data) void service_app_terminate(void *data) { - // Todo: add your code here. + // Todo: add your code here. deinit_wakeup(); - return; + return; } void service_app_control(app_control_h app_control, void *data) { - // Todo: add your code here. - return; + // Todo: add your code here. + return; } static void @@ -278,7 +497,7 @@ service_app_low_memory(app_event_info_h event_info, void *user_data) int main(int argc, char* argv[]) { - char ad[50] = {0,}; + char ad[50] = {0,}; service_app_lifecycle_callback_s event_callback; app_event_handler_h handlers[5] = {NULL, }; diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index ece29e4..cab000d 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -54,7 +54,7 @@ Eina_Bool __send_asr_result(void *data) if (!strcmp((char*)data, "Today's")) { masc_ui_dbus_send_asr_result(-1, 1, "Today's"); } - if (!strcmp((char*)data, "weather.")) { + if (!strcmp((char*)data, "weather.")) { masc_ui_dbus_send_asr_result(-1, 0, "Today's weather."); } @@ -81,18 +81,24 @@ Eina_Bool __request_speech_data(void *data) { MAS_LOGD("[ENTER]"); - int ret = wakeup_service_request_speech_data(); - if (0 != ret) { - MAS_LOGE("[ERROR] Fail to request speech data(%d)", ret); + if (mas_check_client_available()) { + int ret = wakeup_service_request_speech_data(); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to request speech data(%d)", ret); + } + } else { + // Appropriate MA Client not available, trying to launch new one + mas_launch_client(); } MAS_LOGD("END"); return EINA_FALSE; } + static void __wakeup_event_cb(wakeup_service_wakeup_event_e event, const char* wakeup_word, void* user_data) { - MAS_LOGD( "[SUCCESS] __wakeup_event_cb is called, event(%d), wakeup_word(%s)", event, wakeup_word); + MAS_LOGD("[SUCCESS] __wakeup_event_cb is called, event(%d), wakeup_word(%s)", event, wakeup_word); int ret = -1; int retry_cnt = 0; while (0 != ret) { @@ -142,19 +148,6 @@ static void __wakeup_event_cb(wakeup_service_wakeup_event_e event, const char* w } #endif - // app control background launch - /* TO_DO */ - ret = -1; - retry_cnt = 0; - while (0 != ret) { - ret = masc_dbus_send_hello(getpid()); - retry_cnt++; - if (30 == retry_cnt) { - MAS_LOGE("[ERROR] Fail to receive reply for hello, ret(%d)", ret); - break; - } - } - #if 0 /* + TEST_CODE */ if (WAKEUP_EVENT_SUCCESS == event) { ecore_thread_main_loop_begin(); @@ -170,14 +163,12 @@ static void __wakeup_event_cb(wakeup_service_wakeup_event_e event, const char* w ecore_thread_main_loop_end(); } #endif /* - TEST_CODE */ - - // if App is already launched, request speech data - /* TO_DO */ if (WAKEUP_EVENT_SUCCESS == event) { ecore_thread_main_loop_begin(); - ecore_timer_add(0, __request_speech_data, NULL); + ecore_timer_add(0.0f, __request_speech_data, NULL); ecore_thread_main_loop_end(); } + } static void __speech_streaming_cb(wakeup_service_speech_streaming_event_e event, unsigned char* buffer, int len, void *user_data) -- 2.34.1