From 3ceadf491776deecf30085f119ef1bebd17d9fb9 Mon Sep 17 00:00:00 2001 From: Ji-hoon Lee Date: Thu, 11 Oct 2018 15:25:29 +0900 Subject: [PATCH] Support launching clients based on recognized wakeup word Change-Id: I022daeb1269346a084f86638d6faf5fe4b8bc6c9 --- inc/multi_assistant_main.h | 12 +- inc/multi_assistant_service.h | 15 +- src/multi_assistant_dbus.c | 10 +- src/multi_assistant_dbus.h | 2 +- src/multi_assistant_service.c | 270 ++++++++++++++++++--------- src/multi_assistant_service_plugin.c | 27 ++- 6 files changed, 226 insertions(+), 110 deletions(-) diff --git a/inc/multi_assistant_main.h b/inc/multi_assistant_main.h index 5f29664..3394f87 100644 --- a/inc/multi_assistant_main.h +++ b/inc/multi_assistant_main.h @@ -11,14 +11,14 @@ #define LOG_TAG "multi-assistant" #define MAS_SECURE_LOG_(id, prio, tag, fmt, arg...) \ - ({ do { \ - __dlog_print(id, prio, tag, "%s: %s(%d) > [SECURE_LOG] " fmt, __MODULE__, __func__, __LINE__, ##arg); \ - } while (0); }) + ({ do { \ + __dlog_print(id, prio, tag, "%s: %s(%d) > [SECURE_LOG] " fmt, __MODULE__, __func__, __LINE__, ##arg); \ + } while (0); }) #define MAS_LOG_(prio, tag, fmt, arg...) \ - ({ do { \ - dlog_print(prio, tag, "%s: %s(%d) > " fmt, __MODULE__, __func__, __LINE__, ##arg); \ - } while (0); }) + ({ do { \ + dlog_print(prio, tag, "%s: %s(%d) > " fmt, __MODULE__, __func__, __LINE__, ##arg); \ + } while (0); }) #define MAS_LOGD(fmt, args...) MAS_LOG_(DLOG_DEBUG, LOG_TAG, fmt, ##args) #define MAS_LOGI(fmt, args...) MAS_LOG_(DLOG_INFO, LOG_TAG, fmt, ##args) diff --git a/inc/multi_assistant_service.h b/inc/multi_assistant_service.h index ca981d9..d4ca45e 100644 --- a/inc/multi_assistant_service.h +++ b/inc/multi_assistant_service.h @@ -26,14 +26,6 @@ extern "C" { #endif #define MAX_APPID_LEN 255 -typedef struct { - int pid; - char appid[MAX_APPID_LEN]; -} ma_client_s; - -int mas_check_client_available(); - -int mas_launch_client(); int mas_client_initialize(int pid); @@ -53,6 +45,13 @@ int mas_ui_client_deinitialize(int pid); int mas_ui_client_change_assistant(char* app_id); +int mas_get_current_client_pid(); + +int mas_get_client_pid_by_wakeup_word(const char *wakeup_word); + +int mas_set_current_client_by_wakeup_word(const char *wakeup_word); + +int mas_launch_client_by_wakeup_word(const char *wakeup_word); #ifdef __cplusplus } diff --git a/src/multi_assistant_dbus.c b/src/multi_assistant_dbus.c index e0e9d3d..780f611 100644 --- a/src/multi_assistant_dbus.c +++ b/src/multi_assistant_dbus.c @@ -173,7 +173,7 @@ int masc_dbus_send_error_message(int reason, const char* err_msg) return 0; } -int masc_dbus_send_speech_data(int event, unsigned char* data, unsigned int data_size) +int masc_dbus_send_speech_data(int pid, int event, unsigned char* data, unsigned int data_size) { if (0 != __dbus_check()) { return -1; //MAS_ERROR_OPERATION_FAILED; @@ -183,8 +183,12 @@ int masc_dbus_send_speech_data(int event, unsigned char* data, unsigned int data DBusError err; dbus_error_init(&err); + char service_name[64]; + memset(service_name, '\0', 64); + snprintf(service_name, 64, "%s_%d", MA_CLIENT_SERVICE_NAME, pid); + msg = dbus_message_new_method_call( - MA_CLIENT_SERVICE_NAME, + service_name, MA_CLIENT_SERVICE_OBJECT_PATH, MA_CLIENT_SERVICE_INTERFACE, MAS_METHOD_SEND_SPEECH_DATA); @@ -193,7 +197,7 @@ int masc_dbus_send_speech_data(int event, unsigned char* data, unsigned int data MAS_LOGE(">>>> Request mas send speech data : Fail to make message"); return -1; // MAS_ERROR_OPERATION_FAILED; } else { - MAS_LOGD(">>>> Request mas send speech data"); + MAS_LOGD(">>>> Request mas send speech data : %s", service_name); } if (true != dbus_message_append_args(msg, diff --git a/src/multi_assistant_dbus.h b/src/multi_assistant_dbus.h index cf3de55..3b54aa4 100644 --- a/src/multi_assistant_dbus.h +++ b/src/multi_assistant_dbus.h @@ -29,7 +29,7 @@ int masc_dbus_send_hello(int pid); int masc_dbus_send_error_message(int reason, const char* err_msg); -int masc_dbus_send_speech_data(int event, unsigned char* data, unsigned int data_size); +int masc_dbus_send_speech_data(int pid, int event, unsigned char* data, unsigned int data_size); int masc_ui_dbus_send_hello(void); diff --git a/src/multi_assistant_service.c b/src/multi_assistant_service.c index 4e8ef50..d923b43 100644 --- a/src/multi_assistant_service.c +++ b/src/multi_assistant_service.c @@ -36,75 +36,31 @@ static const char *g_current_lang = "en_US"; static int g_local_port_id = -1; -static char g_current_maclient_appid[MAX_APPID_LEN] = {"com.samsung.bixby-voice"}; -static ma_client_s *g_current_maclient = NULL; -/* The following variable would not be needed when ma_request_speech_data() gets implemented */ -static char g_launching_maclient_appid[MAX_APPID_LEN] = {""}; +#define MAX_MACLIENT_INFO_NUM 16 +#define MAX_WAKEUP_WORD_LEN 255 +typedef struct { + Eina_Bool used; + const char wakeup_word[MAX_WAKEUP_WORD_LEN]; + const char appid[MAX_APPID_LEN]; +} ma_client_info; + +static ma_client_info g_maclient_info[MAX_MACLIENT_INFO_NUM] = { + {EINA_TRUE, {"hi bixby"}, {"com.samsung.bixby-voice"}}, + {EINA_TRUE, {"alexa"}, {"com.samsung.alexa-voice"}}, + {EINA_TRUE, {"hi google"}, {"com.samsung.google-voice"}}, +}; + +typedef struct { + int pid; + char appid[MAX_APPID_LEN]; +} ma_client_s; + +static int g_current_maclient_info = 0; +static const char *g_launching_maclient_appid = NULL; /* client list */ static GSList* g_client_list = NULL; -int mas_check_client_available() -{ - if (g_current_maclient) { - if (g_current_maclient->appid && 0 == strncmp(g_current_maclient_appid, g_current_maclient->appid, MAX_APPID_LEN)) { - MAS_LOGD("MA client with appid %s currently seems to be available : %d", - g_current_maclient_appid, g_current_maclient->pid); - return 1; - } - } - return 0; -} - -int mas_launch_client() -{ - app_control_h app_control; - int ret = 0; - - MAS_LOGD("Current MA Client appid : %s", g_current_maclient_appid); - - ret = app_control_create(&app_control); - if (APP_CONTROL_ERROR_NONE != ret) { - MAS_LOGW("app_control_create returned %08x", ret); - return -1; - } - - ret = app_control_set_operation(app_control, APP_CONTROL_OPERATION_DEFAULT); - if (APP_CONTROL_ERROR_NONE != ret) { - MAS_LOGW("app_control_set_operation returned %08x", ret); - app_control_destroy(app_control); - return -1; - } - - /* These extra data key/value should be defined somewhere in multi-assistant framework */ - ret = app_control_add_extra_data(app_control, "request_speech_data", "true"); - if (APP_CONTROL_ERROR_NONE != ret) { - MAS_LOGW("app_control_add_extra_data returned %08x", ret); - app_control_destroy(app_control); - return -1; - } - - ret = app_control_set_app_id(app_control, g_current_maclient_appid); - if (APP_CONTROL_ERROR_NONE != ret) { - MAS_LOGW("app_control_set_app_id returned %08x", ret); - app_control_destroy(app_control); - return -1; - } - - ret = app_control_send_launch_request(app_control, NULL, NULL); - if (APP_CONTROL_ERROR_NONE != ret) { - MAS_LOGW ("app_control_send_launch_request returned %08x, app_id=%s", ret, g_current_maclient_appid); - } - app_control_destroy (app_control); - - if (APP_CONTROL_ERROR_NONE == ret) { - strncpy(g_launching_maclient_appid, g_current_maclient_appid, MAX_APPID_LEN); - g_launching_maclient_appid[MAX_APPID_LEN - 1] = '\0'; - } - - return ret; -} - int ma_client_create(ma_client_s info) { ma_client_s* data = NULL; @@ -141,7 +97,7 @@ ma_client_s* ma_client_find_by_appid(const char *appid) { if (NULL == appid) { MAS_LOGE("Input parameter is NULL"); //LCOV_EXCL_LINE - return -1;// MA_ERROR_OPERATION_FAILED; + return NULL; } ma_client_s *data = NULL; @@ -194,8 +150,8 @@ Eina_Bool __request_speech_data_timer(void *data) char appid[MAX_APPID_LEN] = {'\0',}; if (AUL_R_OK == aul_app_get_appid_bypid(pid, appid, sizeof(appid))) { appid[MAX_APPID_LEN - 1] = '\0'; - if (strncmp(g_launching_maclient_appid, appid, MAX_APPID_LEN) == 0) { - g_launching_maclient_appid[0] = '\0'; + if (0 == strncmp(g_launching_maclient_appid, appid, MAX_APPID_LEN)) { + g_launching_maclient_appid = NULL; mas_client_request_speech_data(pid); } } @@ -235,15 +191,23 @@ int mas_client_initialize(int pid) new_client.appid[MAX_APPID_LEN - 1] = '\0'; ma_client_create(new_client); - if (0 == strncmp(g_current_maclient_appid, appid, MAX_APPID_LEN)) { - g_current_maclient = &new_client; + const char *current_maclient_appid = NULL; + if (g_current_maclient_info >= 0 && g_current_maclient_info < MAX_MACLIENT_INFO_NUM) { + current_maclient_appid = g_maclient_info[g_current_maclient_info].appid; + } + + if (current_maclient_appid && 0 == strncmp(current_maclient_appid, appid, MAX_APPID_LEN)) { MAS_LOGD("MA client with current maclient appid connected!"); /* Since the ma_request_speech_data() is not available, we will request speech data here. Need to remove below code when ma_request_speech_data() get implemented. */ ecore_timer_add(3.0f, __request_speech_data_timer, (void*)(pid)); + } else { + MAS_LOGD("MA client connected, but its appid does not match with current maclient"); } + } else { + MAS_LOGE("[ERROR] Fail to retrieve appid"); } return 0; @@ -280,8 +244,12 @@ int mas_client_request_speech_data(int pid) char appid[MAX_APPID_LEN] = {'\0',}; if (AUL_R_OK == aul_app_get_appid_bypid(pid, appid, sizeof(appid))) { appid[MAX_APPID_LEN - 1] = '\0'; + const char *current_maclient_appid = NULL; + if (g_current_maclient_info >= 0 && g_current_maclient_info < MAX_MACLIENT_INFO_NUM) { + current_maclient_appid = g_maclient_info[g_current_maclient_info].appid; + } - if (0 == strncmp(g_current_maclient_appid, appid, MAX_APPID_LEN)) { + if (current_maclient_appid && 0 == strncmp(current_maclient_appid, appid, MAX_APPID_LEN)) { MAS_LOGD("appid %s matches with current MA Client, requesting speech data", appid); ret = wakeup_service_request_speech_data(); if (0 != ret) { @@ -406,19 +374,17 @@ static int init_wakeup(void) return -1; } - if (0 != multi_assistant_service_plugin_set_wakeup_word("en_US", "hi bixby")) { - MAS_LOGE("Fail to stt wakeup word"); - return -1; - } - - if (0 != multi_assistant_service_plugin_set_wakeup_word("en_US", "alexa")) { - MAS_LOGE("Fail to stt wakeup word"); - return -1; - } - - if (0 != multi_assistant_service_plugin_set_wakeup_word("en_US", "hi google")) { - MAS_LOGE("Fail to stt wakeup word"); - return -1; + for (int loop = 0; loop < MAX_MACLIENT_INFO_NUM; loop++) { + if (g_maclient_info[loop].used && + strlen(g_maclient_info[loop].appid) > 0 && + strlen(g_maclient_info[loop].wakeup_word) > 0) { + MAS_LOGD("Registering wakeup word %s for app %s", + g_maclient_info[loop].wakeup_word, g_maclient_info[loop].appid); + if (0 != multi_assistant_service_plugin_set_wakeup_word("en_US", g_maclient_info[loop].wakeup_word)) { + MAS_LOGE("Fail to stt wakeup word"); + return -1; + } + } } if (0 != multi_assistant_service_plugin_set_callbacks()) { @@ -457,6 +423,142 @@ static void deinit_wakeup(void) } } +static const char* __get_client_appid_by_wakeup_word(const char *wakeup_word) +{ + int loop; + const char *appid = NULL; + + if (NULL == wakeup_word) return NULL; + + for (loop = 0; loop < MAX_MACLIENT_INFO_NUM && appid == NULL; loop++) { + if (g_maclient_info[loop].used && + strlen(g_maclient_info[loop].appid) > 0 && + strlen(g_maclient_info[loop].wakeup_word) > 0) { + if (0 == strncmp(wakeup_word, g_maclient_info[loop].wakeup_word, MAX_WAKEUP_WORD_LEN)) { + appid = g_maclient_info[loop].appid; + } + } + } + + /* Perform extended search, by eliminating blank characters */ + if (NULL == appid) { + for (loop = 0; loop < MAX_MACLIENT_INFO_NUM && appid == NULL; loop++) { + if (g_maclient_info[loop].used && + strlen(g_maclient_info[loop].appid) > 0 && + strlen(g_maclient_info[loop].wakeup_word) > 0) { + char comparand[MAX_WAKEUP_WORD_LEN]; + int comparand_index = 0; + for (int index = 0; index < MAX_WAKEUP_WORD_LEN; index++) { + if (' ' != g_maclient_info[loop].wakeup_word[index]) { + comparand[comparand_index++] = g_maclient_info[loop].wakeup_word[index]; + } + } + if (0 == strncmp(wakeup_word, comparand, MAX_WAKEUP_WORD_LEN)) { + appid = g_maclient_info[loop].appid; + } + } + } + } + + return appid; +} + +int mas_get_current_client_pid() +{ + int ret = -1; + if (g_current_maclient_info >= 0 && g_current_maclient_info < MAX_MACLIENT_INFO_NUM) { + const char *current_maclient_appid = g_maclient_info[g_current_maclient_info].appid; + ma_client_s* client = ma_client_find_by_appid(current_maclient_appid); + if (client) { + ret = client->pid; + } + } + return ret; +} + +int mas_get_client_pid_by_wakeup_word(const char *wakeup_word) +{ + int ret = -1; + + const char *appid = __get_client_appid_by_wakeup_word(wakeup_word); + if (appid) { + ma_client_s *client = NULL; + client = ma_client_find_by_appid(appid); + if (client) { + ret = client->pid; + } + } + + return ret; +} + +int mas_set_current_client_by_wakeup_word(const char *wakeup_word) +{ + int ret = -1; + + for (int loop = 0; loop < MAX_MACLIENT_INFO_NUM; loop++) { + if (g_maclient_info[loop].used && + strlen(g_maclient_info[loop].appid) > 0 && + strlen(g_maclient_info[loop].wakeup_word) > 0) { + if (0 == strncmp(wakeup_word, g_maclient_info[loop].wakeup_word, MAX_WAKEUP_WORD_LEN)) { + g_current_maclient_info = loop; + } + } + } + return ret; +} + +int mas_launch_client_by_wakeup_word(const char *wakeup_word) +{ + app_control_h app_control; + int ret = 0; + + const char *appid = __get_client_appid_by_wakeup_word(wakeup_word); + if (NULL == appid || 0 == strlen(appid)) { + MAS_LOGW("could not find appropriate appid with wakeup word %s", (wakeup_word ? wakeup_word : "NULL")); + return -1; + } + + ret = app_control_create(&app_control); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW("app_control_create returned %08x", ret); + return -1; + } + + ret = app_control_set_operation(app_control, APP_CONTROL_OPERATION_DEFAULT); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW("app_control_set_operation returned %08x", ret); + app_control_destroy(app_control); + return -1; + } + + /* These extra data key/value should be defined somewhere in multi-assistant framework */ + ret = app_control_add_extra_data(app_control, "request_speech_data", "true"); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW("app_control_add_extra_data returned %08x", ret); + app_control_destroy(app_control); + return -1; + } + + ret = app_control_set_app_id(app_control, appid); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW("app_control_set_app_id returned %08x", ret); + app_control_destroy(app_control); + return -1; + } + + ret = app_control_send_launch_request(app_control, NULL, NULL); + if (APP_CONTROL_ERROR_NONE != ret) { + MAS_LOGW ("app_control_send_launch_request returned %08x, app_id=%s", ret, appid); + } + app_control_destroy (app_control); + + if (APP_CONTROL_ERROR_NONE != ret) { + g_launching_maclient_appid = appid; + } + + return ret; +} bool service_app_create(void *data) { diff --git a/src/multi_assistant_service_plugin.c b/src/multi_assistant_service_plugin.c index cab000d..8abfbe7 100644 --- a/src/multi_assistant_service_plugin.c +++ b/src/multi_assistant_service_plugin.c @@ -81,21 +81,27 @@ Eina_Bool __request_speech_data(void *data) { MAS_LOGD("[ENTER]"); - if (mas_check_client_available()) { + char *wakeup_word = (char*)data; + + mas_set_current_client_by_wakeup_word(wakeup_word); + if (mas_get_client_pid_by_wakeup_word(wakeup_word) != -1) { + MAS_LOGD("MA Client with wakeup word %s exists, requesting speech data", (wakeup_word ? wakeup_word : "NULL")); int ret = wakeup_service_request_speech_data(); if (0 != ret) { MAS_LOGE("[ERROR] Fail to request speech data(%d)", ret); } } else { // Appropriate MA Client not available, trying to launch new one - mas_launch_client(); + MAS_LOGD("MA Client with wakeup word %s does not exist, launching client", (wakeup_word ? wakeup_word : "NULL")); + mas_launch_client_by_wakeup_word(wakeup_word); } + if (wakeup_word) free(wakeup_word); + MAS_LOGD("END"); return EINA_FALSE; } - static void __wakeup_event_cb(wakeup_service_wakeup_event_e event, const char* wakeup_word, void* user_data) { MAS_LOGD("[SUCCESS] __wakeup_event_cb is called, event(%d), wakeup_word(%s)", event, wakeup_word); @@ -163,9 +169,9 @@ static void __wakeup_event_cb(wakeup_service_wakeup_event_e event, const char* w ecore_thread_main_loop_end(); } #endif /* - TEST_CODE */ - if (WAKEUP_EVENT_SUCCESS == event) { + if (WAKEUP_EVENT_SUCCESS == event && wakeup_word) { ecore_thread_main_loop_begin(); - ecore_timer_add(0.0f, __request_speech_data, NULL); + ecore_timer_add(0.0f, __request_speech_data, (void*)strdup(wakeup_word)); ecore_thread_main_loop_end(); } @@ -174,9 +180,14 @@ static void __wakeup_event_cb(wakeup_service_wakeup_event_e event, const char* w static void __speech_streaming_cb(wakeup_service_speech_streaming_event_e event, unsigned char* buffer, int len, void *user_data) { MAS_LOGD( "[SUCCESS] __speech_streaming_cb is called, event(%d), buffer(%p), len(%d)", event, buffer, len); - int ret = masc_dbus_send_speech_data(event, buffer, len); - if (0 != ret) { - MAS_LOGE("[ERROR] Fail to send speech data, ret(%d)", ret); + int pid = mas_get_current_client_pid(); + if (pid == -1) { + MAS_LOGE("[ERROR] Fail to retrieve pid of current MA client"); + } else { + int ret = masc_dbus_send_speech_data(pid, event, buffer, len); + if (0 != ret) { + MAS_LOGE("[ERROR] Fail to send speech data, ret(%d)", ret); + } } #ifdef BUF_SAVE_MODE -- 2.34.1