extern "C" {
#endif
+typedef enum {
+ MA_PLUGIN_EVENT_VOICE_KEY_PRESSED = 0,
+ MA_PLUGIN_EVENT_VOICE_KEY_RELEASED,
+} ma_plugin_event_e;
int multi_assistant_service_plugin_initialize(void);
int multi_assistant_service_plugin_deactivate(void);
-int multi_assistant_service_plugin_update_voice_feedback_state(const char *appid, int state);
+int multi_assistant_service_plugin_update_voice_feedback_state(const char* appid, int state);
-int multi_assistant_service_plugin_update_result_state(const char *appid, int state);
+int multi_assistant_service_plugin_send_assistant_specific_command(const char *appid, const char* command);
-int multi_assistant_service_plugin_process_event(int event, void *data, int len);
+int multi_assistant_service_plugin_update_result_state(const char* appid, int state);
+
+int multi_assistant_service_plugin_process_event(int event, void* data, int len);
int multi_assistant_service_plugin_start_streaming_utterance_data(void);
int multi_assistant_service_plugin_stop_streaming_follow_up_data(void);
-int multi_assistant_service_plugin_get_recording_audio_format(int *rate, int *channel, int *audio_type);
+int multi_assistant_service_plugin_get_recording_audio_format(int* rate, int* channel, int* audio_type);
int multi_assistant_service_plugin_set_callbacks(void);
#define MA_WAKEUP_MANAGER_FUNC_DEACTIVATE "wakeup_manager_deactivate"
typedef int (*wakeup_manager_deactivate)(void);
#define MA_WAKEUP_MANAGER_FUNC_UPDATE_VOICE_FEEDBACK_STATE "wakeup_manager_update_voice_feedback_state"
-typedef int (*wakeup_manager_update_voice_feedback_state)(const char *appid, int state);
+typedef int (*wakeup_manager_update_voice_feedback_state)(const char* appid, int state);
+#define MA_WAKEUP_MANAGER_FUNC_SEND_ASSISTANT_SPECIFIC_COMMAND "wakeup_manager_send_assistant_specific_command"
+typedef int (*wakeup_manager_send_assistant_specific_command)(const char* appid, const char* command);
#define MA_WAKEUP_MANAGER_FUNC_UPDATE_RESULT_STATE "wakeup_manager_update_result_state"
-typedef int (*wakeup_manager_update_result_state)(const char *appid, int state);
+typedef int (*wakeup_manager_update_result_state)(const char* appid, int state);
#define MA_WAKEUP_MANAGER_FUNC_PROCESS_EVENT "wakeup_manager_process_event"
typedef int (*wakeup_manager_process_event)(int event, void* data, int len);
#define MA_WAKEUP_MANAGER_FUNC_START_STREAMING_UTTERANCE_DATA "wakeup_manager_start_streaming_utterance_data"
wakeup_manager_activate activate;
wakeup_manager_deactivate deactivate;
wakeup_manager_update_voice_feedback_state update_voice_feedback_state;
+ wakeup_manager_send_assistant_specific_command send_assistant_specific_command;
wakeup_manager_update_result_state update_result_state;
wakeup_manager_process_event process_event;
wakeup_manager_start_streaming_utterance_data start_streaming_utterance_data;
#include <thread>
#include <atomic>
+#include <vector>
#include <audio_io.h>
#include <sound_manager.h>
#define MAX_WAKEUP_ENGINE_NUM 10
static int g_engine_count = 0;
+#ifdef TV_PRODUCT
+#define TV_BT_MODE
+#include <bluetooth_product.h>
+#endif
+
typedef struct {
bool active;
bool enabled;
static std::thread g_speech_data_thread;
static std::atomic_bool g_speech_data_thread_should_stop;
+static bool g_voice_key_pressed = false;
+static std::vector<wakeup_engine_speech_data> g_speech_data;
+
+#ifdef TV_BT_MODE
+
+#define EFL_BETA_API_SUPPORT
+
+#include <Ecore_Wl2.h>
+#include <Key_Mode.h>
+#include <Ecore_Input.h>
+
+static int g_bt_extend_count;
+
+#define SMART_CONTROL_EXTEND_CMD 0x03
+#define SMART_CONTROL_START_CMD 0x04
+
+Ecore_Event_Handler* _key_down_handler = NULL;
+Ecore_Event_Handler* _key_up_handler = NULL;
+
+Eina_Bool _key_down_cb(void* data, int type, void* event)
+{
+ Ecore_Event_Key *ev = (Ecore_Event_Key *) event;
+ if (ev) {
+ MWR_LOGD("KEY[%s], typep[%d]", ev->keyname, type);
+
+ if (ev->keyname && strncmp(ev->keyname, KEY_BT_VOICE, strlen(KEY_BT_VOICE)) == 0 ) {
+ wakeup_manager_send_assistant_specific_command(0, "voice_key_pressed");
+ }
+ }
+
+ return ECORE_CALLBACK_DONE;
+}
+
+Eina_Bool _key_up_cb(void* data, int type, void* event)
+{
+ Ecore_Event_Key *ev = (Ecore_Event_Key *) event;
+ if (ev) {
+ MWR_LOGD("KEY[%s], typep[%d]", ev->keyname, type);
+
+ if (ev->keyname && strncmp(ev->keyname, KEY_BT_VOICE, strlen(KEY_BT_VOICE)) == 0) {
+ bt_hid_rc_stop_sending_voice(NULL);
+ wakeup_manager_send_assistant_specific_command(0, "voice_key_released");
+ }
+ }
+ return ECORE_CALLBACK_DONE;
+}
+
+bool _grab_voice_key(void)
+{
+ Eina_Bool bRet = true;
+ bRet = ecore_wl2_window_keygrab_set(NULL, KEY_BT_VOICE, 0, 0, 0, ECORE_WL2_WINDOW_KEYGRAB_SHARED);
+ MWR_LOGD("ecore_wl2_window_keygrab_set ret[%d] [%s]", bRet, KEY_BT_VOICE);
+ return bRet;
+}
+
+bool _ungrab_voice_key(void)
+{
+ Eina_Bool bRet = true;
+ bRet = ecore_wl2_window_keygrab_unset(NULL, KEY_BT_VOICE, 0, 0);
+ MWR_LOGD("ecore_wl2_window_keygrab_unset ret[%d] [%s]", bRet, KEY_BT_VOICE);
+ return bRet;
+}
+
+bool _add_key_cb()
+{
+ if (_key_down_handler == NULL)
+ {
+ MWR_LOGE("_key_down_handler");
+ _key_down_handler = ecore_event_handler_add(ECORE_EVENT_KEY_DOWN, _key_down_cb, NULL);
+ if(_key_down_handler == NULL)
+ {
+ MWR_LOGE("_key_down_handler == NULL ");
+ }
+ }
+
+ if (_key_up_handler == NULL)
+ {
+ MWR_LOGE("_key_down_handler");
+ _key_up_handler = ecore_event_handler_add(ECORE_EVENT_KEY_UP, _key_up_cb, NULL);
+ if(_key_up_handler == NULL)
+ {
+ MWR_LOGE("_key_up_handler == NULL ");
+ }
+ }
+ return true;
+}
+
+bool _delete_key_cb(void)
+{
+ MWR_LOGE("start");
+ if (_key_down_handler != NULL)
+ {
+ ecore_event_handler_del(_key_down_handler);
+ _key_down_handler = NULL;
+ }
+
+ if (_key_up_handler != NULL)
+ {
+ ecore_event_handler_del(_key_up_handler);
+ _key_up_handler = NULL;
+ }
+ MWR_LOGE("end");
+ return true;
+}
+
+static void _bt_cb_hid_state_changed(int result, bool connected, const char *remote_address, void *user_data)
+{
+ MWR_LOGD("[Recorder] Bluetooth Event [%d] Received address [%s]", result, remote_address);
+ return;
+}
+
+static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void *user_data)
+{
+ static int g_buffer_count = 0;
+ if (nullptr == voice_data) return;
+
+ if (g_voice_key_pressed) {
+ wakeup_engine_speech_data data;
+ data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE;
+ data.len = voice_data->length;
+ data.buffer = malloc(voice_data->length);
+ if (data.buffer) {
+ memcpy(data.buffer, voice_data->audio_buf, voice_data->length);
+ g_speech_data.push_back(data);
+ }
+ } else {
+ MWR_LOGE("[Recorder ERROR] voice key seems to be already released");
+ return;
+ }
+
+ if (0 == g_buffer_count || 0 == g_buffer_count % 50) {
+ MWR_LOGD("[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, voice_data->length);
+
+ if (0 == g_bt_extend_count % 5 && 0 != g_buffer_count) {
+ const unsigned char input_data[2] = {SMART_CONTROL_EXTEND_CMD, 0x10 };
+ if (BT_ERROR_NONE != bt_hid_send_rc_command(NULL, input_data, sizeof(input_data))) {
+ MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command");
+ } else {
+ MWR_LOGD("[Recorder] Extend bt audio recorder");
+ }
+ }
+ g_bt_extend_count++;
+
+ if (100000 == g_buffer_count) {
+ g_buffer_count = 0;
+ }
+ }
+
+ g_buffer_count++;
+
+ return;
+}
+#endif
+
static int recorder_initialize()
{
const int rate = 16000;
audio_in_destroy(g_audio_h);
return -1;
}
+
+#ifdef TV_BT_MODE
+ bool is_bt_failed = false;
+
+ if (false == is_bt_failed && BT_ERROR_NONE != bt_product_init()) {
+ MWR_LOGE("[Recorder ERROR] Fail to init bt");
+ is_bt_failed = true;
+ }
+
+ if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, NULL)) {
+ MWR_LOGE("[Recorder ERROR] Fail bt_hid_host_initialize()");
+ is_bt_failed = true;
+ }
+
+ if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, NULL)) {
+ MWR_LOGE("[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()");
+ is_bt_failed = true;
+ }
+
+ if (false == is_bt_failed) {
+ MWR_LOGD("[Recorder] Bluetooth is available");
+ }
+
+ _grab_voice_key();
+ _add_key_cb();
+#endif
}
static int recorder_deinitialize(void)
{
MWR_LOGD("[ENTER]");
+#ifdef TV_BT_MODE
+ _delete_key_cb();
+ _ungrab_voice_key();
+
+ bt_hid_unset_audio_data_receive_cb();
+ bt_hid_host_deinitialize();
+ bt_product_deinit();
+#endif
+
int ret = 0;
ret = audio_in_unprepare(g_audio_h);
if (AUDIO_IO_ERROR_NONE != ret) {
static void start_recorder_thread()
{
join_recorder_thread();
- g_recorder_thread_should_stop.store(false);
- g_recorder_thread = std::thread(recorder_thread_func);
+
+#ifdef TV_BT_MODE
+ /* Do not start normal recorder thread if TV_BT_MODE and g_voice_key_pressed,
+ just send bt_hid start message */
+ if (g_voice_key_pressed)
+ {
+ const unsigned char input_data[2] = {SMART_CONTROL_START_CMD, 0x00};
+ int bt_retry = 0;
+ const int max_retry = 5;
+ while (max_retry > bt_retry) {
+ int ret = bt_hid_send_rc_command(NULL, input_data, sizeof(input_data));
+ if (BT_ERROR_NONE == ret) {
+ MWR_LOGD("[Recorder] Start bt audio recorder");
+ break;
+ } else if (BT_ERROR_NOW_IN_PROGRESS == ret) {
+ MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command : %d", ret);
+ usleep(50000);
+ bt_retry++;
+ } else {
+ break;
+ }
+ }
+ if (max_retry == bt_retry) {
+ MWR_LOGE("[Recorder ERROR] Fail to start bt audio");
+ return;
+ }
+
+ g_bt_extend_count = 0;
+ }
+ else
+#endif
+ {
+ g_recorder_thread_should_stop.store(false);
+ g_recorder_thread = std::thread(recorder_thread_func);
+ }
}
static void wakeup_engine_audio_data_require_status_cb(bool require, void* user_data)
}
if (audio_data_require_count > 0) {
g_audio_data_required = true;
- if (g_audio_data_required != prev_audio_data_required) {
+ if (g_audio_data_required != prev_audio_data_required &&
+ g_voice_key_pressed != true) {
start_recorder_thread();
}
} else {
g_audio_data_required = false;
- if (g_audio_data_required != prev_audio_data_required) {
+ if (g_audio_data_required != prev_audio_data_required &&
+ g_voice_key_pressed != true) {
join_recorder_thread();
}
}
g_wakeup_engine_info[g_engine_count].interface.get_utterance_data =
(wakeup_engine_get_utterance_data)dlsym(handle,
MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA);
+ g_wakeup_engine_info[g_engine_count].interface.set_assistant_specific_command =
+ (wakeup_engine_set_assistant_specific_command)dlsym(handle,
+ MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND);
g_wakeup_engine_info[g_engine_count].interface.set_wakeup_event_callback =
(wakeup_engine_set_wakeup_event_callback)dlsym(handle,
MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK);
recorder_deinitialize();
+ for (const auto &data : g_speech_data) {
+ free(data.buffer);
+ }
+ g_speech_data.clear();
+
if (g_current_language) {
free(g_current_language);
g_current_language = NULL;
{
MWR_LOGD("[ENTER]");
+ if (NULL == appid) {
+ MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)",
+ (appid ? appid : "NULL"));
+ return -1;
+ }
+
if (state) {
if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_LISTENING ||
g_wakeup_manager_state == WAKEUP_MANAGER_STATE_PROCESSING) {
return 0;
}
+int wakeup_manager_send_assistant_specific_command(const char* appid, const char* command)
+{
+ MWR_LOGD("[ENTER]");
+
+ if (NULL == appid || NULL == command) {
+ MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), command(%s)",
+ (appid ? appid : "NULL"), (command ? command : "NULL"));
+ return -1;
+ }
+
+ const char* voice_key_pressed = "voice_key_pressed";
+ const char* voice_key_released = "voice_key_released";
+
+ if (command) {
+ if (0 == strncmp(command, voice_key_pressed, strlen(voice_key_pressed))) {
+ wakeup_manager_process_event(MA_PLUGIN_EVENT_VOICE_KEY_PRESSED, NULL, 0);
+ } else if (0 == strncmp(command, voice_key_released, strlen(voice_key_released))) {
+ wakeup_manager_process_event(MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, NULL, 0);
+ }
+ }
+
+ for (int loop = 0;loop < g_engine_count;loop++) {
+ /* Need to find the appropriate engine for this assistant,
+ for now assuming 0 is the one */
+ if(loop == 0) {
+ if (g_wakeup_engine_info[loop].interface.set_assistant_specific_command) {
+ g_wakeup_engine_info[loop].interface.set_assistant_specific_command(appid, command);
+ }
+ }
+ }
+
+ MWR_LOGD("[END]");
+ return 0;
+}
+
int wakeup_manager_update_result_state(const char* appid, int state)
{
MWR_LOGD("[ENTER]");
+ if (NULL == appid) {
+ MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)",
+ (appid ? appid : "NULL"));
+ return -1;
+ }
if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_PROCESSING) {
wakeup_manager_change_state(WAKEUP_MANAGER_STATE_LISTENING);
}
{
MWR_LOGD("[ENTER]");
+ // LOCK REQUIRED
+ if (event == MA_PLUGIN_EVENT_VOICE_KEY_PRESSED) {
+ if (g_voice_key_pressed != true) {
+ /* Clear all existing data */
+ for (const auto &speech_data : g_speech_data) {
+ if (speech_data.buffer) free(speech_data.buffer);
+ }
+ g_speech_data.clear();
+
+ g_voice_key_pressed = true;
+ /* (Re)Start recorder thread using bt hid */
+ start_recorder_thread();
+ for (int loop = 0;loop < g_engine_count;loop++) {
+ /* Need to find the default assistant, for now assuming 0 is the one */
+ if(loop == 0) {
+ g_wakeup_event_cb(WAKEUP_EVENT_SUCCESS,
+ "hi bixby", g_wakeup_event_user_data);
+ }
+ }
+ }
+ } else if (event == MA_PLUGIN_EVENT_VOICE_KEY_RELEASED) {
+ if (g_voice_key_pressed != false) {
+ unsigned char final_buffer[2] = {'\0', };
+
+ g_voice_key_pressed = false;
+ if (g_audio_data_required == true) {
+ /* Restart recorder thread using standard mic */
+ start_recorder_thread();
+ } else {
+ join_recorder_thread();
+ }
+ wakeup_engine_speech_data speech_data;
+ speech_data.event = WAKEUP_SPEECH_STREAMING_EVENT_FINISH;
+ speech_data.len = sizeof(final_buffer);
+ speech_data.buffer = malloc(speech_data.len);
+ if (speech_data.buffer) {
+ memcpy(speech_data.buffer, final_buffer, speech_data.len);
+ g_speech_data.push_back(speech_data);
+ }
+ }
+ }
+ // UNLOCK REQUIRED
+
MWR_LOGD("[END]");
return 0;
}
}
}
-static void speech_data_thread_func(void)
+static void manager_data_thread_func(void)
+{
+ MWR_LOGD("[ENTER]");
+
+ MWR_LOGD("data_count : %zu", g_speech_data.size());
+
+ int index = 0;
+
+ while (1) {
+ int ret = -1;
+ int cnt = 0;
+
+ /* get feedback data */
+ if (index >= g_speech_data.size()) {
+ /* empty queue */
+ MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
+
+ /* waiting */
+ while (1) {
+ usleep(10000);
+ if (index < g_speech_data.size()) {
+ MWR_LOGI("[INFO] Resume thread");
+ break;
+ }
+ if (200 < cnt) {
+ MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
+ __wakeup_service_streaming_cb(
+ WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0);
+ return;
+ }
+ cnt++;
+ }
+ MWR_LOGI("[INFO] Finish to wait for new feedback data come");
+
+ /* resume feedback thread */
+ continue;
+ }
+
+ wakeup_engine_speech_data &speech_data = g_speech_data.at(index);
+ __wakeup_service_streaming_cb(
+ speech_data.event, speech_data.buffer, speech_data.len);
+
+ if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
+ MWR_LOGI("[INFO] Finish to get and send speech data");
+ break;
+ }
+
+ index++;
+ }
+}
+
+static void engine_data_thread_func(void)
{
MWR_LOGD("[ENTER]");
g_speech_data_thread.join();
}
g_speech_data_thread_should_stop.store(false);
- g_speech_data_thread = std::thread(speech_data_thread_func);
+ if (g_voice_key_pressed) {
+ g_speech_data_thread = std::thread(manager_data_thread_func);
+ } else {
+ g_speech_data_thread = std::thread(engine_data_thread_func);
+ }
MWR_LOGD("[END]");
return 0;
projects / platform / core / uifw / multi-assistant-service.git / commitdiff