plugins/wakeup-manager/src/wakeup_manager.cpp

   1 /*
   2  * Copyright 2018  Samsung Electronics Co., Ltd
   3  *
   4  * Licensed under the Flora License, Version 1.1 (the License);
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://floralicense.org/license/
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an AS IS BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 #include <Ecore.h>
  17 #include <dirent.h>
  18 #include <dlfcn.h>
  19
  20 #include <sstream>
  21 #include <thread>
  22 #include <atomic>
  23 #include <vector>
  24 #include <memory>
  25 #include <algorithm>
  26
  27 #include <audio_io.h>
  28 #include <sound_manager.h>
  29 #include <vconf.h>
  30
  31 #include "wakeup_manager_main.h"
  32 #include "wakeup_manager.h"
  33 #include "wakeup_manager_audio.h"
  34 #include "wakeup_policy_default.h"
  35
  36 static wakeup_service_wakeup_event_cb g_wakeup_event_cb;
  37 static void* g_wakeup_event_user_data;
  38
  39 static wakeup_service_speech_streaming_cb g_utterance_streaming_cb;
  40 static void* g_utterance_streaming_user_data;
  41
  42 static wakeup_service_speech_streaming_cb g_previous_utterance_streaming_cb;
  43 static void* g_previous_utterance_streaming_user_data;
  44
  45 static wakeup_service_speech_streaming_cb g_follow_up_streaming_cb;
  46 static void* g_follow_up_streaming_user_data;
  47
  48 static wakeup_service_speech_status_cb g_speech_status_cb;
  49 static void* g_speech_status_user_data;
  50
  51 static wakeup_service_error_cb g_error_cb;
  52 static void* g_error_user_data;
  53
  54 #define MAX_WAKEUP_ENGINE_NUM 10
  55 static int g_engine_count = 0;
  56
  57 #ifdef TV_PRODUCT
  58 #define TV_BT_MODE
  59 #include <bluetooth_product.h>
  60 #endif
  61
  62 typedef struct {
  63         bool active{false};
  64         bool enabled{false};
  65         bool audio_data_require_status{false};
  66         char engine_name[_POSIX_PATH_MAX];
  67         char engine_path[_POSIX_PATH_MAX];
  68         wakeup_engine_interface interface{nullptr, };
  69         std::vector<std::string> assistant_list;
  70 } wakeup_engine_info;
  71
  72 //static int g_wakeup_engine_selected = -1;
  73 static int g_wakeup_engine_selected = 0;
  74 static wakeup_engine_info g_wakeup_engine_info[MAX_WAKEUP_ENGINE_NUM];
  75
  76 static char* g_current_language = NULL;
  77 static wakeup_manager_state_e g_wakeup_manager_state;
  78
  79 static bool g_audio_data_required = false;
  80 static audio_in_h g_audio_h = NULL;
  81 static sound_stream_info_h g_stream_info_h = NULL;
  82
  83 static std::thread g_recorder_thread;
  84 static std::atomic_bool g_recorder_thread_should_stop;
  85
  86 static std::thread g_speech_data_thread;
  87 static std::atomic_bool g_speech_data_thread_should_stop;
  88
  89 static bool g_voice_key_pressed = false;
  90 static std::vector<wakeup_engine_speech_data> g_speech_data;
  91
  92 #define DEFAULT_ASSISTANT_APPID "com.samsung.bixby-voice"
  93
  94 #define WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID "db/multi-assistant/default_assistant_appid"
  95 #define WAKEUP_SETTINGS_KEY_UI_PANEL_ENABLED "db/multi-assistant/ui_panel_enabled"
  96 #define WAKEUP_SETTINGS_KEY_CONVERSATION_TIMEOUT "db/multi-assistant/conversation_timeout"
  97 #define WAKEUP_SETTINGS_KEY_MULTIPLE_MODE "db/multi-assistant/multiple_mode"
  98 #define WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS "db/multi-assistant/enabled_assistants"
  99 #define WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_DELAY "db/multi-assistant/wakeup_policy_delay"
 100 #define WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_PRIORITY "db/multi-assistant/wakeup_policy_priority"
 101
 102 typedef struct {
 103         std::string default_assistant_appid{DEFAULT_ASSISTANT_APPID};
 104         bool ui_panel_enabled{true};
 105         float conversation_timeout{5.0};
 106         bool multiple_mode{true};
 107         std::vector<std::string> enabled_assistants{DEFAULT_ASSISTANT_APPID};
 108         float wakeup_policy_delay{0.1};
 109         std::vector<std::string> wakeup_policy_priority; // No priority by default
 110 } wakeup_settings;
 111
 112 static wakeup_settings g_wakeup_settings;
 113
 114 class CWakeupEventObserver : public IWakeupEventObserver
 115 {
 116 public:
 117         void on_wakeup(wakeup_event_info info) override;
 118 };
 119 static std::unique_ptr<CWakeupPolicy> g_wakeup_policy;
 120 static CWakeupEventObserver g_wakeup_event_observer;
 121
 122 #ifdef TV_BT_MODE
 123
 124 #define EFL_BETA_API_SUPPORT
 125
 126 #include <Ecore_Wl2.h>
 127 #include <Key_Mode.h>
 128 #include <Ecore_Input.h>
 129
 130 static int g_bt_extend_count;
 131
 132 #define SMART_CONTROL_EXTEND_CMD    0x03
 133 #define SMART_CONTROL_START_CMD     0x04
 134
 135 Ecore_Event_Handler* _key_down_handler = NULL;
 136 Ecore_Event_Handler* _key_up_handler = NULL;
 137
 138 Eina_Bool _key_down_cb(void* data, int type, void* event)
 139 {
 140         Ecore_Event_Key *ev = (Ecore_Event_Key *) event;
 141         if (ev) {
 142                 MWR_LOGD("KEY[%s], typep[%d]", ev->keyname, type);
 143
 144                 if (ev->keyname && strncmp(ev->keyname, KEY_BT_VOICE, strlen(KEY_BT_VOICE)) == 0 ) {
 145                         wakeup_manager_send_assistant_specific_command(0, "voice_key_pressed");
 146                 }
 147         }
 148
 149         return ECORE_CALLBACK_DONE;
 150 }
 151
 152 Eina_Bool _key_up_cb(void* data, int type, void* event)
 153 {
 154         Ecore_Event_Key *ev = (Ecore_Event_Key *) event;
 155         if (ev) {
 156                 MWR_LOGD("KEY[%s], typep[%d]", ev->keyname, type);
 157
 158                 if (ev->keyname && strncmp(ev->keyname, KEY_BT_VOICE, strlen(KEY_BT_VOICE)) == 0) {
 159                         bt_hid_rc_stop_sending_voice(NULL);
 160                         wakeup_manager_send_assistant_specific_command(0, "voice_key_released");
 161                 }
 162         }
 163         return ECORE_CALLBACK_DONE;
 164 }
 165
 166 bool _grab_voice_key(void)
 167 {
 168         Eina_Bool bRet = true;
 169         bRet = ecore_wl2_window_keygrab_set(NULL,  KEY_BT_VOICE,  0, 0,  0, ECORE_WL2_WINDOW_KEYGRAB_SHARED);
 170         MWR_LOGD("ecore_wl2_window_keygrab_set ret[%d] [%s]", bRet, KEY_BT_VOICE);
 171         return bRet;
 172 }
 173
 174 bool _ungrab_voice_key(void)
 175 {
 176         Eina_Bool bRet = true;
 177         bRet = ecore_wl2_window_keygrab_unset(NULL,  KEY_BT_VOICE,  0, 0);
 178         MWR_LOGD("ecore_wl2_window_keygrab_unset ret[%d] [%s]", bRet, KEY_BT_VOICE);
 179         return bRet;
 180 }
 181
 182 bool _add_key_cb()
 183 {
 184         if (_key_down_handler == NULL)
 185         {
 186                 MWR_LOGE("_key_down_handler");
 187                 _key_down_handler = ecore_event_handler_add(ECORE_EVENT_KEY_DOWN, _key_down_cb, NULL);
 188                 if(_key_down_handler == NULL)
 189                 {
 190                         MWR_LOGE("_key_down_handler == NULL ");
 191                 }
 192         }
 193
 194         if (_key_up_handler == NULL)
 195         {
 196                 MWR_LOGE("_key_down_handler");
 197                 _key_up_handler = ecore_event_handler_add(ECORE_EVENT_KEY_UP, _key_up_cb, NULL);
 198                 if(_key_up_handler == NULL)
 199                 {
 200                         MWR_LOGE("_key_up_handler == NULL ");
 201                 }
 202         }
 203         return true;
 204 }
 205
 206 bool _delete_key_cb(void)
 207 {
 208         MWR_LOGE("start");
 209         if (_key_down_handler != NULL)
 210         {
 211                 ecore_event_handler_del(_key_down_handler);
 212                 _key_down_handler = NULL;
 213         }
 214
 215         if (_key_up_handler != NULL)
 216         {
 217                 ecore_event_handler_del(_key_up_handler);
 218                 _key_up_handler = NULL;
 219         }
 220         MWR_LOGE("end");
 221         return true;
 222 }
 223
 224 static void _bt_cb_hid_state_changed(int result, bool connected, const char *remote_address, void *user_data)
 225 {
 226         MWR_LOGD("[Recorder] Bluetooth Event [%d] Received address [%s]", result, remote_address);
 227         return;
 228 }
 229
 230 static void _bt_hid_audio_data_receive_cb(bt_hid_voice_data_s *voice_data, void *user_data)
 231 {
 232         static int g_buffer_count = 0;
 233         if (nullptr == voice_data) return;
 234
 235         if (g_voice_key_pressed) {
 236                 wakeup_engine_speech_data data;
 237                 data.event = WAKEUP_SPEECH_STREAMING_EVENT_CONTINUE;
 238                 data.len = voice_data->length;
 239                 data.buffer = malloc(voice_data->length);
 240                 if (data.buffer) {
 241                         memcpy(data.buffer, voice_data->audio_buf, voice_data->length);
 242                         g_speech_data.push_back(data);
 243                 }
 244         } else {
 245                 MWR_LOGE("[Recorder ERROR] voice key seems to be already released");
 246                 return;
 247         }
 248
 249         if (0 == g_buffer_count || 0 == g_buffer_count % 50) {
 250                 MWR_LOGD("[Recorder][%d] Recording... : read_size(%d)", g_buffer_count, voice_data->length);
 251
 252                 if (0 == g_bt_extend_count % 5 && 0 != g_buffer_count) {
 253                         const unsigned char input_data[2] = {SMART_CONTROL_EXTEND_CMD, 0x10 };
 254                         if (BT_ERROR_NONE != bt_hid_send_rc_command(NULL, input_data, sizeof(input_data))) {
 255                                 MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command");
 256                         } else {
 257                                 MWR_LOGD("[Recorder] Extend bt audio recorder");
 258                         }
 259                 }
 260                 g_bt_extend_count++;
 261
 262                 if (100000 == g_buffer_count) {
 263                         g_buffer_count = 0;
 264                 }
 265         }
 266
 267         g_buffer_count++;
 268
 269         return;
 270 }
 271 #endif
 272
 273 static int recorder_initialize(void)
 274 {
 275         const int rate = 16000;
 276         const audio_channel_e channel = AUDIO_CHANNEL_MONO;
 277         const audio_sample_type_e type = AUDIO_SAMPLE_TYPE_S16_LE;
 278
 279         int ret = audio_in_create(rate, channel, type, &g_audio_h);
 280         if (AUDIO_IO_ERROR_NONE != ret) {
 281                 MWR_LOGD("[Recorder ERROR] Rate(%d) Channel(%d) Type(%d)", rate, channel, type);
 282                 MWR_LOGD("[Recorder ERROR] Fail to create audio handle : %d", ret);
 283                 return -1;
 284         }
 285
 286         if (0 != sound_manager_create_stream_information(
 287                 SOUND_STREAM_TYPE_VOICE_RECOGNITION, NULL, NULL, &g_stream_info_h)) {
 288                 MWR_LOGD("[Recorder ERROR] Fail to create stream info");
 289                 audio_in_destroy(g_audio_h);
 290                 return -1;
 291         }
 292
 293         ret = audio_in_set_sound_stream_info(g_audio_h, g_stream_info_h);
 294         if (AUDIO_IO_ERROR_NONE != ret) {
 295                 MWR_LOGD("[Recorder ERROR] Fail to set stream info : %d", ret);
 296                 sound_manager_destroy_stream_information(g_stream_info_h);
 297                 audio_in_destroy(g_audio_h);
 298                 return -1;
 299         }
 300
 301         ret = audio_in_prepare(g_audio_h);
 302         if (AUDIO_IO_ERROR_NONE != ret) {
 303                 if (AUDIO_IO_ERROR_SOUND_POLICY == ret)
 304                 {
 305                         MWR_LOGD("[Recorder ERROR] Audio is busy.");
 306                 } else {
 307                         MWR_LOGD("[Recorder ERROR] Fail to start audio : %d", ret);
 308                 }
 309                 sound_manager_destroy_stream_information(g_stream_info_h);
 310                 audio_in_destroy(g_audio_h);
 311                 return -1;
 312         }
 313
 314 #ifdef TV_BT_MODE
 315         bool is_bt_failed = false;
 316
 317         if (false == is_bt_failed && BT_ERROR_NONE != bt_product_init()) {
 318                 MWR_LOGE("[Recorder ERROR] Fail to init bt");
 319                 is_bt_failed = true;
 320         }
 321
 322         if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_host_initialize(_bt_cb_hid_state_changed, NULL)) {
 323                 MWR_LOGE("[Recorder ERROR] Fail bt_hid_host_initialize()");
 324                 is_bt_failed = true;
 325         }
 326
 327         if (false == is_bt_failed && BT_ERROR_NONE != bt_hid_set_audio_data_receive_cb(_bt_hid_audio_data_receive_cb, NULL)) {
 328                 MWR_LOGE("[Recorder ERROR] Fail bt_hid_set_audio_data_receive_cb()");
 329                 is_bt_failed = true;
 330         }
 331
 332         if (false == is_bt_failed) {
 333                 MWR_LOGD("[Recorder] Bluetooth is available");
 334         }
 335
 336         _grab_voice_key();
 337         _add_key_cb();
 338 #endif
 339 }
 340
 341 static int recorder_deinitialize(void)
 342 {
 343         MWR_LOGD("[ENTER]");
 344
 345 #ifdef TV_BT_MODE
 346         _delete_key_cb();
 347         _ungrab_voice_key();
 348
 349         bt_hid_unset_audio_data_receive_cb();
 350         bt_hid_host_deinitialize();
 351         bt_product_deinit();
 352 #endif
 353
 354         int ret = 0;
 355         ret = audio_in_unprepare(g_audio_h);
 356         if (AUDIO_IO_ERROR_NONE != ret) {
 357                 MWR_LOGD("[Recorder ERROR] Fail to stop audio : %d", ret);
 358         }
 359
 360         if (0 != sound_manager_destroy_stream_information(g_stream_info_h)) {
 361                 MWR_LOGD("[Recorder ERROR] Fail to destroy stream info");
 362         }
 363
 364         ret = audio_in_destroy(g_audio_h);
 365         if (AUDIO_IO_ERROR_NONE != ret) {
 366                 MWR_LOGD("[Recorder ERROR] Fail to destroy audio : %d", ret);
 367         }
 368
 369         MWR_LOGD("[END]");
 370         return 0;
 371 }
 372
 373 #define FRAME_LENGTH 160
 374 #define BUFFER_LENGTH FRAME_LENGTH * 2
 375
 376 static void recorder_thread_func()
 377 {
 378         static int buffer_count = 0;
 379
 380         while (!(g_recorder_thread_should_stop.load())) {
 381                 unsigned char buffer[BUFFER_LENGTH];
 382                 int ret;
 383                 memset(buffer, '\0', BUFFER_LENGTH);
 384
 385                 auto now = std::chrono::system_clock::now();
 386                 auto now_ms = std::chrono::time_point_cast<std::chrono::milliseconds>(now);
 387                 /* number of milliseconds since the epoch of system_clock */
 388                 auto value = now_ms.time_since_epoch();
 389
 390                 static long time = 0;
 391                 if (time == value.count()) {
 392                         LOGE("[Recorder WARNING] Time value duplicated : %lu", time);
 393                 }
 394                 time = value.count();
 395
 396                 int read_bytes = audio_in_read(g_audio_h, buffer, BUFFER_LENGTH);
 397                 if (0 > read_bytes) {
 398                         LOGE("[Recorder WARNING] Fail to read audio : %d", read_bytes);
 399                         break;
 400                 }
 401                 // LOCK REQUIRED
 402                 if (!g_audio_data_required) return;
 403                 for (int loop = 0;loop < g_engine_count;loop++) {
 404                         if (g_wakeup_engine_info[loop].audio_data_require_status &&
 405                                 g_wakeup_engine_info[loop].interface.feed_audio_data) {
 406                                 ret = g_wakeup_engine_info[loop].interface.feed_audio_data(time, buffer, read_bytes);
 407                                 if (0 == ret) {
 408                                         LOGE("[ERROR] Fail to feed speech data, ret(%d)", ret);
 409                                 }
 410                         }
 411                 }
 412                 // UNLOCK REQUIRED
 413                 /* Audio read log */
 414                 if (0 == buffer_count % 100) {
 415                         LOGD("[Recorder][%d] Recording... : read_size(%d)", buffer_count, read_bytes);
 416                 }
 417
 418                 buffer_count++;
 419
 420 #ifdef BUF_SAVE_MODE
 421                 /* write pcm buffer */
 422                 if (g_pFile)
 423                         fwrite(buffer, 1, BUFFER_LENGTH, g_pFile);
 424 #endif
 425         }
 426 }
 427
 428 static void wakeup_engine_wakeup_event_cb(wakeup_event_info info, void* user_data)
 429 {
 430         MWR_LOGD("[ENTER]");
 431         if (NULL == user_data) return;
 432
 433         for (int loop = 0;loop < g_engine_count;loop++) {
 434                 if (NULL != g_wakeup_engine_info[loop].engine_name &&
 435                         strncmp(g_wakeup_engine_info[loop].engine_name,
 436                         (const char*)user_data, _POSIX_PATH_MAX) == 0) {
 437                                 if (g_wakeup_policy) {
 438                                         g_wakeup_policy->wakeup_candidate(info);
 439                                 }
 440                                 //g_wakeup_event_cb(event, g_wakeup_event_user_data);
 441                 }
 442         }
 443 }
 444
 445 static void wakeup_engine_speech_status_cb(wakeup_service_speech_status_e status, void* user_data)
 446 {
 447         MWR_LOGD("[ENTER]");
 448         if (NULL == user_data) return;
 449
 450         for (int loop = 0;loop < g_engine_count;loop++) {
 451                 if (NULL != g_wakeup_engine_info[loop].engine_name &&
 452                         strncmp(g_wakeup_engine_info[loop].engine_name,
 453                         (const char*)user_data, _POSIX_PATH_MAX) == 0) {
 454                 }
 455         }
 456 }
 457
 458 static void wakeup_engine_error_cb(int error, const char* err_msg, void* user_data)
 459 {
 460         MWR_LOGD("[ENTER]");
 461         if (NULL == user_data) return;
 462
 463         for (int loop = 0;loop < g_engine_count;loop++) {
 464                 if (NULL != g_wakeup_engine_info[loop].engine_name &&
 465                         strncmp(g_wakeup_engine_info[loop].engine_name,
 466                         (const char*)user_data, _POSIX_PATH_MAX) == 0) {
 467                 }
 468         }
 469 }
 470
 471 static void join_recorder_thread()
 472 {
 473         if (g_recorder_thread.joinable()) {
 474                 MWR_LOGD("g_recorder_thread is joinable, trying join()");
 475                 g_recorder_thread_should_stop.store(true);
 476                 g_recorder_thread.join();
 477         }
 478 }
 479
 480 static void start_recorder_thread()
 481 {
 482         join_recorder_thread();
 483
 484 #ifdef TV_BT_MODE
 485         /* Do not start normal recorder thread if TV_BT_MODE and g_voice_key_pressed,
 486                 just send bt_hid start message */
 487         if (g_voice_key_pressed)
 488         {
 489                 const unsigned char input_data[2] = {SMART_CONTROL_START_CMD, 0x00};
 490                 int bt_retry = 0;
 491                 const int max_retry = 5;
 492                 while (max_retry > bt_retry) {
 493                         int ret = bt_hid_send_rc_command(NULL, input_data, sizeof(input_data));
 494                         if (BT_ERROR_NONE == ret) {
 495                                 MWR_LOGD("[Recorder] Start bt audio recorder");
 496                                 break;
 497                         } else if (BT_ERROR_NOW_IN_PROGRESS == ret) {
 498                                 MWR_LOGE("[Recorder ERROR] Fail bt_hid_send_rc_command : %d", ret);
 499                                 usleep(50000);
 500                                 bt_retry++;
 501                         } else {
 502                                 break;
 503                         }
 504                 }
 505                 if (max_retry == bt_retry) {
 506                         MWR_LOGE("[Recorder ERROR] Fail to start bt audio");
 507                         return;
 508                 }
 509
 510                 g_bt_extend_count = 0;
 511         }
 512         else
 513 #endif
 514         {
 515                 g_recorder_thread_should_stop.store(false);
 516                 g_recorder_thread = std::thread(recorder_thread_func);
 517         }
 518 }
 519
 520 static void wakeup_engine_audio_data_require_status_cb(bool require, void* user_data)
 521 {
 522         MWR_LOGD("[ENTER]");
 523         if (NULL == user_data) return;
 524
 525         bool prev_audio_data_required = g_audio_data_required;
 526         // LOCK REQUIRED
 527         int audio_data_require_count = 0;
 528         for (int loop = 0;loop < g_engine_count;loop++) {
 529                 if (NULL != g_wakeup_engine_info[loop].engine_name &&
 530                         strncmp(g_wakeup_engine_info[loop].engine_name,
 531                         (const char*)user_data, _POSIX_PATH_MAX) == 0) {
 532                         g_wakeup_engine_info[loop].audio_data_require_status = require;
 533                 }
 534                 if (g_wakeup_engine_info[loop].enabled &&
 535                         g_wakeup_engine_info[loop].audio_data_require_status) {
 536                         audio_data_require_count++;
 537                 }
 538         }
 539         if (audio_data_require_count > 0) {
 540                 g_audio_data_required = true;
 541                 if (g_audio_data_required != prev_audio_data_required &&
 542                         g_voice_key_pressed != true) {
 543                         start_recorder_thread();
 544                 }
 545         } else {
 546                 g_audio_data_required = false;
 547                 if (g_audio_data_required != prev_audio_data_required &&
 548                         g_voice_key_pressed != true) {
 549                         join_recorder_thread();
 550                 }
 551         }
 552         // UNLOCK REQUIRED
 553 }
 554
 555 static void wakeup_engine_add_directory(const char* name, const char* path)
 556 {
 557         if (NULL == path) return;
 558
 559         DIR* dp = opendir(path);
 560         if (NULL == dp) {
 561                 MWR_LOGD("Failed opening directory : %s", path);
 562         } else {
 563                 struct dirent *dirp = NULL;
 564                 char filepath[_POSIX_PATH_MAX];
 565                 do {
 566                         dirp = readdir(dp);
 567
 568                         if (NULL != dirp) {
 569                                 if (g_engine_count >= MAX_WAKEUP_ENGINE_NUM) break;
 570
 571                                 if (!strcmp(".", dirp->d_name) || !strcmp("..", dirp->d_name))
 572                                         continue;
 573
 574                                 if (DT_REG != dirp->d_type) /* If not a regular file */
 575                                         continue;
 576
 577                                 int filepath_len = strlen(MA_WAKEUP_ENGINE_PATH) + strlen(dirp->d_name) + 2;
 578                                 if (filepath_len >= _POSIX_PATH_MAX) {
 579                                         MWR_LOGD("File path is too long : %s", dirp->d_name);
 580                                         closedir(dp);
 581                                         return;
 582                                 }
 583
 584                                 memset(filepath, '\0', _POSIX_PATH_MAX);
 585                                 snprintf(filepath, _POSIX_PATH_MAX, "%s/%s", path, dirp->d_name);
 586
 587                                 MWR_LOGD("Name (%s), Filepath(%s)", name, filepath);
 588                                 strncpy(g_wakeup_engine_info[g_engine_count].engine_path, filepath, _POSIX_PATH_MAX);
 589                                 strncpy(g_wakeup_engine_info[g_engine_count].engine_name, name, _POSIX_PATH_MAX);
 590
 591                                 char* error = NULL;
 592                                 void* handle = dlopen(filepath, RTLD_LAZY);
 593                                 if (NULL != (error = dlerror()) || NULL == handle) {
 594                                         MWR_LOGD("[ERROR] Fail to dlopen(%s), error(%s)", filepath, error);
 595                                 } else {
 596                                         g_wakeup_engine_info[g_engine_count].interface.initialize =
 597                                                 (wakeup_engine_initialize)dlsym(handle,
 598                                                 MA_WAKEUP_ENGINE_FUNC_INITIALIZE);
 599                                         g_wakeup_engine_info[g_engine_count].interface.deinitialize =
 600                                                 (wakeup_engine_deinitialize)dlsym(handle,
 601                                                 MA_WAKEUP_ENGINE_FUNC_DEINITIALIZE);
 602                                         g_wakeup_engine_info[g_engine_count].interface.activate =
 603                                                 (wakeup_engine_activate)dlsym(handle,
 604                                                 MA_WAKEUP_ENGINE_FUNC_ACTIVATE);
 605                                         g_wakeup_engine_info[g_engine_count].interface.deactivate =
 606                                                 (wakeup_engine_deactivate)dlsym(handle,
 607                                                 MA_WAKEUP_ENGINE_FUNC_DEACTIVATE);
 608                                         g_wakeup_engine_info[g_engine_count].interface.add_wakeup_word =
 609                                                 (wakeup_engine_add_wakeup_word)dlsym(handle,
 610                                                 MA_WAKEUP_ENGINE_FUNC_ADD_WAKEUP_WORD);
 611                                         g_wakeup_engine_info[g_engine_count].interface.add_language =
 612                                                 (wakeup_engine_add_language)dlsym(handle,
 613                                                 MA_WAKEUP_ENGINE_FUNC_ADD_LANGUAGE);
 614                                         g_wakeup_engine_info[g_engine_count].interface.set_language =
 615                                                 (wakeup_engine_set_language)dlsym(handle,
 616                                                 MA_WAKEUP_ENGINE_FUNC_SET_LANGUAGE);
 617                                         g_wakeup_engine_info[g_engine_count].interface.update_manager_state =
 618                                                 (wakeup_engine_update_manager_state)dlsym(handle,
 619                                                 MA_WAKEUP_ENGINE_FUNC_UPDATE_MANAGER_STATE);
 620                                         g_wakeup_engine_info[g_engine_count].interface.set_audio_format =
 621                                                 (wakeup_engine_set_audio_format)dlsym(handle,
 622                                                 MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_FORMAT);
 623                                         g_wakeup_engine_info[g_engine_count].interface.get_audio_format =
 624                                                 (wakeup_engine_get_audio_format)dlsym(handle,
 625                                                 MA_WAKEUP_ENGINE_FUNC_GET_AUDIO_FORMAT);
 626                                         g_wakeup_engine_info[g_engine_count].interface.feed_audio_data =
 627                                                 (wakeup_engine_feed_audio_data)dlsym(handle,
 628                                                 MA_WAKEUP_ENGINE_FUNC_FEED_AUDIO_DATA);
 629                                         g_wakeup_engine_info[g_engine_count].interface.get_utterance_data_count =
 630                                                 (wakeup_engine_get_utterance_data_count)dlsym(handle,
 631                                                 MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA_COUNT);
 632                                         g_wakeup_engine_info[g_engine_count].interface.get_utterance_data =
 633                                                 (wakeup_engine_get_utterance_data)dlsym(handle,
 634                                                 MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA);
 635                                         g_wakeup_engine_info[g_engine_count].interface.set_assistant_specific_command =
 636                                                 (wakeup_engine_set_assistant_specific_command)dlsym(handle,
 637                                                 MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND);
 638                                         g_wakeup_engine_info[g_engine_count].interface.set_wakeup_event_callback =
 639                                                 (wakeup_engine_set_wakeup_event_callback)dlsym(handle,
 640                                                 MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK);
 641                                         g_wakeup_engine_info[g_engine_count].interface.set_speech_status_callback =
 642                                                 (wakeup_engine_set_speech_status_callback)dlsym(handle,
 643                                                 MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK);
 644                                         g_wakeup_engine_info[g_engine_count].interface.set_error_callback =
 645                                                 (wakeup_engine_set_error_callback)dlsym(handle,
 646                                                 MA_WAKEUP_ENGINE_FUNC_SET_ERROR_CALLBACK);
 647                                         g_wakeup_engine_info[g_engine_count].interface.set_audio_data_require_status_callback =
 648                                                 (wakeup_engine_set_audio_data_require_status_callback)dlsym(handle,
 649                                                 MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_DATA_REQUIRE_STATUS_CALLBACK);
 650                                 }
 651                                 ++g_engine_count;
 652                         }
 653                 } while (NULL != dirp && g_engine_count < MAX_WAKEUP_ENGINE_NUM);
 654
 655                 closedir(dp);
 656         }
 657 }
 658
 659 static int wakeup_engine_info_initialize()
 660 {
 661         DIR* dp = opendir(MA_WAKEUP_ENGINE_PATH);
 662         if (NULL == dp) {
 663                 MWR_LOGD("Failed opening directory : %s", (const char*)MA_WAKEUP_ENGINE_PATH);
 664         } else {
 665                 struct dirent *dirp = NULL;
 666                 char dirpath[_POSIX_PATH_MAX];
 667                 do {
 668                         dirp = readdir(dp);
 669
 670                         if (NULL != dirp) {
 671                                 if (!strcmp(".", dirp->d_name) || !strcmp("..", dirp->d_name))
 672                                         continue;
 673
 674                                 if (DT_DIR != dirp->d_type) /* If not a directory */
 675                                         continue;
 676
 677                                 int dirpath_len = strlen(MA_WAKEUP_ENGINE_PATH) + strlen(dirp->d_name) + 1;
 678                                 if (dirpath_len >= _POSIX_PATH_MAX) {
 679                                         MWR_LOGD("File path is too long : %s", dirp->d_name);
 680                                         closedir(dp);
 681                                         return -1;
 682                                 }
 683
 684                                 memset(dirpath, '\0', _POSIX_PATH_MAX);
 685                                 snprintf(dirpath, _POSIX_PATH_MAX, "%s/%s",
 686                                         (const char*)(MA_WAKEUP_ENGINE_PATH), dirp->d_name);
 687
 688                                 wakeup_engine_add_directory(dirp->d_name, dirpath);
 689                         }
 690                 } while (NULL != dirp && g_engine_count < MAX_WAKEUP_ENGINE_NUM);
 691
 692                 closedir(dp);
 693         }
 694
 695         for (int loop = 0;loop < g_engine_count;loop++) {
 696                 g_wakeup_engine_info[loop].active = false;
 697                 /* We'll need to check vconf for enabled wakeup engines */
 698                 g_wakeup_engine_info[loop].enabled = true;
 699                 g_wakeup_engine_info[loop].audio_data_require_status = false;
 700                 MWR_LOGD("Initializing wakeup engine : %s %p", g_wakeup_engine_info[loop].engine_path, g_wakeup_engine_info[loop].interface.initialize);
 701                 if (g_wakeup_engine_info[loop].interface.initialize) {
 702                         g_wakeup_engine_info[loop].interface.initialize();
 703                 }
 704                 if (g_wakeup_engine_info[loop].interface.set_wakeup_event_callback) {
 705                         g_wakeup_engine_info[loop].interface.set_wakeup_event_callback(
 706                                 wakeup_engine_wakeup_event_cb, g_wakeup_engine_info[loop].engine_name);
 707                 }
 708                 if (g_wakeup_engine_info[loop].interface.set_speech_status_callback) {
 709                         g_wakeup_engine_info[loop].interface.set_speech_status_callback(
 710                                 wakeup_engine_speech_status_cb, g_wakeup_engine_info[loop].engine_name);
 711                 }
 712                 if (g_wakeup_engine_info[loop].interface.set_error_callback) {
 713                         g_wakeup_engine_info[loop].interface.set_error_callback(
 714                                 wakeup_engine_error_cb, g_wakeup_engine_info[loop].engine_name);
 715                 }
 716                 if (g_wakeup_engine_info[loop].interface.set_audio_data_require_status_callback) {
 717                         g_wakeup_engine_info[loop].interface.set_audio_data_require_status_callback(
 718                                 wakeup_engine_audio_data_require_status_cb, g_wakeup_engine_info[loop].engine_name);
 719                 }
 720         }
 721         return 0;
 722 }
 723
 724 void CWakeupEventObserver::on_wakeup(wakeup_event_info info)
 725 {
 726         g_wakeup_event_cb(info, g_wakeup_event_user_data);
 727 }
 728
 729 void wakeup_policy_initialize(void)
 730 {
 731         g_wakeup_policy.reset(new CWakeupPolicyDefault);
 732         if (g_wakeup_policy) {
 733                 g_wakeup_policy->subscribe(&g_wakeup_event_observer);
 734         }
 735
 736         /* Default Policy specific initialization */
 737         CWakeupPolicyDefault *default_policy = dynamic_cast<CWakeupPolicyDefault*>(g_wakeup_policy.get());
 738         if (default_policy) {
 739                 int priority = 0;
 740
 741                 default_policy->set_delay(g_wakeup_settings.wakeup_policy_delay);
 742                 MWR_LOGD("Setting Delay : %f", g_wakeup_settings.wakeup_policy_delay);
 743                 for (const auto& assistant : g_wakeup_settings.wakeup_policy_priority) {
 744                         default_policy->set_assistant_priority(assistant, ++priority);
 745                         MWR_LOGD("Setting Priority : %d %s", priority, assistant.c_str());
 746                 }
 747         }
 748 }
 749
 750 int wakeup_manager_initialize(void)
 751 {
 752         MWR_LOGD("[ENTER]");
 753
 754         g_wakeup_event_cb = NULL;
 755         g_wakeup_event_user_data = NULL;
 756
 757         g_utterance_streaming_cb = NULL;
 758         g_utterance_streaming_user_data = NULL;
 759
 760         g_follow_up_streaming_cb = NULL;
 761         g_follow_up_streaming_user_data = NULL;
 762
 763         g_speech_status_cb = NULL;
 764         g_speech_status_user_data = NULL;
 765
 766         g_error_cb = NULL;
 767         g_error_user_data = NULL;
 768
 769         g_engine_count = 0;
 770
 771         int vconf_ret;
 772         char *vconf_str;
 773         int vconf_bool;
 774         double vconf_double;
 775
 776         vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_DEFAULT_ASSISTANT_APPID);
 777         if (vconf_str) {
 778                 g_wakeup_settings.default_assistant_appid = vconf_str;
 779                 MWR_LOGD("default_assistant_appid : %s", g_wakeup_settings.default_assistant_appid.c_str());
 780                 free(vconf_str);
 781                 vconf_str = nullptr;
 782         }
 783         vconf_ret = vconf_get_bool(WAKEUP_SETTINGS_KEY_UI_PANEL_ENABLED, &vconf_bool);
 784         if (0 == vconf_ret) {
 785                 g_wakeup_settings.ui_panel_enabled = vconf_bool;
 786                 MWR_LOGD("ui_panel_enabled : %s", (g_wakeup_settings.ui_panel_enabled ? "true" : "false"));
 787         }
 788         vconf_ret = vconf_get_dbl(WAKEUP_SETTINGS_KEY_CONVERSATION_TIMEOUT, &vconf_double);
 789         if (0 == vconf_ret) {
 790                 g_wakeup_settings.conversation_timeout = vconf_double;
 791                 MWR_LOGD("conversation_timeout : %f", g_wakeup_settings.conversation_timeout);
 792         }
 793         vconf_ret = vconf_get_bool(WAKEUP_SETTINGS_KEY_MULTIPLE_MODE, &vconf_bool);
 794         if (0 == vconf_ret) {
 795                 g_wakeup_settings.multiple_mode = vconf_bool;
 796                 MWR_LOGD("multiple_mode : %s", (g_wakeup_settings.multiple_mode ? "true" : "false"));
 797         }
 798         vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_ENABLED_ASSISTANTS);
 799         if (vconf_str) {
 800                 std::string token;
 801                 std::istringstream iss(vconf_str);
 802                 g_wakeup_settings.enabled_assistants.clear();
 803                 while (std::getline(iss, token, ';')) {
 804                         g_wakeup_settings.enabled_assistants.push_back(token);
 805                         MWR_LOGD("enabled_assistants : %s", token.c_str());
 806                 }
 807                 free(vconf_str);
 808                 vconf_str = nullptr;
 809         }
 810         vconf_ret = vconf_get_dbl(WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_DELAY, &vconf_double);
 811         if (0 == vconf_ret) {
 812                 g_wakeup_settings.wakeup_policy_delay = vconf_double;
 813                 MWR_LOGD("conversation_timeout : %f", g_wakeup_settings.wakeup_policy_delay);
 814         }
 815         vconf_str = vconf_get_str(WAKEUP_SETTINGS_KEY_WAKEUP_POLICY_PRIORITY);
 816         if (vconf_str) {
 817                 std::string token;
 818                 std::istringstream iss(vconf_str);
 819                 g_wakeup_settings.wakeup_policy_priority.clear();
 820                 while (std::getline(iss, token, ';')) {
 821                         g_wakeup_settings.wakeup_policy_priority.push_back(token);
 822                         MWR_LOGD("wakeup_policy_priority : %s", token.c_str());
 823                 }
 824                 free(vconf_str);
 825                 vconf_str = nullptr;
 826         }
 827
 828         wakeup_policy_initialize();
 829
 830         recorder_initialize();
 831         wakeup_engine_info_initialize();
 832
 833         MWR_LOGD("[END]");
 834         return 0;
 835 }
 836
 837 int wakeup_manager_deinitialize(void)
 838 {
 839         MWR_LOGD("[ENTER]");
 840
 841         recorder_deinitialize();
 842
 843         for (const auto &data : g_speech_data) {
 844                 free(data.buffer);
 845         }
 846         g_speech_data.clear();
 847
 848         if (g_current_language) {
 849                 free(g_current_language);
 850                 g_current_language = NULL;
 851         }
 852
 853         for (int loop = 0;loop < g_engine_count;loop++) {
 854                 if (g_wakeup_engine_info[loop].interface.deinitialize) {
 855                         g_wakeup_engine_info[loop].interface.deinitialize();
 856                 }
 857         }
 858
 859         MWR_LOGD("[END]");
 860         return 0;
 861 }
 862
 863 static int _is_valid_language(const char* language, bool* is_valid)
 864 {
 865         if (language && is_valid) {
 866                 *is_valid = true;
 867         }
 868         return 0;
 869 }
 870
 871 int wakeup_manager_add_assistant_wakeup_word(const char* appid, const char* wakeup_word, const char* language)
 872 {
 873         MWR_LOGD("[ENTER]");
 874
 875         if (NULL == appid || NULL == wakeup_word) {
 876                 MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), wakeup_word(%s), language(%s)", appid, wakeup_word, language);
 877                 return -1;
 878         }
 879
 880         MWR_LOGD("[DEBUG] appid(%s), wakeup word(%s),language(%s)", appid, wakeup_word, language);
 881         for (int loop = 0;loop < g_engine_count;loop++) {
 882                 auto iter = std::find(
 883                         g_wakeup_engine_info[loop].assistant_list.begin(),
 884                         g_wakeup_engine_info[loop].assistant_list.end(),
 885                         std::string{appid});
 886                 if (iter != g_wakeup_engine_info[loop].assistant_list.end()) {
 887                         if (g_wakeup_engine_info[loop].interface.add_wakeup_word) {
 888                                 g_wakeup_engine_info[loop].interface.add_wakeup_word(
 889                                         appid, wakeup_word, language);
 890                         }
 891                 }
 892         }
 893
 894         MWR_LOGD("[END]");
 895         return 0;
 896 }
 897
 898 int wakeup_manager_add_assistant_language(const char* appid, const char* language)
 899 {
 900         MWR_LOGD("[ENTER]");
 901
 902         if (NULL == appid || NULL == language) {
 903                 MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), language(%s)", appid, language);
 904                 return -1;
 905         }
 906
 907         bool is_valid = false;
 908         int ret = _is_valid_language(language, &is_valid);
 909         if (0 != ret) {
 910                 MWR_LOGE("[ERROR] Fail to do wakeup_service_is_valid_language function (%d)", ret);
 911                 return ret;
 912         }
 913         if (is_valid == false) {
 914                 MWR_LOGE("[ERROR] Not supported language (%s)", language);
 915                 return -1;
 916         }
 917
 918         MWR_LOGD("[DEBUG] language(%s)", language);
 919
 920         MWR_LOGD("[END]");
 921         return 0;
 922 }
 923
 924 int wakeup_manager_set_assistant_wakeup_engine(const char* appid, const char* engine)
 925 {
 926         MWR_LOGD("[ENTER]");
 927
 928         if (NULL == appid || NULL == engine) {
 929                 MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), wakeup engine(%s)", appid, engine);
 930                 return -1;
 931         }
 932
 933         MWR_LOGD("[DEBUG] appid(%s), wakeup engine(%s)", appid, engine);
 934         for (int loop = 0;loop < g_engine_count;loop++) {
 935                 if (0 == strncmp(g_wakeup_engine_info[loop].engine_name, engine, _POSIX_PATH_MAX)) {
 936                         g_wakeup_engine_info[loop].assistant_list.push_back(std::string{appid});
 937                 }
 938         }
 939
 940         MWR_LOGD("[END]");
 941         return 0;
 942 }
 943
 944 int wakeup_manager_set_assistant_enabled(const char* appid, int enabbled)
 945 {
 946         MWR_LOGD("[ENTER]");
 947
 948         MWR_LOGD("[END]");
 949
 950         return 0;
 951 }
 952
 953 int wakeup_manager_set_language(const char* language)
 954 {
 955         MWR_LOGD("[ENTER]");
 956
 957         int ret = 0;
 958         bool is_valid = false;
 959         ret = _is_valid_language(language, &is_valid);
 960         if (0 != ret) {
 961                 MWR_LOGE("[ERROR] wakeup_service_is_valid_language fail (%d)", ret);
 962                 return ret;
 963         }
 964         if (is_valid == false) {
 965                 MWR_LOGE("[ERROR] Not supported language (%s)", language);
 966                 return -1;
 967         }
 968
 969         g_current_language = strdup(language);
 970
 971         MWR_LOGD("[END]");
 972         return 0;
 973 }
 974
 975 int wakeup_manager_change_state(wakeup_manager_state_e state)
 976 {
 977         for (int loop = 0;loop < g_engine_count;loop++) {
 978                 if (g_wakeup_engine_info[loop].interface.update_manager_state) {
 979                         g_wakeup_engine_info[loop].interface.update_manager_state(state);
 980                 }
 981         }
 982 }
 983
 984 int wakeup_manager_activate(void)
 985 {
 986         MWR_LOGD("[ENTER]");
 987
 988         wakeup_manager_change_state(WAKEUP_MANAGER_STATE_LISTENING);
 989
 990         for (int loop = 0;loop < g_engine_count;loop++) {
 991                 if (g_wakeup_engine_info[loop].interface.activate) {
 992                         g_wakeup_engine_info[loop].interface.activate();
 993                 }
 994         }
 995
 996         MWR_LOGD("[END]");
 997         return 0;
 998 }
 999
1000 int wakeup_manager_deactivate(void)
1001 {
1002         MWR_LOGD("[ENTER]");
1003
1004         for (int loop = 0;loop < g_engine_count;loop++) {
1005                 if (g_wakeup_engine_info[loop].interface.deactivate) {
1006                         g_wakeup_engine_info[loop].interface.deactivate();
1007                 }
1008         }
1009         wakeup_manager_change_state(WAKEUP_MANAGER_STATE_INACTIVE);
1010
1011         MWR_LOGD("[END]");
1012         return 0;
1013 }
1014
1015 int wakeup_manager_update_voice_feedback_state(const char* appid, int state)
1016 {
1017         MWR_LOGD("[ENTER]");
1018
1019         if (NULL == appid) {
1020                 MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)",
1021                         (appid ? appid : "NULL"));
1022                 return -1;
1023         }
1024
1025         if (state) {
1026                 if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_LISTENING ||
1027                         g_wakeup_manager_state == WAKEUP_MANAGER_STATE_PROCESSING) {
1028                         wakeup_manager_change_state(WAKEUP_MANAGER_STATE_VOICE_FEEDBACK);
1029                 }
1030         } else {
1031                 if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_VOICE_FEEDBACK) {
1032                         wakeup_manager_change_state(WAKEUP_MANAGER_STATE_LISTENING);
1033                 }
1034         }
1035
1036         MWR_LOGD("[END]");
1037         return 0;
1038 }
1039
1040 int wakeup_manager_send_assistant_specific_command(const char* appid, const char* command)
1041 {
1042         MWR_LOGD("[ENTER]");
1043
1044         if (NULL == appid || NULL == command) {
1045                 MWR_LOGD("[ERROR] Parameter is invalid, appid(%s), command(%s)",
1046                         (appid ? appid : "NULL"), (command ? command : "NULL"));
1047                 return -1;
1048         }
1049
1050         const char* voice_key_pressed = "voice_key_pressed";
1051         const char* voice_key_released = "voice_key_released";
1052
1053         if (command) {
1054                 if (0 == strncmp(command, voice_key_pressed, strlen(voice_key_pressed))) {
1055                         wakeup_manager_process_event(MA_PLUGIN_EVENT_VOICE_KEY_PRESSED, NULL, 0);
1056                 } else if (0 == strncmp(command, voice_key_released, strlen(voice_key_released))) {
1057                         wakeup_manager_process_event(MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, NULL, 0);
1058                 }
1059         }
1060
1061         for (int loop = 0;loop < g_engine_count;loop++) {
1062                 /* Need to find the appropriate engine for this assistant,
1063                    for now assuming 0 is the one */
1064                 if(loop == 0) {
1065                         if (g_wakeup_engine_info[loop].interface.set_assistant_specific_command) {
1066                                 g_wakeup_engine_info[loop].interface.set_assistant_specific_command(appid, command);
1067                         }
1068                 }
1069         }
1070
1071         MWR_LOGD("[END]");
1072         return 0;
1073 }
1074
1075 int wakeup_manager_update_result_state(const char* appid, int state)
1076 {
1077         MWR_LOGD("[ENTER]");
1078         if (NULL == appid) {
1079                 MWR_LOGD("[ERROR] Parameter is invalid, appid(%s)",
1080                         (appid ? appid : "NULL"));
1081                 return -1;
1082         }
1083         if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_PROCESSING) {
1084                 wakeup_manager_change_state(WAKEUP_MANAGER_STATE_LISTENING);
1085         }
1086         MWR_LOGD("[END]");
1087         return 0;
1088 }
1089
1090 int wakeup_manager_process_event(int event, void* data, int len)
1091 {
1092         MWR_LOGD("[ENTER]");
1093
1094         // LOCK REQUIRED
1095         if (event == MA_PLUGIN_EVENT_VOICE_KEY_PRESSED) {
1096                 if (g_voice_key_pressed != true) {
1097                         /* Clear all existing data */
1098                         for (const auto &speech_data : g_speech_data) {
1099                                 if (speech_data.buffer) free(speech_data.buffer);
1100                         }
1101                         g_speech_data.clear();
1102
1103                         g_voice_key_pressed = true;
1104                         /* (Re)Start recorder thread using bt hid */
1105                         start_recorder_thread();
1106                         for (int loop = 0;loop < g_engine_count;loop++) {
1107                                 /* Need to find the default assistant, for now assuming 0 is the one */
1108                                 if(loop == 0) {
1109                                         wakeup_event_info event;
1110                                         event.wakeup_word = "hi bixby";
1111                                         event.wakeup_appid = g_wakeup_engine_info[loop].assistant_list.at(0).c_str();
1112                                         g_wakeup_event_cb(event, g_wakeup_event_user_data);
1113                                 }
1114                         }
1115                 }
1116         } else if (event == MA_PLUGIN_EVENT_VOICE_KEY_RELEASED) {
1117                 if (g_voice_key_pressed != false) {
1118                         unsigned char final_buffer[2] = {'\0', };
1119
1120                         g_voice_key_pressed = false;
1121                         if (g_audio_data_required == true) {
1122                                 /* Restart recorder thread using standard mic */
1123                                 start_recorder_thread();
1124                         } else {
1125                                 join_recorder_thread();
1126                         }
1127                         wakeup_engine_speech_data speech_data;
1128                         speech_data.event = WAKEUP_SPEECH_STREAMING_EVENT_FINISH;
1129                         speech_data.len = sizeof(final_buffer);
1130                         speech_data.buffer = malloc(speech_data.len);
1131                         if (speech_data.buffer) {
1132                                 memcpy(speech_data.buffer, final_buffer, speech_data.len);
1133                                 g_speech_data.push_back(speech_data);
1134                         }
1135                 }
1136         }
1137         // UNLOCK REQUIRED
1138
1139         MWR_LOGD("[END]");
1140         return 0;
1141 }
1142
1143 void __wakeup_service_streaming_cb(wakeup_service_speech_streaming_event_e event, void* buffer, unsigned int len)
1144 {
1145         if (WAKEUP_SPEECH_STREAMING_EVENT_START == event) {
1146                 MWR_LOGD("streaming_cb START");
1147         }
1148         if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == event) {
1149                 MWR_LOGD("streaming_cb FINISH");
1150         }
1151         if (NULL != g_utterance_streaming_cb) {
1152                 g_utterance_streaming_cb(event, buffer, len, g_utterance_streaming_user_data);
1153         } else {
1154                 MWR_LOGI("[INFO] No service streaming callback");
1155         }
1156 }
1157
1158 static void manager_data_thread_func(void)
1159 {
1160         MWR_LOGD("[ENTER]");
1161
1162         MWR_LOGD("data_count : %zu", g_speech_data.size());
1163
1164         int index = 0;
1165
1166         while (1) {
1167                 int ret = -1;
1168                 int cnt = 0;
1169
1170                 /* get feedback data */
1171                 if (index >= g_speech_data.size()) {
1172                         /* empty queue */
1173                         MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
1174
1175                         /* waiting */
1176                         while (1) {
1177                                 usleep(10000);
1178                                 if (index < g_speech_data.size()) {
1179                                         MWR_LOGI("[INFO] Resume thread");
1180                                         break;
1181                                 }
1182                                 if (200 < cnt) {
1183                                         MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
1184                                         __wakeup_service_streaming_cb(
1185                                                 WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0);
1186                                         return;
1187                                 }
1188                                 cnt++;
1189                         }
1190                         MWR_LOGI("[INFO] Finish to wait for new feedback data come");
1191
1192                         /* resume feedback thread */
1193                         continue;
1194                 }
1195
1196                 wakeup_engine_speech_data &speech_data = g_speech_data.at(index);
1197                 __wakeup_service_streaming_cb(
1198                         speech_data.event, speech_data.buffer, speech_data.len);
1199
1200                 if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
1201                         MWR_LOGI("[INFO] Finish to get and send speech data");
1202                         break;
1203                 }
1204
1205                 index++;
1206         }
1207 }
1208
1209 static void engine_data_thread_func(void)
1210 {
1211         MWR_LOGD("[ENTER]");
1212
1213         if (g_wakeup_engine_selected < 0 ||
1214                 g_wakeup_engine_selected >= MAX_WAKEUP_ENGINE_NUM)
1215                 return;
1216
1217         wakeup_engine_interface *interface =
1218                 &(g_wakeup_engine_info[g_wakeup_engine_selected].interface);
1219
1220         if (NULL == interface ||
1221                 NULL == interface->get_utterance_data ||
1222                 NULL == interface->get_utterance_data_count)
1223                 return;
1224
1225         MWR_LOGD("data_count : %d", interface->get_utterance_data_count());
1226
1227         wakeup_engine_speech_data speech_data;
1228         int index = 0;
1229
1230         while (1) {
1231                 int ret = -1;
1232                 int cnt = 0;
1233
1234                 /* get feedback data */
1235                 if (interface && interface->get_utterance_data) {
1236                         ret = interface->get_utterance_data(index, &speech_data);
1237                         if (0 != ret) {
1238                                 /* empty queue */
1239                                 MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
1240
1241                                 /* waiting */
1242                                 while (1) {
1243                                         usleep(10000);
1244                                         if (index < interface->get_utterance_data_count()) {
1245                                                 MWR_LOGI("[INFO] Resume thread");
1246                                                 break;
1247                                         }
1248                                         if (200 < cnt) {
1249                                                 MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
1250                                                 __wakeup_service_streaming_cb(
1251                                                         WAKEUP_SPEECH_STREAMING_EVENT_FAIL, NULL, 0);
1252                                                 return;
1253                                         }
1254                                         cnt++;
1255                                 }
1256                                 MWR_LOGI("[INFO] Finish to wait for new feedback data come");
1257
1258                                 /* resume feedback thread */
1259                                 continue;
1260                         }
1261
1262                         __wakeup_service_streaming_cb(
1263                                 speech_data.event, speech_data.buffer, speech_data.len);
1264
1265                         if (WAKEUP_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
1266                                 MWR_LOGI("[INFO] Finish to get and send speech data");
1267                                 break;
1268                         }
1269
1270                         index++;
1271                 }
1272         }
1273 }
1274
1275 int wakeup_manager_start_streaming_utterance_data(void)
1276 {
1277         MWR_LOGD("[ENTER]");
1278
1279         if (g_speech_data_thread.joinable()) {
1280                 MWR_LOGD("g_speech_data_thread is joinable, trying join()");
1281                 g_speech_data_thread_should_stop.store(true);
1282                 g_speech_data_thread.join();
1283         }
1284         g_speech_data_thread_should_stop.store(false);
1285         if (g_voice_key_pressed) {
1286                 g_speech_data_thread = std::thread(manager_data_thread_func);
1287         } else {
1288                 g_speech_data_thread = std::thread(engine_data_thread_func);
1289         }
1290
1291         MWR_LOGD("[END]");
1292         return 0;
1293 }
1294
1295 int wakeup_manager_stop_streaming_utterance_data(void)
1296 {
1297         MWR_LOGD("[ENTER]");
1298         if (g_wakeup_manager_state == WAKEUP_MANAGER_STATE_UTTERANCE) {
1299                 wakeup_manager_change_state(WAKEUP_MANAGER_STATE_PROCESSING);
1300                 if (g_speech_data_thread.joinable()) {
1301                         MWR_LOGD("g_speech_data_thread is joinable, trying join()");
1302                         g_speech_data_thread_should_stop.store(true);
1303                         g_speech_data_thread.join();
1304                 }
1305         }
1306         MWR_LOGD("[END]");
1307         return 0;
1308 }
1309
1310 int wakeup_manager_start_streaming_follow_up_data(void)
1311 {
1312         MWR_LOGD("[ENTER]");
1313
1314         MWR_LOGD("[END]");
1315         return 0;
1316 }
1317
1318 int wakeup_manager_stop_streaming_follow_up_data(void)
1319 {
1320         MWR_LOGD("[ENTER]");
1321
1322         MWR_LOGD("[END]");
1323         return 0;
1324 }
1325
1326 int wakeup_manager_start_streaming_previous_utterance_data(void)
1327 {
1328         MWR_LOGD("[ENTER]");
1329
1330         MWR_LOGD("[END]");
1331         return 0;
1332 }
1333
1334 int wakeup_manager_stop_streaming_previous_utterance_data(void)
1335 {
1336         MWR_LOGD("[ENTER]");
1337
1338         MWR_LOGD("[END]");
1339         return 0;
1340 }
1341
1342 int wakeup_manager_get_audio_format(int* rate, int* channel, int* audio_type)
1343 {
1344         MWR_LOGD("[ENTER]");
1345
1346         if (!audio_type || !rate || !channel) {
1347                 MWR_LOGE("[ERROR] Invalid parameter");
1348                 return -1;
1349         }
1350
1351         *rate = 16000;
1352         *channel = 0;
1353         *audio_type = 0;
1354
1355         MWR_LOGD("[END] rate(%d), channel(%d), audio_type(%d)", *rate, *channel, *audio_type);
1356         return 0;
1357 }
1358
1359 int wakeup_manager_set_wakeup_event_callback(wakeup_service_wakeup_event_cb callback, void* user_data)
1360 {
1361         MWR_LOGD("[ENTER]");
1362
1363         if (NULL == callback) {
1364                 MWR_LOGE("[ERROR] Input parameter is NULL");
1365                 return -1;
1366         }
1367
1368         g_wakeup_event_cb = callback;
1369         g_wakeup_event_user_data = user_data;
1370
1371         MWR_LOGD("[END]");
1372         return 0;
1373 }
1374
1375 int wakeup_manager_set_utterance_streaming_callback(wakeup_service_speech_streaming_cb callback, void* user_data)
1376 {
1377         MWR_LOGD("[ENTER]");
1378
1379         if (NULL == callback) {
1380                 MWR_LOGE("[ERROR] Input parameter is NULL");
1381                 return -1;
1382         }
1383
1384         g_utterance_streaming_cb = callback;
1385         g_utterance_streaming_user_data = user_data;
1386
1387         MWR_LOGD("[END]");
1388         return 0;
1389 }
1390
1391 int wakeup_manager_set_previous_utterance_streaming_callback(wakeup_service_speech_streaming_cb callback, void* user_data)
1392 {
1393         MWR_LOGD("[ENTER]");
1394
1395         if (NULL == callback) {
1396                 MWR_LOGE("[ERROR] Input parameter is NULL");
1397                 return -1;
1398         }
1399
1400         g_previous_utterance_streaming_cb = callback;
1401         g_previous_utterance_streaming_user_data = user_data;
1402
1403         MWR_LOGD("[END]");
1404         return 0;
1405 }
1406
1407 int wakeup_manager_set_follow_up_streaming_callback(wakeup_service_speech_streaming_cb callback, void* user_data)
1408 {
1409         MWR_LOGD("[ENTER]");
1410
1411         if (NULL == callback) {
1412                 MWR_LOGE("[ERROR] Input parameter is NULL");
1413                 return -1;
1414         }
1415
1416         g_follow_up_streaming_cb = callback;
1417         g_follow_up_streaming_user_data = user_data;
1418
1419         MWR_LOGD("[END]");
1420         return 0;
1421 }
1422
1423 int wakeup_manager_set_speech_status_callback(wakeup_service_speech_status_cb callback, void* user_data)
1424 {
1425         MWR_LOGD("[ENTER]");
1426
1427         if (NULL == callback) {
1428                 MWR_LOGE("[ERROR] Input parameter is NULL");
1429                 return -1;
1430         }
1431
1432         g_speech_status_cb = callback;
1433         g_speech_status_user_data = user_data;
1434
1435         MWR_LOGD("[END]");
1436         return 0;
1437 }
1438
1439 int wakeup_manager_set_error_callback(wakeup_service_error_cb callback, void* user_data)
1440 {
1441         MWR_LOGD("[ENTER]");
1442
1443         if (NULL == callback) {
1444                 MWR_LOGE("[ERROR] Input parameter is NULL");
1445                 return -1;
1446         }
1447
1448         g_error_cb = callback;
1449         g_error_user_data = user_data;
1450
1451         MWR_LOGD("[END]");
1452         return 0;
1453 }