plugins/wakeup-manager/src/wakeup_engine_manager.cpp

   1 #include <multi_assistant_service.h>
   2
   3 #include "wakeup_engine_manager.h"
   4 #include "wakeup_manager_main.h"
   5
   6 #include <dlfcn.h>
   7 #include <algorithm>
   8 #include <pkgmgr-info.h>
   9 #include <deque>
  10
  11 namespace multiassistant
  12 {
  13 namespace wakeup
  14 {
  15
  16 /* Sound buf save for test */
  17 #if 0
  18 #define BUF_SAVE_MODE
  19 #endif
  20
  21 #ifdef BUF_SAVE_MODE
  22 static char g_temp_file_name[128] = {'\0', };
  23
  24 static FILE* g_pFile = NULL;
  25
  26 static int g_count = 1;
  27 #endif
  28
  29 /* Need to check whether this value needs to be configurable */
  30 static int g_speech_pcm_wait_count = 800;
  31
  32 /* Utility function for checking if an element exists in a container */
  33 template<class C, class T>
  34 static auto contains(const C& v, const T& x) -> decltype(end(v), true)
  35 {
  36         return end(v) != find(begin(v), end(v), x);
  37 }
  38
  39 CWakeupEngineManager::CWakeupEngineManager()
  40 {
  41 }
  42
  43 CWakeupEngineManager::~CWakeupEngineManager()
  44 {
  45         MWR_LOGI("Wakeup Engine Manager is now being destroyed");
  46 }
  47
  48 CWakeupEngineManager::CWakeupEngineManager(IEngineEventObserver *observer) : CWakeupEngineManager()
  49 {
  50         subscribe(observer);
  51 }
  52
  53 void CWakeupEngineManager::initialize()
  54 {
  55         DIR* dp = opendir(MA_WAKEUP_ENGINE_PATH);
  56         if (nullptr == dp) {
  57                 MWR_LOGD("Failed opening directory : %s", (const char*)MA_WAKEUP_ENGINE_PATH);
  58         } else {
  59                 struct dirent *dirp = nullptr;
  60                 char dirpath[_POSIX_PATH_MAX];
  61                 do {
  62                         dirp = readdir(dp);
  63
  64                         if (nullptr != dirp) {
  65                                 const string current_directory{"."};
  66                                 const string parent_directory{".."};
  67                                 if (0 == current_directory.compare(dirp->d_name) ||
  68                                         0 == parent_directory.compare(dirp->d_name))
  69                                         continue;
  70
  71                                 if (DT_DIR != dirp->d_type) /* If not a directory */
  72                                         continue;
  73
  74                                 int dirpath_len = strlen(MA_WAKEUP_ENGINE_PATH) + strlen(dirp->d_name) + 1;
  75                                 if (dirpath_len >= _POSIX_PATH_MAX) {
  76                                         MWR_LOGD("File path is too long : %s", dirp->d_name);
  77                                         closedir(dp);
  78                                         return;
  79                                 }
  80
  81                                 memset(dirpath, '\0', _POSIX_PATH_MAX);
  82                                 snprintf(dirpath, _POSIX_PATH_MAX, "%s/%s",
  83                                         (const char*)(MA_WAKEUP_ENGINE_PATH), dirp->d_name);
  84
  85                                 add_engine_directory(string{dirp->d_name}, dirpath);
  86                         }
  87                 } while (nullptr != dirp);
  88
  89                 closedir(dp);
  90         }
  91 }
  92
  93 void CWakeupEngineManager::deinitialize()
  94 {
  95         MWR_LOGI("[START]");
  96         if (mStreamingThread.joinable()) {
  97                 MWR_LOGD("mStreamingThread is joinable, trying join()");
  98                 mStopStreamingThread.store(true);
  99                 mStreamingThread.join();
 100         }
 101         mStopStreamingThread.store(false);
 102
 103         for (auto& info : mEngineInfo) {
 104                 try {
 105                         if (info.interface.set_wakeup_event_callback) {
 106                                 info.interface.set_wakeup_event_callback(nullptr, nullptr);
 107                         }
 108                         if (info.interface.set_speech_status_callback) {
 109                                 info.interface.set_speech_status_callback(nullptr, nullptr);
 110                         }
 111                         if (info.interface.set_error_callback) {
 112                                 info.interface.set_error_callback(nullptr, nullptr);
 113                         }
 114                         if (info.interface.set_audio_data_require_status_callback) {
 115                                 info.interface.set_audio_data_require_status_callback(nullptr, nullptr);
 116                         }
 117                         if (info.interface.set_wakeup_engine_command_callback) {
 118                                 info.interface.set_wakeup_engine_command_callback(nullptr, nullptr);
 119                         }
 120                         MWR_LOGI("Trying to deinitialize engine : %s", info.engine_name.c_str());
 121                         if (info.interface.deinitialize) {
 122                                 int ret = info.interface.deinitialize();
 123                                 MWR_LOGI("Deinitialization [%s] returned %d", info.engine_name.c_str(), ret);
 124                         }
 125                 } catch (const std::exception& e) {
 126                         MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 127                                 info.engine_name.c_str(), e.what());
 128                 }
 129                 if (info.engine_handle) {
 130                         int ret = dlclose(info.engine_handle);
 131                         MWR_LOGI("Closing [%s] returned %d, [%s]", info.engine_name.c_str(), ret,
 132                                 ((0 == ret) ? "" : dlerror()));
 133                         info.engine_handle = nullptr;
 134                 }
 135         }
 136         mSelectedEngine = nullptr;
 137         mEngineInfo.clear();
 138         MWR_LOGI("[END]");
 139 }
 140
 141 void CWakeupEngineManager::subscribe(IEngineEventObserver *observer)
 142 {
 143         mObservers.push_back(observer);
 144         MWR_LOGD("Added Observer : %p %zu", observer, mObservers.size());
 145 }
 146
 147 void CWakeupEngineManager::unsubscribe(IEngineEventObserver *observer)
 148 {
 149         auto iter = find(mObservers.begin(), mObservers.end(), observer);
 150         if (iter != mObservers.end()) {
 151                 mObservers.erase(iter);
 152         }
 153 }
 154
 155 bool CWakeupEngineManager::get_audio_data_required()
 156 {
 157         return mAudioDataRequired;
 158 }
 159
 160 void CWakeupEngineManager::set_selected_wakeup_info(mas_wakeup_event_info wakeup_info)
 161 {
 162         mSelectedEngine = nullptr;
 163
 164         const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(),
 165                 [wakeup_info](const EngineInfo& info) {
 166                         if (nullptr == wakeup_info.wakeup_engine)
 167                                 return false;
 168
 169                         return (0 == info.engine_name.compare(wakeup_info.wakeup_engine));
 170                 });
 171
 172         if (iter != mEngineInfo.end()) {
 173                 mSelectedEngine = &(*iter);
 174                 MWR_LOGD("Selected : %s", iter->engine_name.c_str());
 175         }
 176 }
 177
 178 bool CWakeupEngineManager::set_language(string language)
 179 {
 180         for (const auto& info : mEngineInfo) {
 181                 if (info.interface.set_language) {
 182                         try {
 183                                 info.interface.set_language(language.c_str());
 184                         } catch (const std::exception& e) {
 185                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 186                                         info.engine_name.c_str(), e.what());
 187                         }
 188                 }
 189         }
 190         return true;
 191 }
 192
 193 bool CWakeupEngineManager::set_assistant_language(string appid, string language)
 194 {
 195         for (auto& info : mEngineInfo) {
 196                 const auto& iter = find_if(info.assistant_list.begin(), info.assistant_list.end(),
 197                         [appid](const string& assistant) {
 198                                 return (0 == assistant.compare(appid));
 199                         });
 200
 201                 /* If the appid is in the assistant list */
 202                 if (info.assistant_list.end() != iter) {
 203                         try {
 204                                 int ret = info.interface.set_language(language.c_str());
 205                                 MWR_LOGI("set_language returned %d : %s %s %s",
 206                                         ret, appid.c_str(), info.engine_name.c_str(), language.c_str());
 207                         } catch (const std::exception& e) {
 208                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 209                                         info.engine_name.c_str(), e.what());
 210                         }
 211                 }
 212         }
 213         return true;
 214 }
 215
 216 void CWakeupEngineManager::set_assistant_activated(string appid, bool activated)
 217 {
 218         MWR_LOGI("[ENTER] : %s %d", appid.c_str(), activated);
 219         for (auto& info : mEngineInfo) {
 220                 const auto& iter = find_if(info.assistant_list.begin(), info.assistant_list.end(),
 221                         [appid](const string& assistant) {
 222                                 return (0 == assistant.compare(appid));
 223                         });
 224
 225                 /* If the appid is in the assistant list */
 226                 if (info.assistant_list.end() != iter) {
 227                         bool previously_activated = info.activated;
 228                         if (activated) {
 229                                 info.activated_assistants.insert(appid);
 230                         } else {
 231                                 info.activated_assistants.erase(appid);
 232                         }
 233                         info.activated = (info.activated_assistants.size() > 0);
 234                         if (previously_activated != info.activated) {
 235                                 if (info.activated) {
 236                                         try {
 237                                                 info.interface.activate();
 238                                         } catch (const std::exception& e) {
 239                                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 240                                                         info.engine_name.c_str(), e.what());
 241                                         }
 242                                 } else {
 243                                         try {
 244                                                 info.interface.deactivate();
 245                                         } catch (const std::exception& e) {
 246                                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 247                                                         info.engine_name.c_str(), e.what());
 248                                         }
 249                                 }
 250                                 /* Activated status changed, need to update audio_data_require_status too */
 251                                 on_audio_data_require_status(info.engine_name, info.audio_data_require_status);
 252                         }
 253                 }
 254         }
 255 }
 256
 257 void CWakeupEngineManager::set_wake_word_audio_require_flag(bool require)
 258 {
 259         MWR_LOGD("[ENTER]");
 260         mWakeWordAudioRequired = require;
 261         for (const auto& info : mEngineInfo) {
 262                 if (info.interface.set_wake_word_audio_require_flag) {
 263                         try {
 264                                 info.interface.set_wake_word_audio_require_flag(require);
 265                         } catch (const std::exception& e) {
 266                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 267                                         info.engine_name.c_str(), e.what());
 268                         }
 269                 }
 270         }
 271 }
 272
 273 void CWakeupEngineManager::streaming_speech_data_thread_func()
 274 {
 275         MWR_LOGI("[ENTER]");
 276
 277         if (nullptr == mSelectedEngine) {
 278                 MWR_LOGE("No Engine Selected");
 279                 return;
 280         }
 281
 282         const wakeup_engine_interface *interface = &(mSelectedEngine->interface);
 283
 284         if (NULL == interface ||
 285                 NULL == interface->get_utterance_data ||
 286                 NULL == interface->get_utterance_data_count)
 287                 return;
 288
 289         MWR_LOGD("data_count : %d", interface->get_utterance_data_count());
 290
 291 #ifdef BUF_SAVE_MODE
 292         if (g_pFile) {
 293                 fclose(g_pFile);
 294                 g_pFile = NULL;
 295         } else {
 296                 MWR_LOGD("[Recorder Info] File not found!");
 297         }
 298
 299         while (1) {
 300                 snprintf(g_temp_file_name, sizeof(g_temp_file_name), "/tmp/ma_wue_%d_%d", getpid(), g_count);
 301                 int ret = access(g_temp_file_name, 0);
 302
 303                 if (0 == ret) {
 304                         MWR_LOGD("[Recorder ERROR] File is already exist");
 305                         if (0 == remove(g_temp_file_name)) {
 306                                 MWR_LOGD("[Recorder] Remove file");
 307                                 break;
 308                         } else {
 309                                 g_count++;
 310                         }
 311                 } else {
 312                         break;
 313                 }
 314         }
 315
 316         MWR_LOGD("[Recorder] Temp file name=[%s]", g_temp_file_name);
 317
 318         /* open test file */
 319         g_pFile = fopen(g_temp_file_name, "wb+x");
 320         if (!g_pFile) {
 321                 MWR_LOGD("[Recorder ERROR] File not found!");
 322                 return;
 323         }
 324         g_count++;
 325 #endif
 326
 327         mas_speech_data speech_data;
 328         int index = 0;
 329         bool finish_event_sent = false;
 330
 331         if (mWakeWordAudioRequired &&
 332                 NULL != interface->get_wake_word_data &&
 333                 NULL != interface->get_wake_word_data_count) {
 334                 for (const auto& observer : mObservers) {
 335                         if (observer) {
 336                                 if (!observer->on_audio_streaming_data_section(MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD)) {
 337                                         LOGE("[Recorder WARNING] One of the observer returned false");
 338                                 }
 339                         }
 340                 }
 341                 int count = interface->get_wake_word_data_count();
 342                 while (!(mStopStreamingThread.load()) && index < count) {
 343                         int ret = interface->get_wake_word_data(index, &speech_data);
 344                         if (0 == ret) {
 345 #ifdef BUF_SAVE_MODE
 346                                 if (g_pFile)
 347                                         fwrite(speech_data.buffer, 1, speech_data.len, g_pFile);
 348 #endif
 349                                 for (const auto& observer : mObservers) {
 350                                         if (observer) {
 351                                                 if (!observer->on_streaming_audio_data(
 352                                                         speech_data.event, speech_data.buffer, speech_data.len)) {
 353                                                         LOGE("[Recorder WARNING] One of the observer returned false");
 354                                                 }
 355                                         }
 356                                 }
 357                         } else {
 358                                 break;
 359                         }
 360                         index++;
 361                 }
 362                 index = 0;
 363                 for (const auto& observer : mObservers) {
 364                         if (observer) {
 365                                 if (!observer->on_audio_streaming_data_section(MA_AUDIO_STREAMING_DATA_SECTION_UTTERANCE)) {
 366                                         LOGE("[Recorder WARNING] One of the observer returned false");
 367                                 }
 368                         }
 369                 }
 370         }
 371
 372         int burst_count = 0;
 373         while (!(mStopStreamingThread.load())) {
 374                 int ret = -1;
 375                 int cnt = 0;
 376
 377                 /* get feedback data */
 378                 if (interface && interface->get_utterance_data) {
 379                         ret = interface->get_utterance_data(index, &speech_data);
 380                         if (0 != ret) {
 381                                 /* empty queue */
 382                                 MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
 383
 384                                 /* waiting */
 385                                 while (!(mStopStreamingThread.load())) {
 386                                         burst_count = 0;
 387                                         this_thread::sleep_for(chrono::milliseconds(10));
 388                                         if (index < interface->get_utterance_data_count()) {
 389                                                 MWR_LOGD("[INFO] Resume thread");
 390                                                 break;
 391                                         }
 392                                         if (g_speech_pcm_wait_count < cnt) {
 393                                                 unsigned char final_buffer[2] = {'\0', };
 394                                                 MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
 395 #ifdef BUF_SAVE_MODE
 396                                                 if (g_pFile) {
 397                                                         fwrite(final_buffer, 1, sizeof(final_buffer), g_pFile);
 398                                                         MWR_LOGE("[Recorder SUCCESS] File Close");
 399                                                         fclose(g_pFile);
 400                                                         g_pFile = NULL;
 401                                                 }
 402 #endif
 403                                                 for (const auto& observer : mObservers) {
 404                                                         if (observer) {
 405                                                                 if (!observer->on_streaming_audio_data(
 406                                                                         MAS_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) {
 407                                                                         LOGE("[Recorder WARNING] One of the observer returned false");
 408                                                                 }
 409                                                                 if (!observer->on_streaming_audio_data(
 410                                                                         MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 411                                                                         LOGE("[Recorder WARNING] One of the observer returned false");
 412                                                                 }
 413                                                         }
 414                                                 }
 415                                                 return;
 416                                         }
 417                                         cnt++;
 418                                 }
 419                                 MWR_LOGD("[INFO] Finish to wait for new feedback data come");
 420
 421                                 /* resume feedback thread */
 422                                 continue;
 423                         }
 424
 425 #ifdef BUF_SAVE_MODE
 426                         if (g_pFile)
 427                                 fwrite(speech_data.buffer, 1, speech_data.len, g_pFile);
 428
 429                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 430                                 if (g_pFile) {
 431                                         MWR_LOGE("[Recorder SUCCESS] File Close");
 432                                         fclose(g_pFile);
 433                                         g_pFile = NULL;
 434                                 } else {
 435                                         MWR_LOGE("[Recorder ERROR] File not found!");
 436                                 }
 437                         }
 438 #endif
 439                         const int sleep_duration_in_millis = 10;
 440                         const int max_burst_count = 3;
 441                         if (++burst_count >= max_burst_count) {
 442                                 burst_count = 0;
 443                                 this_thread::sleep_for(chrono::milliseconds(sleep_duration_in_millis));
 444                                 MWR_LOGI("[INFO] Streaming data burst transmission detected, forcing sleep");
 445                         }
 446                         for (const auto& observer : mObservers) {
 447                                 if (observer) {
 448                                         if (!observer->on_streaming_audio_data(
 449                                                 speech_data.event, speech_data.buffer, speech_data.len)) {
 450                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 451                                         }
 452                                 }
 453                         }
 454
 455                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 456                                 MWR_LOGI("[INFO] Finish to get and send speech data");
 457                                 finish_event_sent = true;
 458                                 break;
 459                         }
 460
 461                         index++;
 462                 }
 463         }
 464
 465         if (true != finish_event_sent) {
 466                 unsigned char final_buffer[2] = {'\0', };
 467                 for (const auto& observer : mObservers) {
 468                         if (observer) {
 469                                 MWR_LOGI("No FINISH event sent yet, adding to finalize streaming session");
 470                                 if (!observer->on_streaming_audio_data(
 471                                         MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 472                                         LOGE("[Recorder WARNING] One of the observer returned false");
 473                                 }
 474                         }
 475                 }
 476 #ifdef BUF_SAVE_MODE
 477                 if (g_pFile) {
 478                         fwrite(final_buffer, 1, sizeof(final_buffer), g_pFile);
 479                         MWR_LOGE("[Recorder SUCCESS] File Close");
 480                         fclose(g_pFile);
 481                         g_pFile = NULL;
 482                 }
 483 #endif
 484         }
 485
 486         MWR_LOGI("[EXIT]");
 487 }
 488
 489 void CWakeupEngineManager::start_streaming_current_utterance_data()
 490 {
 491         MWR_LOGI("[ENTER]");
 492         if (mStreamingThread.joinable()) {
 493                 MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread");
 494                 return;
 495         }
 496         mStreamingThread = thread(&CWakeupEngineManager::streaming_speech_data_thread_func, this);
 497 }
 498
 499 void CWakeupEngineManager::stop_streaming_current_utterance_data()
 500 {
 501         MWR_LOGI("[ENTER]");
 502         if (mStreamingThread.joinable()) {
 503                 MWR_LOGD("mStreamingThread is joinable, trying join()");
 504                 mStopStreamingThread.store(true);
 505                 mStreamingThread.join();
 506         }
 507         mStopStreamingThread.store(false);
 508 }
 509
 510 void CWakeupEngineManager::update_manager_state(wakeup_manager_state_e state)
 511 {
 512         for (const auto& info : mEngineInfo) {
 513                 if (info.interface.update_manager_state) {
 514                         try {
 515                                 info.interface.update_manager_state(state);
 516                         } catch (const std::exception& e) {
 517                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 518                                         info.engine_name.c_str(), e.what());
 519                         }
 520                 }
 521         }
 522         mWakeupManagerState = state;
 523 }
 524
 525 void CWakeupEngineManager::update_recognition_result(string appid, int result)
 526 {
 527         if (mSelectedEngine) {
 528                 if (mSelectedEngine->interface.update_recognition_result) {
 529                         mSelectedEngine->interface.update_recognition_result(appid.c_str(), result);
 530                 }
 531         }
 532 }
 533
 534 void CWakeupEngineManager::engine_add_target_assistant(string engine_name, string appid)
 535 {
 536         const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(),
 537                 [engine_name](const EngineInfo& info) {
 538                         return (0 == info.engine_name.compare(engine_name));
 539                 });
 540
 541         if (mEngineInfo.end() == iter) {
 542                 /* Not found, add new library */
 543                 pkgmgrinfo_appinfo_h handle;
 544                 int ret = pkgmgrinfo_appinfo_get_appinfo(engine_name.c_str(), &handle);
 545                 if (PMINFO_R_OK == ret) {
 546                         char *root_path = nullptr;
 547                         ret = pkgmgrinfo_appinfo_get_root_path(handle, &root_path);
 548                         if (PMINFO_R_OK == ret && nullptr != root_path) {
 549                                 string path = root_path;
 550                                 path += "/";
 551                                 path += MA_WAKEUP_DEDICATED_ENGINE_PATH;
 552                                 add_engine(engine_name, path);
 553                         }
 554                         pkgmgrinfo_appinfo_destroy_appinfo(handle);
 555                 }
 556                 /* Find again to add appid to the newly created engine's assistant list */
 557                 const auto &new_iter = find_if(mEngineInfo.begin(), mEngineInfo.end(),
 558                         [engine_name](const EngineInfo& info) {
 559                                 return (0 == info.engine_name.compare(engine_name));
 560                         });
 561                 if (mEngineInfo.end() != new_iter) {
 562                         new_iter->assistant_list.push_back(appid);
 563                 }
 564         } else {
 565                 /* If the engine already exists, simply add the appid to the assistant list */
 566                 iter->assistant_list.push_back(appid);
 567         }
 568 }
 569
 570 void CWakeupEngineManager::engine_add_wakeup_word(string appid, string wakeup_word, string language)
 571 {
 572         for (const auto& info : mEngineInfo) {
 573                 bool found = contains(info.assistant_list, appid);
 574                 if (found) {
 575                         if (info.interface.add_wakeup_word) {
 576                                 try {
 577                                         info.interface.add_wakeup_word(appid.c_str(), wakeup_word.c_str(), language.c_str());
 578                                 } catch (const std::exception& e) {
 579                                         MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 580                                                 info.engine_name.c_str(), e.what());
 581                                 }
 582                         } else {
 583                                 MWR_LOGE("Wakeup Engine does not provide add_wakeup_word");
 584                         }
 585                 }
 586         }
 587 }
 588
 589 void CWakeupEngineManager::engine_remove_wakeup_word(string appid, string wakeup_word, string language)
 590 {
 591         for (const auto& info : mEngineInfo) {
 592                 bool found = contains(info.assistant_list, appid);
 593                 if (found) {
 594                         if (info.interface.remove_wakeup_word) {
 595                                 try {
 596                                         info.interface.remove_wakeup_word(appid.c_str(), wakeup_word.c_str(), language.c_str());
 597                                 } catch (const std::exception& e) {
 598                                         MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 599                                                 info.engine_name.c_str(), e.what());
 600                                 }
 601                         } else {
 602                                 MWR_LOGE("Wakeup Engine does not provide remove_wakeup_word");
 603                         }
 604                 }
 605         }
 606 }
 607
 608 void CWakeupEngineManager::engine_set_assistant_specific_command(string appid, string command)
 609 {
 610         for (const auto& info : mEngineInfo) {
 611                 bool found = contains(info.assistant_list, appid);
 612                 if (found) {
 613                         if (info.interface.set_assistant_specific_command) {
 614                                 try {
 615                                         info.interface.set_assistant_specific_command(appid.c_str(), command.c_str());
 616                                 } catch (const std::exception& e) {
 617                                         MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 618                                                 info.engine_name.c_str(), e.what());
 619                                 }
 620                         }
 621                 }
 622         }
 623 }
 624
 625 void CWakeupEngineManager::engine_feed_audio_data(long time, void* data, int len)
 626 {
 627         for (const auto& info : mEngineInfo) {
 628                 if (info.activated &&
 629                         info.audio_data_require_status &&
 630                         info.interface.feed_audio_data) {
 631                         try {
 632                                 bool filtered_out = false;
 633                                 /* After a wakeup event, wakeup engines other than the selected one
 634                                    does not need to receive audio data. */
 635                                 if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 636                                         if (mSelectedEngine && &info != mSelectedEngine) {
 637                                                 filtered_out = true;
 638                                         }
 639                                 }
 640                                 if (!filtered_out) {
 641                                         int ret = info.interface.feed_audio_data(time, data, len);
 642                                         if (0 != ret) {
 643                                                 LOGE("[ERROR] Fail to feed speech data, ret(%d) : %s", ret, info.engine_name.c_str());
 644                                         }
 645                                 }
 646                         } catch (const std::exception& e) {
 647                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 648                                         info.engine_name.c_str(), e.what());
 649                         }
 650                 }
 651         }
 652 }
 653
 654 void CWakeupEngineManager::engine_set_dependency_module_command(string engine_name, string command)
 655 {
 656         const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(),
 657                 [engine_name](const EngineInfo& info) {
 658                         return (0 == info.engine_name.compare(engine_name));
 659                 });
 660
 661         if (mEngineInfo.end() != iter) {
 662                 if (iter->activated &&
 663                         iter->interface.set_dependency_module_command) {
 664                         try {
 665                                 int ret = iter->interface.set_dependency_module_command(command.c_str());
 666                                 if (0 != ret) {
 667                                         LOGE("[ERROR] Fail to set dependency module command, ret(%d) : %s",
 668                                                 ret, iter->engine_name.c_str());
 669                                 }
 670                         } catch (const std::exception& e) {
 671                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 672                                         iter->engine_name.c_str(), e.what());
 673                         }
 674                 }
 675         }
 676 }
 677
 678 bool CWakeupEngineManager::on_wakeup_event(string engine_name, mas_wakeup_event_info info)
 679 {
 680         MWR_LOGD("[ENTER]");
 681
 682         for (const auto& observer : mObservers) {
 683                 if (observer) {
 684                         if (!observer->on_wakeup_event(engine_name, info)) {
 685                                 LOGE("[Recorder WARNING] One of the observer returned false");
 686                         }
 687                 }
 688         }
 689
 690         return true;
 691 }
 692
 693 bool CWakeupEngineManager::on_speech_status(string engine_name, mas_speech_status_e status)
 694 {
 695         MWR_LOGD("[ENTER]");
 696
 697         for (const auto& observer : mObservers) {
 698                 if (observer) {
 699                         if (!observer->on_speech_status(engine_name, status)) {
 700                                 LOGE("[Recorder WARNING] One of the observer returned false");
 701                         }
 702                 }
 703         }
 704
 705         return true;
 706 }
 707
 708 bool CWakeupEngineManager::on_error(string engine_name, int error_code, string error_message)
 709 {
 710         MWR_LOGD("[ENTER]");
 711
 712         for (const auto& observer : mObservers) {
 713                 if (observer) {
 714                         if (!observer->on_error(engine_name, error_code, error_message)) {
 715                                 LOGE("[Recorder WARNING] One of the observer returned false");
 716                         }
 717                 }
 718         }
 719
 720         return true;
 721 }
 722
 723 bool CWakeupEngineManager::on_audio_data_require_status(string engine_name, bool require)
 724 {
 725         MWR_LOGI("[ENTER] %s, %d", engine_name.c_str(), require);
 726
 727         bool found = false;
 728         // LOCK REQUIRED
 729         int count = 0;
 730         for (auto& info : mEngineInfo) {
 731                 if (info.engine_name.compare(engine_name) == 0) {
 732                         found = true;
 733                         info.audio_data_require_status = require;
 734                 }
 735                 if (info.activated && info.audio_data_require_status) {
 736                         count++;
 737                 }
 738         }
 739         MWR_LOGD("count : %d", count);
 740         if (count > 0) {
 741                 mAudioDataRequired = true;
 742         } else {
 743                 mAudioDataRequired = false;
 744         }
 745
 746         if (found) {
 747                 for (const auto& observer : mObservers) {
 748                         if (observer) {
 749                                 if (!observer->on_audio_data_require_status(engine_name, require)) {
 750                                         LOGE("[Recorder WARNING] One of the observer returned false");
 751                                 }
 752                         }
 753                 }
 754         }
 755         // UNLOCK REQUIRED
 756         return true;
 757 }
 758
 759 bool CWakeupEngineManager::on_wakeup_engine_command(string engine_name, mas_wakeup_engine_command_target_e target, string assistant_name, string command)
 760 {
 761         MWR_LOGI("[ENTER] : %s %d %s %s",
 762                 engine_name.c_str(), target, assistant_name.c_str(), command.c_str());
 763
 764         for (const auto& observer : mObservers) {
 765                 if (observer) {
 766                         if (MAS_WAKEUP_ENGINE_COMMAND_TARGET_DEPENDENCY_MODULE == target) {
 767                                 if (!observer->on_wakeup_engine_command(target, engine_name, assistant_name, command)) {
 768                                         LOGE("[Recorder WARNING] One of the observer returned false");
 769                                 }
 770                         } else {
 771                                 const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(),
 772                                         [engine_name](const EngineInfo& info) {
 773                                                 return (0 == info.engine_name.compare(engine_name));
 774                                         });
 775                                 if (mEngineInfo.end() != iter) {
 776                                         for (const auto& assistant : iter->assistant_list) {
 777                                                 if (0 == assistant_name.compare(assistant) ||
 778                                                         MAS_WAKEUP_ENGINE_COMMAND_TARGET_ALL_ASSISTANTS == target) {
 779                                                                 MWR_LOGI("Calling on_wakeup_engine_command for %s", assistant.c_str());
 780                                                         if (!observer->on_wakeup_engine_command(target, engine_name, assistant, command)) {
 781                                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 782                                                         }
 783                                                 }
 784                                         }
 785                                 }
 786                         }
 787                 }
 788         }
 789
 790         return true;
 791 }
 792
 793 void CWakeupEngineManager::add_engine(string name, string path)
 794 {
 795         MWR_LOGD("Name (%s), Filepath(%s)", name.c_str(), path.c_str());
 796
 797         char* error = NULL;
 798         EngineInfo info;
 799         info.engine_handle = dlopen(path.c_str(), RTLD_LAZY);
 800         if (nullptr != (error = dlerror()) || nullptr == info.engine_handle) {
 801                 MWR_LOGD("[ERROR] Fail to dlopen(%s), error(%s)", path.c_str(), error);
 802                 if (info.engine_handle) dlclose(info.engine_handle);
 803                 return;
 804         }
 805
 806         /* Interfaces without version information */
 807         info.interface.initialize =
 808                 (wakeup_engine_initialize)dlsym(info.engine_handle,
 809                 MA_WAKEUP_ENGINE_FUNC_INITIALIZE);
 810         info.interface.deinitialize =
 811                 (wakeup_engine_deinitialize)dlsym(info.engine_handle,
 812                 MA_WAKEUP_ENGINE_FUNC_DEINITIALIZE);
 813         info.interface.activate =
 814                 (wakeup_engine_activate)dlsym(info.engine_handle,
 815                 MA_WAKEUP_ENGINE_FUNC_ACTIVATE);
 816         info.interface.deactivate =
 817                 (wakeup_engine_deactivate)dlsym(info.engine_handle,
 818                 MA_WAKEUP_ENGINE_FUNC_DEACTIVATE);
 819         info.interface.add_wakeup_word =
 820                 (wakeup_engine_add_wakeup_word)dlsym(info.engine_handle,
 821                 MA_WAKEUP_ENGINE_FUNC_ADD_WAKEUP_WORD);
 822         info.interface.remove_wakeup_word =
 823                 (wakeup_engine_remove_wakeup_word)dlsym(info.engine_handle,
 824                 MA_WAKEUP_ENGINE_FUNC_REMOVE_WAKEUP_WORD);
 825         info.interface.add_language =
 826                 (wakeup_engine_add_language)dlsym(info.engine_handle,
 827                 MA_WAKEUP_ENGINE_FUNC_ADD_LANGUAGE);
 828         info.interface.set_language =
 829                 (wakeup_engine_set_language)dlsym(info.engine_handle,
 830                 MA_WAKEUP_ENGINE_FUNC_SET_LANGUAGE);
 831         info.interface.update_manager_state =
 832                 (wakeup_engine_update_manager_state)dlsym(info.engine_handle,
 833                 MA_WAKEUP_ENGINE_FUNC_UPDATE_MANAGER_STATE);
 834         info.interface.update_recognition_result =
 835                 (wakeup_engine_update_recognition_result)dlsym(info.engine_handle,
 836                 MA_WAKEUP_ENGINE_FUNC_UPDATE_RECOGNITION_RESULT);
 837         info.interface.set_audio_format =
 838                 (wakeup_engine_set_audio_format)dlsym(info.engine_handle,
 839                 MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_FORMAT);
 840         info.interface.get_audio_format =
 841                 (wakeup_engine_get_audio_format)dlsym(info.engine_handle,
 842                 MA_WAKEUP_ENGINE_FUNC_GET_AUDIO_FORMAT);
 843         info.interface.feed_audio_data =
 844                 (wakeup_engine_feed_audio_data)dlsym(info.engine_handle,
 845                 MA_WAKEUP_ENGINE_FUNC_FEED_AUDIO_DATA);
 846         info.interface.get_utterance_data_count =
 847                 (wakeup_engine_get_utterance_data_count)dlsym(info.engine_handle,
 848                 MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA_COUNT);
 849         info.interface.get_utterance_data =
 850                 (wakeup_engine_get_utterance_data)dlsym(info.engine_handle,
 851                 MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA);
 852         info.interface.get_wake_word_data_count =
 853                 (wakeup_engine_get_wake_word_data_count)dlsym(info.engine_handle,
 854                 MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA_COUNT);
 855         info.interface.get_wake_word_data =
 856                 (wakeup_engine_get_wake_word_data)dlsym(info.engine_handle,
 857                 MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA);
 858         info.interface.set_assistant_specific_command =
 859                 (wakeup_engine_set_assistant_specific_command)dlsym(info.engine_handle,
 860                 MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND);
 861         info.interface.set_wake_word_audio_require_flag =
 862                 (wakeup_engine_set_wake_word_audio_require_flag)dlsym(info.engine_handle,
 863                 MA_WAKEUP_ENGINE_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG);
 864         info.interface.set_wakeup_event_callback =
 865                 (wakeup_engine_set_wakeup_event_callback)dlsym(info.engine_handle,
 866                 MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK);
 867         info.interface.set_speech_status_callback =
 868                 (wakeup_engine_set_speech_status_callback)dlsym(info.engine_handle,
 869                 MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK);
 870         info.interface.set_error_callback =
 871                 (wakeup_engine_set_error_callback)dlsym(info.engine_handle,
 872                 MA_WAKEUP_ENGINE_FUNC_SET_ERROR_CALLBACK);
 873         info.interface.set_audio_data_require_status_callback =
 874                 (wakeup_engine_set_audio_data_require_status_callback)dlsym(info.engine_handle,
 875                 MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_DATA_REQUIRE_STATUS_CALLBACK);
 876         info.interface.set_wakeup_engine_command_callback =
 877                 (wakeup_engine_set_wakeup_engine_command_callback)dlsym(info.engine_handle,
 878                 MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_ENGINE_COMMAND_CALLBACK);
 879
 880         /* Interfaces after version 1 */
 881         info.interface.get_version =
 882                 (wakeup_engine_get_version)dlsym(info.engine_handle,
 883                 MA_WAKEUP_ENGINE_FUNC_GET_VERSION);
 884         info.interface.set_dependency_module_command =
 885                 (wakeup_engine_set_dependency_module_command)dlsym(info.engine_handle,
 886                 MA_WAKEUP_ENGINE_FUNC_SET_DEPENDENCY_MODULE_COMMAND);
 887
 888         info.version = 0;
 889         info.engine_path = path;
 890         info.engine_name = name;
 891
 892         info.activated = false;
 893         info.audio_data_require_status = false;
 894
 895         /* All the necessary information has already been set properly */
 896         mEngineInfo.push_back(info);
 897
 898         /* Workaround for registering C-style callbacks */
 899         typedef struct {
 900                 CWakeupEngineManager *manager;
 901                 string engine_name;
 902         } CallbackUserData;
 903
 904         static deque<CallbackUserData> callback_user_data;
 905
 906         CallbackUserData user_data;
 907         user_data.manager = this;
 908         user_data.engine_name = info.engine_name;
 909         callback_user_data.push_back(user_data);
 910
 911         MWR_LOGI("Initializing wakeup engine : %s %p %p",
 912                 info.engine_path.c_str(),
 913                 info.interface.initialize,
 914                 &callback_user_data.back());
 915
 916         try {
 917                 if (info.interface.set_wakeup_event_callback) {
 918                         info.interface.set_wakeup_event_callback(
 919                                 [](mas_wakeup_event_info info, void* user_data) {
 920                                         MWR_LOGI("user_data : %p", user_data);
 921                                         CallbackUserData *data = static_cast<CallbackUserData*>(user_data);
 922                                         if (nullptr == data) return;
 923                                         if (nullptr == data->manager) return;
 924                                         info.wakeup_engine = data->engine_name.c_str();
 925                                         data->manager->on_wakeup_event(data->engine_name, info);
 926                                 }, &(callback_user_data.back()));
 927                 }
 928
 929                 if (info.interface.set_audio_data_require_status_callback) {
 930                         info.interface.set_audio_data_require_status_callback(
 931                                 [](bool require, void* user_data) {
 932                                         MWR_LOGI("user_data : %p", user_data);
 933                                         CallbackUserData *data = static_cast<CallbackUserData*>(user_data);
 934                                         if (nullptr == data) return;
 935                                         if (nullptr == data->manager) return;
 936                                         data->manager->on_audio_data_require_status(data->engine_name, require);
 937                                 }, &(callback_user_data.back()));
 938                 }
 939
 940                 if (info.interface.set_wakeup_engine_command_callback) {
 941                         info.interface.set_wakeup_engine_command_callback(
 942                                 [](mas_wakeup_engine_command_target_e target,
 943                                         const char* assistant_name, const char* command, void* user_data) {
 944                                         MWR_LOGI("user_data : %p", user_data);
 945                                         CallbackUserData* data = static_cast<CallbackUserData*>(user_data);
 946                                         if (nullptr == data) return;
 947                                         if (nullptr == data->manager) return;
 948                                         if (nullptr == command) return;
 949                                         data->manager->on_wakeup_engine_command(
 950                                                 data->engine_name, target, (assistant_name ? assistant_name : ""), command);
 951                                 }, &(callback_user_data.back()));
 952                 }
 953
 954                 if (info.interface.initialize) {
 955                         info.interface.initialize();
 956                 }
 957                 if (info.interface.get_version) {
 958                         int version;
 959                         if (0 == info.interface.get_version(&version)) {
 960                                 info.version = version;
 961                         }
 962                 }
 963         } catch (const std::exception& e) {
 964                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 965                         info.engine_name.c_str(), e.what());
 966         }
 967 }
 968
 969 void CWakeupEngineManager::add_engine_directory(string name, string path)
 970 {
 971         if (0 == path.size()) return;
 972
 973         DIR* dp = opendir(path.c_str());
 974         if (NULL == dp) {
 975                 MWR_LOGD("Failed opening directory : %s", path.c_str());
 976         } else {
 977                 struct dirent *dirp = NULL;
 978                 string filepath;
 979                 do {
 980                         dirp = readdir(dp);
 981
 982                         if (NULL != dirp) {
 983                                 if (!strcmp(".", dirp->d_name) || !strcmp("..", dirp->d_name))
 984                                         continue;
 985
 986                                 if (DT_REG != dirp->d_type) /* If not a regular file */
 987                                         continue;
 988
 989                                 filepath = path;
 990                                 filepath += "/";
 991                                 filepath += dirp->d_name;
 992
 993                                 if (filepath.length() >= _POSIX_PATH_MAX) {
 994                                         MWR_LOGD("File path is too long : %s", filepath.c_str());
 995                                         closedir(dp);
 996                                         return;
 997                                 }
 998                                 add_engine(name, filepath);
 999                         }
1000                 } while (NULL != dirp);
1001
1002                 closedir(dp);
1003         }
1004 }
1005
1006 } // wakeup
1007 } // multiassistant