plugins/wakeup-manager/src/wakeup_engine_manager.cpp

   1 #include <multi_assistant_service.h>
   2
   3 #include "wakeup_engine_manager.h"
   4 #include "wakeup_manager_main.h"
   5
   6 #include <dlfcn.h>
   7 #include <algorithm>
   8 #include <pkgmgr-info.h>
   9 #include <deque>
  10
  11 namespace multiassistant
  12 {
  13 namespace wakeup
  14 {
  15
  16 /* Sound buf save for test */
  17 #if 0
  18 #define BUF_SAVE_MODE
  19 #endif
  20
  21 #ifdef BUF_SAVE_MODE
  22 static char g_temp_file_name[128] = {'\0', };
  23
  24 static FILE* g_pFile = NULL;
  25
  26 static int g_count = 1;
  27 #endif
  28
  29 /* Need to check whether this value needs to be configurable */
  30 static int g_speech_pcm_wait_count = 800;
  31
  32 /* Utility function for checking if an element exists in a container */
  33 template<class C, class T>
  34 static auto contains(const C& v, const T& x) -> decltype(end(v), true)
  35 {
  36         return end(v) != find(begin(v), end(v), x);
  37 }
  38
  39 CWakeupEngineManager::CWakeupEngineManager()
  40 {
  41 }
  42
  43 CWakeupEngineManager::~CWakeupEngineManager()
  44 {
  45         MWR_LOGI("Wakeup Engine Manager is now being destroyed");
  46 }
  47
  48 CWakeupEngineManager::CWakeupEngineManager(IEngineEventObserver *observer) : CWakeupEngineManager()
  49 {
  50         subscribe(observer);
  51 }
  52
  53 void CWakeupEngineManager::initialize()
  54 {
  55         DIR* dp = opendir(MA_WAKEUP_ENGINE_PATH);
  56         if (nullptr == dp) {
  57                 MWR_LOGD("Failed opening directory : %s", (const char*)MA_WAKEUP_ENGINE_PATH);
  58         } else {
  59                 struct dirent *dirp = nullptr;
  60                 char dirpath[_POSIX_PATH_MAX];
  61                 do {
  62                         dirp = readdir(dp);
  63
  64                         if (nullptr != dirp) {
  65                                 const string current_directory{"."};
  66                                 const string parent_directory{".."};
  67                                 if (0 == current_directory.compare(dirp->d_name) ||
  68                                         0 == parent_directory.compare(dirp->d_name))
  69                                         continue;
  70
  71                                 if (DT_DIR != dirp->d_type) /* If not a directory */
  72                                         continue;
  73
  74                                 int dirpath_len = strlen(MA_WAKEUP_ENGINE_PATH) + strlen(dirp->d_name) + 1;
  75                                 if (dirpath_len >= _POSIX_PATH_MAX) {
  76                                         MWR_LOGD("File path is too long : %s", dirp->d_name);
  77                                         closedir(dp);
  78                                         return;
  79                                 }
  80
  81                                 memset(dirpath, '\0', _POSIX_PATH_MAX);
  82                                 snprintf(dirpath, _POSIX_PATH_MAX, "%s/%s",
  83                                         (const char*)(MA_WAKEUP_ENGINE_PATH), dirp->d_name);
  84
  85                                 add_engine_directory(string{dirp->d_name}, dirpath);
  86                         }
  87                 } while (nullptr != dirp);
  88
  89                 closedir(dp);
  90         }
  91 }
  92
  93 void CWakeupEngineManager::deinitialize()
  94 {
  95         MWR_LOGI("[START]");
  96         if (mStreamingThread.joinable()) {
  97                 MWR_LOGD("mStreamingThread is joinable, trying join()");
  98                 mStopStreamingThread.store(true);
  99                 mStreamingThread.join();
 100         }
 101         mStopStreamingThread.store(false);
 102
 103         for (auto& info : mEngineInfo) {
 104                 try {
 105                         if (info.interface.set_wakeup_event_callback) {
 106                                 info.interface.set_wakeup_event_callback(nullptr, nullptr);
 107                         }
 108                         if (info.interface.set_speech_status_callback) {
 109                                 info.interface.set_speech_status_callback(nullptr, nullptr);
 110                         }
 111                         if (info.interface.set_error_callback) {
 112                                 info.interface.set_error_callback(nullptr, nullptr);
 113                         }
 114                         if (info.interface.set_audio_data_require_status_callback) {
 115                                 info.interface.set_audio_data_require_status_callback(nullptr, nullptr);
 116                         }
 117                         if (info.interface.set_wakeup_engine_command_callback) {
 118                                 info.interface.set_wakeup_engine_command_callback(nullptr, nullptr);
 119                         }
 120                         MWR_LOGI("Trying to deinitialize engine : %s", info.engine_name.c_str());
 121                         if (info.interface.deinitialize) {
 122                                 int ret = info.interface.deinitialize();
 123                                 MWR_LOGI("Deinitialization [%s] returned %d", info.engine_name.c_str(), ret);
 124                         }
 125                 } catch (const std::exception& e) {
 126                         MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 127                                 info.engine_name.c_str(), e.what());
 128                 }
 129                 if (info.engine_handle) {
 130                         int ret = dlclose(info.engine_handle);
 131                         MWR_LOGI("Closing [%s] returned %d, [%s]", info.engine_name.c_str(), ret,
 132                                 ((0 == ret) ? "" : dlerror()));
 133                         info.engine_handle = nullptr;
 134                 }
 135         }
 136         mSelectedEngine = nullptr;
 137         mEngineInfo.clear();
 138         MWR_LOGI("[END]");
 139 }
 140
 141 void CWakeupEngineManager::subscribe(IEngineEventObserver *observer)
 142 {
 143         mObservers.push_back(observer);
 144         MWR_LOGD("Added Observer : %p %zu", observer, mObservers.size());
 145 }
 146
 147 void CWakeupEngineManager::unsubscribe(IEngineEventObserver *observer)
 148 {
 149         auto iter = find(mObservers.begin(), mObservers.end(), observer);
 150         if (iter != mObservers.end()) {
 151                 mObservers.erase(iter);
 152         }
 153 }
 154
 155 bool CWakeupEngineManager::get_audio_data_required()
 156 {
 157         return mAudioDataRequired;
 158 }
 159
 160 void CWakeupEngineManager::set_selected_wakeup_info(mas_wakeup_event_info wakeup_info)
 161 {
 162         mSelectedEngine = nullptr;
 163         for (const auto& info : mEngineInfo) {
 164                 string appid = string{wakeup_info.wakeup_appid};
 165                 bool found = contains(info.assistant_list, appid);
 166
 167                 if (found) {
 168                         mSelectedEngine = &info;
 169                         MWR_LOGD("Selected : %s", info.engine_name.c_str());
 170                 }
 171         }
 172 }
 173
 174 bool CWakeupEngineManager::set_language(string language)
 175 {
 176         for (const auto& info : mEngineInfo) {
 177                 if (info.interface.set_language) {
 178                         try {
 179                                 info.interface.set_language(language.c_str());
 180                         } catch (const std::exception& e) {
 181                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 182                                         info.engine_name.c_str(), e.what());
 183                         }
 184                 }
 185         }
 186         return true;
 187 }
 188
 189 bool CWakeupEngineManager::set_assistant_language(string appid, string language)
 190 {
 191         for (auto& info : mEngineInfo) {
 192                 const auto& iter = find_if(info.assistant_list.begin(), info.assistant_list.end(),
 193                         [appid](const string& assistant) {
 194                                 return (0 == assistant.compare(appid));
 195                         });
 196
 197                 /* If the appid is in the assistant list */
 198                 if (info.assistant_list.end() != iter) {
 199                         try {
 200                                 int ret = info.interface.set_language(language.c_str());
 201                                 MWR_LOGI("set_language returned %d : %s %s %s",
 202                                         ret, appid.c_str(), info.engine_name.c_str(), language.c_str());
 203                         } catch (const std::exception& e) {
 204                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 205                                         info.engine_name.c_str(), e.what());
 206                         }
 207                 }
 208         }
 209         return true;
 210 }
 211
 212 void CWakeupEngineManager::set_assistant_activated(string appid, bool activated)
 213 {
 214         MWR_LOGI("[ENTER] : %s %d", appid.c_str(), activated);
 215         for (auto& info : mEngineInfo) {
 216                 const auto& iter = find_if(info.assistant_list.begin(), info.assistant_list.end(),
 217                         [appid](const string& assistant) {
 218                                 return (0 == assistant.compare(appid));
 219                         });
 220
 221                 /* If the appid is in the assistant list */
 222                 if (info.assistant_list.end() != iter) {
 223                         bool previously_activated = info.activated;
 224                         if (activated) {
 225                                 info.activated_assistants.insert(appid);
 226                         } else {
 227                                 info.activated_assistants.erase(appid);
 228                         }
 229                         info.activated = (info.activated_assistants.size() > 0);
 230                         if (previously_activated != info.activated) {
 231                                 if (info.activated) {
 232                                         try {
 233                                                 info.interface.activate();
 234                                         } catch (const std::exception& e) {
 235                                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 236                                                         info.engine_name.c_str(), e.what());
 237                                         }
 238                                 } else {
 239                                         try {
 240                                                 info.interface.deactivate();
 241                                         } catch (const std::exception& e) {
 242                                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 243                                                         info.engine_name.c_str(), e.what());
 244                                         }
 245                                 }
 246                                 /* Activated status changed, need to update audio_data_require_status too */
 247                                 on_audio_data_require_status(info.engine_name, info.audio_data_require_status);
 248                         }
 249                 }
 250         }
 251 }
 252
 253 void CWakeupEngineManager::set_wake_word_audio_require_flag(bool require)
 254 {
 255         MWR_LOGD("[ENTER]");
 256         mWakeWordAudioRequired = require;
 257         for (const auto& info : mEngineInfo) {
 258                 if (info.interface.set_wake_word_audio_require_flag) {
 259                         try {
 260                                 info.interface.set_wake_word_audio_require_flag(require);
 261                         } catch (const std::exception& e) {
 262                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 263                                         info.engine_name.c_str(), e.what());
 264                         }
 265                 }
 266         }
 267 }
 268
 269 void CWakeupEngineManager::streaming_speech_data_thread_func()
 270 {
 271         MWR_LOGI("[ENTER]");
 272
 273         if (nullptr == mSelectedEngine)
 274                 return;
 275
 276         const wakeup_engine_interface *interface = &(mSelectedEngine->interface);
 277
 278         if (NULL == interface ||
 279                 NULL == interface->get_utterance_data ||
 280                 NULL == interface->get_utterance_data_count)
 281                 return;
 282
 283         MWR_LOGD("data_count : %d", interface->get_utterance_data_count());
 284
 285 #ifdef BUF_SAVE_MODE
 286         if (g_pFile) {
 287                 fclose(g_pFile);
 288                 g_pFile = NULL;
 289         } else {
 290                 MWR_LOGD("[Recorder Info] File not found!");
 291         }
 292
 293         while (1) {
 294                 snprintf(g_temp_file_name, sizeof(g_temp_file_name), "/tmp/ma_wue_%d_%d", getpid(), g_count);
 295                 int ret = access(g_temp_file_name, 0);
 296
 297                 if (0 == ret) {
 298                         MWR_LOGD("[Recorder ERROR] File is already exist");
 299                         if (0 == remove(g_temp_file_name)) {
 300                                 MWR_LOGD("[Recorder] Remove file");
 301                                 break;
 302                         } else {
 303                                 g_count++;
 304                         }
 305                 } else {
 306                         break;
 307                 }
 308         }
 309
 310         MWR_LOGD("[Recorder] Temp file name=[%s]", g_temp_file_name);
 311
 312         /* open test file */
 313         g_pFile = fopen(g_temp_file_name, "wb+x");
 314         if (!g_pFile) {
 315                 MWR_LOGD("[Recorder ERROR] File not found!");
 316                 return;
 317         }
 318         g_count++;
 319 #endif
 320
 321         mas_speech_data speech_data;
 322         int index = 0;
 323         bool finish_event_sent = false;
 324
 325         if (mWakeWordAudioRequired &&
 326                 NULL != interface->get_wake_word_data &&
 327                 NULL != interface->get_wake_word_data_count) {
 328                 for (const auto& observer : mObservers) {
 329                         if (observer) {
 330                                 if (!observer->on_audio_streaming_data_section(MA_AUDIO_STREAMING_DATA_SECTION_WAKE_WORD)) {
 331                                         LOGE("[Recorder WARNING] One of the observer returned false");
 332                                 }
 333                         }
 334                 }
 335                 int count = interface->get_wake_word_data_count();
 336                 while (!(mStopStreamingThread.load()) && index < count) {
 337                         int ret = interface->get_wake_word_data(index, &speech_data);
 338                         if (0 == ret) {
 339 #ifdef BUF_SAVE_MODE
 340                                 if (g_pFile)
 341                                         fwrite(speech_data.buffer, 1, speech_data.len, g_pFile);
 342 #endif
 343                                 for (const auto& observer : mObservers) {
 344                                         if (observer) {
 345                                                 if (!observer->on_streaming_audio_data(
 346                                                         speech_data.event, speech_data.buffer, speech_data.len)) {
 347                                                         LOGE("[Recorder WARNING] One of the observer returned false");
 348                                                 }
 349                                         }
 350                                 }
 351                         } else {
 352                                 break;
 353                         }
 354                         index++;
 355                 }
 356                 index = 0;
 357                 for (const auto& observer : mObservers) {
 358                         if (observer) {
 359                                 if (!observer->on_audio_streaming_data_section(MA_AUDIO_STREAMING_DATA_SECTION_UTTERANCE)) {
 360                                         LOGE("[Recorder WARNING] One of the observer returned false");
 361                                 }
 362                         }
 363                 }
 364         }
 365
 366         int burst_count = 0;
 367         while (!(mStopStreamingThread.load())) {
 368                 int ret = -1;
 369                 int cnt = 0;
 370
 371                 /* get feedback data */
 372                 if (interface && interface->get_utterance_data) {
 373                         ret = interface->get_utterance_data(index, &speech_data);
 374                         if (0 != ret) {
 375                                 /* empty queue */
 376                                 MWR_LOGD("[DEBUG] No feedback data. Waiting mode : %d", ret);
 377
 378                                 /* waiting */
 379                                 while (!(mStopStreamingThread.load())) {
 380                                         burst_count = 0;
 381                                         this_thread::sleep_for(chrono::milliseconds(10));
 382                                         if (index < interface->get_utterance_data_count()) {
 383                                                 MWR_LOGD("[INFO] Resume thread");
 384                                                 break;
 385                                         }
 386                                         if (g_speech_pcm_wait_count < cnt) {
 387                                                 unsigned char final_buffer[2] = {'\0', };
 388                                                 MWR_LOGE("[ERROR] Wrong request, there's no pcm data");
 389 #ifdef BUF_SAVE_MODE
 390                                                 if (g_pFile) {
 391                                                         fwrite(final_buffer, 1, sizeof(final_buffer), g_pFile);
 392                                                         MWR_LOGE("[Recorder SUCCESS] File Close");
 393                                                         fclose(g_pFile);
 394                                                         g_pFile = NULL;
 395                                                 }
 396 #endif
 397                                                 for (const auto& observer : mObservers) {
 398                                                         if (observer) {
 399                                                                 if (!observer->on_streaming_audio_data(
 400                                                                         MAS_SPEECH_STREAMING_EVENT_FAIL, NULL, 0)) {
 401                                                                         LOGE("[Recorder WARNING] One of the observer returned false");
 402                                                                 }
 403                                                                 if (!observer->on_streaming_audio_data(
 404                                                                         MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 405                                                                         LOGE("[Recorder WARNING] One of the observer returned false");
 406                                                                 }
 407                                                         }
 408                                                 }
 409                                                 return;
 410                                         }
 411                                         cnt++;
 412                                 }
 413                                 MWR_LOGD("[INFO] Finish to wait for new feedback data come");
 414
 415                                 /* resume feedback thread */
 416                                 continue;
 417                         }
 418
 419 #ifdef BUF_SAVE_MODE
 420                         if (g_pFile)
 421                                 fwrite(speech_data.buffer, 1, speech_data.len, g_pFile);
 422
 423                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 424                                 if (g_pFile) {
 425                                         MWR_LOGE("[Recorder SUCCESS] File Close");
 426                                         fclose(g_pFile);
 427                                         g_pFile = NULL;
 428                                 } else {
 429                                         MWR_LOGE("[Recorder ERROR] File not found!");
 430                                 }
 431                         }
 432 #endif
 433                         const int sleep_duration_in_millis = 10;
 434                         const int max_burst_count = 3;
 435                         if (++burst_count >= max_burst_count) {
 436                                 burst_count = 0;
 437                                 this_thread::sleep_for(chrono::milliseconds(sleep_duration_in_millis));
 438                                 MWR_LOGI("[INFO] Streaming data burst transmission detected, forcing sleep");
 439                         }
 440                         for (const auto& observer : mObservers) {
 441                                 if (observer) {
 442                                         if (!observer->on_streaming_audio_data(
 443                                                 speech_data.event, speech_data.buffer, speech_data.len)) {
 444                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 445                                         }
 446                                 }
 447                         }
 448
 449                         if (MAS_SPEECH_STREAMING_EVENT_FINISH == speech_data.event) {
 450                                 MWR_LOGI("[INFO] Finish to get and send speech data");
 451                                 finish_event_sent = true;
 452                                 break;
 453                         }
 454
 455                         index++;
 456                 }
 457         }
 458
 459         if (true != finish_event_sent) {
 460                 unsigned char final_buffer[2] = {'\0', };
 461                 for (const auto& observer : mObservers) {
 462                         if (observer) {
 463                                 if (!observer->on_streaming_audio_data(
 464                                         MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer))) {
 465                                         LOGE("[Recorder WARNING] One of the observer returned false");
 466                                 }
 467                         }
 468                 }
 469 #ifdef BUF_SAVE_MODE
 470                 if (g_pFile) {
 471                         fwrite(final_buffer, 1, sizeof(final_buffer), g_pFile);
 472                         MWR_LOGE("[Recorder SUCCESS] File Close");
 473                         fclose(g_pFile);
 474                         g_pFile = NULL;
 475                 }
 476 #endif
 477         }
 478
 479         MWR_LOGI("[EXIT]");
 480 }
 481
 482 void CWakeupEngineManager::start_streaming_current_utterance_data()
 483 {
 484         if (mStreamingThread.joinable()) {
 485                 MWR_LOGE("ERROR : mStreamingThread is joinable, will not start a new thread");
 486                 return;
 487         }
 488         mStreamingThread = thread(&CWakeupEngineManager::streaming_speech_data_thread_func, this);
 489 }
 490
 491 void CWakeupEngineManager::stop_streaming_current_utterance_data()
 492 {
 493         if (mStreamingThread.joinable()) {
 494                 MWR_LOGD("mStreamingThread is joinable, trying join()");
 495                 mStopStreamingThread.store(true);
 496                 mStreamingThread.join();
 497         }
 498         mStopStreamingThread.store(false);
 499 }
 500
 501 void CWakeupEngineManager::update_manager_state(wakeup_manager_state_e state)
 502 {
 503         for (const auto& info : mEngineInfo) {
 504                 if (info.interface.update_manager_state) {
 505                         try {
 506                                 info.interface.update_manager_state(state);
 507                         } catch (const std::exception& e) {
 508                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 509                                         info.engine_name.c_str(), e.what());
 510                         }
 511                 }
 512         }
 513         mWakeupManagerState = state;
 514 }
 515
 516 void CWakeupEngineManager::update_recognition_result(string appid, int result)
 517 {
 518         if (mSelectedEngine) {
 519                 if (mSelectedEngine->interface.update_recognition_result) {
 520                         mSelectedEngine->interface.update_recognition_result(appid.c_str(), result);
 521                 }
 522         }
 523 }
 524
 525 void CWakeupEngineManager::engine_add_target_assistant(string engine_name, string appid)
 526 {
 527         const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(),
 528                 [engine_name](const EngineInfo& info) {
 529                         return (0 == info.engine_name.compare(engine_name));
 530                 });
 531
 532         if (mEngineInfo.end() == iter) {
 533                 /* Not found, add new library */
 534                 pkgmgrinfo_appinfo_h handle;
 535                 int ret = pkgmgrinfo_appinfo_get_appinfo(engine_name.c_str(), &handle);
 536                 if (PMINFO_R_OK == ret) {
 537                         char *root_path = nullptr;
 538                         ret = pkgmgrinfo_appinfo_get_root_path(handle, &root_path);
 539                         if (PMINFO_R_OK == ret && nullptr != root_path) {
 540                                 string path = root_path;
 541                                 path += "/";
 542                                 path += MA_WAKEUP_DEDICATED_ENGINE_PATH;
 543                                 add_engine(engine_name, path);
 544                         }
 545                         pkgmgrinfo_appinfo_destroy_appinfo(handle);
 546                 }
 547                 /* Find again to add appid to the newly created engine's assistant list */
 548                 const auto &new_iter = find_if(mEngineInfo.begin(), mEngineInfo.end(),
 549                         [engine_name](const EngineInfo& info) {
 550                                 return (0 == info.engine_name.compare(engine_name));
 551                         });
 552                 if (mEngineInfo.end() != new_iter) {
 553                         new_iter->assistant_list.push_back(appid);
 554                 }
 555         } else {
 556                 /* If the engine already exists, simply add the appid to the assistant list */
 557                 iter->assistant_list.push_back(appid);
 558         }
 559 }
 560
 561 void CWakeupEngineManager::engine_add_wakeup_word(string appid, string wakeup_word, string language)
 562 {
 563         for (const auto& info : mEngineInfo) {
 564                 bool found = contains(info.assistant_list, appid);
 565                 if (found) {
 566                         if (info.interface.add_wakeup_word) {
 567                                 try {
 568                                         info.interface.add_wakeup_word(appid.c_str(), wakeup_word.c_str(), language.c_str());
 569                                 } catch (const std::exception& e) {
 570                                         MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 571                                                 info.engine_name.c_str(), e.what());
 572                                 }
 573                         } else {
 574                                 MWR_LOGE("Wakeup Engine does not provide add_wakeup_word");
 575                         }
 576                 }
 577         }
 578 }
 579
 580 void CWakeupEngineManager::engine_remove_wakeup_word(string appid, string wakeup_word, string language)
 581 {
 582         for (const auto& info : mEngineInfo) {
 583                 bool found = contains(info.assistant_list, appid);
 584                 if (found) {
 585                         if (info.interface.remove_wakeup_word) {
 586                                 try {
 587                                         info.interface.remove_wakeup_word(appid.c_str(), wakeup_word.c_str(), language.c_str());
 588                                 } catch (const std::exception& e) {
 589                                         MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 590                                                 info.engine_name.c_str(), e.what());
 591                                 }
 592                         } else {
 593                                 MWR_LOGE("Wakeup Engine does not provide remove_wakeup_word");
 594                         }
 595                 }
 596         }
 597 }
 598
 599 void CWakeupEngineManager::engine_set_assistant_specific_command(string appid, string command)
 600 {
 601         for (const auto& info : mEngineInfo) {
 602                 bool found = contains(info.assistant_list, appid);
 603                 if (found) {
 604                         if (info.interface.set_assistant_specific_command) {
 605                                 try {
 606                                         info.interface.set_assistant_specific_command(appid.c_str(), command.c_str());
 607                                 } catch (const std::exception& e) {
 608                                         MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 609                                                 info.engine_name.c_str(), e.what());
 610                                 }
 611                         }
 612                 }
 613         }
 614 }
 615
 616 void CWakeupEngineManager::engine_feed_audio_data(long time, void* data, int len)
 617 {
 618         for (const auto& info : mEngineInfo) {
 619                 if (info.activated &&
 620                         info.audio_data_require_status &&
 621                         info.interface.feed_audio_data) {
 622                         try {
 623                                 bool filtered_out = false;
 624                                 /* After a wakeup event, wakeup engines other than the selected one
 625                                    does not need to receive audio data. */
 626                                 if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 627                                         if (mSelectedEngine && &info != mSelectedEngine) {
 628                                                 filtered_out = true;
 629                                         }
 630                                 }
 631                                 if (!filtered_out) {
 632                                         int ret = info.interface.feed_audio_data(time, data, len);
 633                                         if (0 != ret) {
 634                                                 LOGE("[ERROR] Fail to feed speech data, ret(%d) : %s", ret, info.engine_name.c_str());
 635                                         }
 636                                 }
 637                         } catch (const std::exception& e) {
 638                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 639                                         info.engine_name.c_str(), e.what());
 640                         }
 641                 }
 642         }
 643 }
 644
 645 void CWakeupEngineManager::engine_set_dependency_module_command(string engine_name, string command)
 646 {
 647         const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(),
 648                 [engine_name](const EngineInfo& info) {
 649                         return (0 == info.engine_name.compare(engine_name));
 650                 });
 651
 652         if (mEngineInfo.end() != iter) {
 653                 if (iter->activated &&
 654                         iter->interface.set_dependency_module_command) {
 655                         try {
 656                                 int ret = iter->interface.set_dependency_module_command(command.c_str());
 657                                 if (0 != ret) {
 658                                         LOGE("[ERROR] Fail to set dependency module command, ret(%d) : %s",
 659                                                 ret, iter->engine_name.c_str());
 660                                 }
 661                         } catch (const std::exception& e) {
 662                                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 663                                         iter->engine_name.c_str(), e.what());
 664                         }
 665                 }
 666         }
 667 }
 668
 669 bool CWakeupEngineManager::on_wakeup_event(string engine_name, mas_wakeup_event_info info)
 670 {
 671         MWR_LOGD("[ENTER]");
 672
 673         for (const auto& observer : mObservers) {
 674                 if (observer) {
 675                         if (!observer->on_wakeup_event(engine_name, info)) {
 676                                 LOGE("[Recorder WARNING] One of the observer returned false");
 677                         }
 678                 }
 679         }
 680
 681         return true;
 682 }
 683
 684 bool CWakeupEngineManager::on_speech_status(string engine_name, mas_speech_status_e status)
 685 {
 686         MWR_LOGD("[ENTER]");
 687
 688         for (const auto& observer : mObservers) {
 689                 if (observer) {
 690                         if (!observer->on_speech_status(engine_name, status)) {
 691                                 LOGE("[Recorder WARNING] One of the observer returned false");
 692                         }
 693                 }
 694         }
 695
 696         return true;
 697 }
 698
 699 bool CWakeupEngineManager::on_error(string engine_name, int error_code, string error_message)
 700 {
 701         MWR_LOGD("[ENTER]");
 702
 703         for (const auto& observer : mObservers) {
 704                 if (observer) {
 705                         if (!observer->on_error(engine_name, error_code, error_message)) {
 706                                 LOGE("[Recorder WARNING] One of the observer returned false");
 707                         }
 708                 }
 709         }
 710
 711         return true;
 712 }
 713
 714 bool CWakeupEngineManager::on_audio_data_require_status(string engine_name, bool require)
 715 {
 716         MWR_LOGI("[ENTER] %s, %d", engine_name.c_str(), require);
 717
 718         bool found = false;
 719         // LOCK REQUIRED
 720         int count = 0;
 721         for (auto& info : mEngineInfo) {
 722                 if (info.engine_name.compare(engine_name) == 0) {
 723                         found = true;
 724                         info.audio_data_require_status = require;
 725                 }
 726                 if (info.activated && info.audio_data_require_status) {
 727                         count++;
 728                 }
 729         }
 730         MWR_LOGD("count : %d", count);
 731         if (count > 0) {
 732                 mAudioDataRequired = true;
 733         } else {
 734                 mAudioDataRequired = false;
 735         }
 736
 737         if (found) {
 738                 for (const auto& observer : mObservers) {
 739                         if (observer) {
 740                                 if (!observer->on_audio_data_require_status(engine_name, require)) {
 741                                         LOGE("[Recorder WARNING] One of the observer returned false");
 742                                 }
 743                         }
 744                 }
 745         }
 746         // UNLOCK REQUIRED
 747         return true;
 748 }
 749
 750 bool CWakeupEngineManager::on_wakeup_engine_command(string engine_name, mas_wakeup_engine_command_target_e target, string assistant_name, string command)
 751 {
 752         MWR_LOGD("[ENTER]");
 753
 754         for (const auto& observer : mObservers) {
 755                 if (observer) {
 756                         if (MAS_WAKEUP_ENGINE_COMMAND_TARGET_DEPENDENCY_MODULE == target) {
 757                                 if (!observer->on_wakeup_engine_command(target, engine_name, assistant_name, command)) {
 758                                         LOGE("[Recorder WARNING] One of the observer returned false");
 759                                 }
 760                         } else {
 761                                 const auto& iter = find_if(mEngineInfo.begin(), mEngineInfo.end(),
 762                                         [engine_name](const EngineInfo& info) {
 763                                                 return (0 == info.engine_name.compare(engine_name));
 764                                         });
 765                                 if (mEngineInfo.end() != iter) {
 766                                         for (const auto& assistant : iter->assistant_list) {
 767                                                 if (0 == assistant_name.compare(assistant) ||
 768                                                         MAS_WAKEUP_ENGINE_COMMAND_TARGET_ALL_ASSISTANTS == target) {
 769                                                         if (!observer->on_wakeup_engine_command(target, engine_name, assistant, command)) {
 770                                                                 LOGE("[Recorder WARNING] One of the observer returned false");
 771                                                         }
 772                                                 }
 773                                         }
 774                                 }
 775                         }
 776                 }
 777         }
 778
 779         return true;
 780 }
 781
 782 void CWakeupEngineManager::add_engine(string name, string path)
 783 {
 784         MWR_LOGD("Name (%s), Filepath(%s)", name.c_str(), path.c_str());
 785
 786         char* error = NULL;
 787         EngineInfo info;
 788         info.engine_handle = dlopen(path.c_str(), RTLD_LAZY);
 789         if (nullptr != (error = dlerror()) || nullptr == info.engine_handle) {
 790                 MWR_LOGD("[ERROR] Fail to dlopen(%s), error(%s)", path.c_str(), error);
 791                 if (info.engine_handle) dlclose(info.engine_handle);
 792                 return;
 793         }
 794
 795         /* Interfaces without version information */
 796         info.interface.initialize =
 797                 (wakeup_engine_initialize)dlsym(info.engine_handle,
 798                 MA_WAKEUP_ENGINE_FUNC_INITIALIZE);
 799         info.interface.deinitialize =
 800                 (wakeup_engine_deinitialize)dlsym(info.engine_handle,
 801                 MA_WAKEUP_ENGINE_FUNC_DEINITIALIZE);
 802         info.interface.activate =
 803                 (wakeup_engine_activate)dlsym(info.engine_handle,
 804                 MA_WAKEUP_ENGINE_FUNC_ACTIVATE);
 805         info.interface.deactivate =
 806                 (wakeup_engine_deactivate)dlsym(info.engine_handle,
 807                 MA_WAKEUP_ENGINE_FUNC_DEACTIVATE);
 808         info.interface.add_wakeup_word =
 809                 (wakeup_engine_add_wakeup_word)dlsym(info.engine_handle,
 810                 MA_WAKEUP_ENGINE_FUNC_ADD_WAKEUP_WORD);
 811         info.interface.remove_wakeup_word =
 812                 (wakeup_engine_remove_wakeup_word)dlsym(info.engine_handle,
 813                 MA_WAKEUP_ENGINE_FUNC_REMOVE_WAKEUP_WORD);
 814         info.interface.add_language =
 815                 (wakeup_engine_add_language)dlsym(info.engine_handle,
 816                 MA_WAKEUP_ENGINE_FUNC_ADD_LANGUAGE);
 817         info.interface.set_language =
 818                 (wakeup_engine_set_language)dlsym(info.engine_handle,
 819                 MA_WAKEUP_ENGINE_FUNC_SET_LANGUAGE);
 820         info.interface.update_manager_state =
 821                 (wakeup_engine_update_manager_state)dlsym(info.engine_handle,
 822                 MA_WAKEUP_ENGINE_FUNC_UPDATE_MANAGER_STATE);
 823         info.interface.update_recognition_result =
 824                 (wakeup_engine_update_recognition_result)dlsym(info.engine_handle,
 825                 MA_WAKEUP_ENGINE_FUNC_UPDATE_RECOGNITION_RESULT);
 826         info.interface.set_audio_format =
 827                 (wakeup_engine_set_audio_format)dlsym(info.engine_handle,
 828                 MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_FORMAT);
 829         info.interface.get_audio_format =
 830                 (wakeup_engine_get_audio_format)dlsym(info.engine_handle,
 831                 MA_WAKEUP_ENGINE_FUNC_GET_AUDIO_FORMAT);
 832         info.interface.feed_audio_data =
 833                 (wakeup_engine_feed_audio_data)dlsym(info.engine_handle,
 834                 MA_WAKEUP_ENGINE_FUNC_FEED_AUDIO_DATA);
 835         info.interface.get_utterance_data_count =
 836                 (wakeup_engine_get_utterance_data_count)dlsym(info.engine_handle,
 837                 MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA_COUNT);
 838         info.interface.get_utterance_data =
 839                 (wakeup_engine_get_utterance_data)dlsym(info.engine_handle,
 840                 MA_WAKEUP_ENGINE_FUNC_GET_UTTERANCE_DATA);
 841         info.interface.get_wake_word_data_count =
 842                 (wakeup_engine_get_wake_word_data_count)dlsym(info.engine_handle,
 843                 MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA_COUNT);
 844         info.interface.get_wake_word_data =
 845                 (wakeup_engine_get_wake_word_data)dlsym(info.engine_handle,
 846                 MA_WAKEUP_ENGINE_FUNC_GET_WAKE_WORD_DATA);
 847         info.interface.set_assistant_specific_command =
 848                 (wakeup_engine_set_assistant_specific_command)dlsym(info.engine_handle,
 849                 MA_WAKEUP_ENGINE_FUNC_SET_ASSISTANT_SPECIFIC_COMMAND);
 850         info.interface.set_wake_word_audio_require_flag =
 851                 (wakeup_engine_set_wake_word_audio_require_flag)dlsym(info.engine_handle,
 852                 MA_WAKEUP_ENGINE_FUNC_SET_WAKE_WORD_AUDIO_REQUIRE_FLAG);
 853         info.interface.set_wakeup_event_callback =
 854                 (wakeup_engine_set_wakeup_event_callback)dlsym(info.engine_handle,
 855                 MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_EVENT_CALLBACK);
 856         info.interface.set_speech_status_callback =
 857                 (wakeup_engine_set_speech_status_callback)dlsym(info.engine_handle,
 858                 MA_WAKEUP_ENGINE_FUNC_SET_SPEECH_STATUS_CALLBACK);
 859         info.interface.set_error_callback =
 860                 (wakeup_engine_set_error_callback)dlsym(info.engine_handle,
 861                 MA_WAKEUP_ENGINE_FUNC_SET_ERROR_CALLBACK);
 862         info.interface.set_audio_data_require_status_callback =
 863                 (wakeup_engine_set_audio_data_require_status_callback)dlsym(info.engine_handle,
 864                 MA_WAKEUP_ENGINE_FUNC_SET_AUDIO_DATA_REQUIRE_STATUS_CALLBACK);
 865         info.interface.set_wakeup_engine_command_callback =
 866                 (wakeup_engine_set_wakeup_engine_command_callback)dlsym(info.engine_handle,
 867                 MA_WAKEUP_ENGINE_FUNC_SET_WAKEUP_ENGINE_COMMAND_CALLBACK);
 868
 869         /* Interfaces after version 1 */
 870         info.interface.get_version =
 871                 (wakeup_engine_get_version)dlsym(info.engine_handle,
 872                 MA_WAKEUP_ENGINE_FUNC_GET_VERSION);
 873         info.interface.set_dependency_module_command =
 874                 (wakeup_engine_set_dependency_module_command)dlsym(info.engine_handle,
 875                 MA_WAKEUP_ENGINE_FUNC_SET_DEPENDENCY_MODULE_COMMAND);
 876
 877         info.version = 0;
 878         info.engine_path = path;
 879         info.engine_name = name;
 880
 881         info.activated = false;
 882         info.audio_data_require_status = false;
 883
 884         /* All the necessary information has already been set properly */
 885         mEngineInfo.push_back(info);
 886
 887         MWR_LOGD("Initializing wakeup engine : %s %p",
 888                 info.engine_path.c_str(),
 889                 info.interface.initialize);
 890
 891         /* Workaround for registering C-style callbacks */
 892         typedef struct {
 893                 CWakeupEngineManager *manager;
 894                 string engine_name;
 895         } CallbackUserData;
 896
 897         static deque<CallbackUserData> callback_user_data;
 898
 899         CallbackUserData user_data;
 900         user_data.manager = this;
 901         user_data.engine_name = info.engine_name;
 902         callback_user_data.push_back(user_data);
 903
 904         try {
 905                 if (info.interface.set_wakeup_event_callback) {
 906                         info.interface.set_wakeup_event_callback(
 907                                 [](mas_wakeup_event_info info, void* user_data) {
 908                                         CallbackUserData *data = static_cast<CallbackUserData*>(user_data);
 909                                         if (nullptr == data) return;
 910                                         if (nullptr == data->manager) return;
 911                                         info.wakeup_engine = data->engine_name.c_str();
 912                                         data->manager->on_wakeup_event(data->engine_name, info);
 913                                 }, &(callback_user_data.back()));
 914                 }
 915
 916                 if (info.interface.set_audio_data_require_status_callback) {
 917                         info.interface.set_audio_data_require_status_callback(
 918                                 [](bool require, void* user_data) {
 919                                         CallbackUserData *data = static_cast<CallbackUserData*>(user_data);
 920                                         if (nullptr == data) return;
 921                                         if (nullptr == data->manager) return;
 922                                         data->manager->on_audio_data_require_status(data->engine_name, require);
 923                                 }, &(callback_user_data.back()));
 924                 }
 925
 926                 if (info.interface.set_wakeup_engine_command_callback) {
 927                         info.interface.set_wakeup_engine_command_callback(
 928                                 [](mas_wakeup_engine_command_target_e target,
 929                                         const char* assistant_name, const char* command, void* user_data) {
 930                                         CallbackUserData* data = static_cast<CallbackUserData*>(user_data);
 931                                         if (nullptr == data) return;
 932                                         if (nullptr == data->manager) return;
 933                                         if (nullptr == command) return;
 934                                         data->manager->on_wakeup_engine_command(
 935                                                 data->engine_name, target, (assistant_name ? assistant_name : ""), command);
 936                                 }, &(callback_user_data.back()));
 937                 }
 938
 939                 if (info.interface.initialize) {
 940                         info.interface.initialize();
 941                 }
 942                 if (info.interface.get_version) {
 943                         int version;
 944                         if (0 == info.interface.get_version(&version)) {
 945                                 info.version = version;
 946                         }
 947                 }
 948         } catch (const std::exception& e) {
 949                 MWR_LOGE("[ERROR] wakeup engine %s threw exception : %s",
 950                         info.engine_name.c_str(), e.what());
 951         }
 952 }
 953
 954 void CWakeupEngineManager::add_engine_directory(string name, string path)
 955 {
 956         if (0 == path.size()) return;
 957
 958         DIR* dp = opendir(path.c_str());
 959         if (NULL == dp) {
 960                 MWR_LOGD("Failed opening directory : %s", path.c_str());
 961         } else {
 962                 struct dirent *dirp = NULL;
 963                 string filepath;
 964                 do {
 965                         dirp = readdir(dp);
 966
 967                         if (NULL != dirp) {
 968                                 if (!strcmp(".", dirp->d_name) || !strcmp("..", dirp->d_name))
 969                                         continue;
 970
 971                                 if (DT_REG != dirp->d_type) /* If not a regular file */
 972                                         continue;
 973
 974                                 filepath = path;
 975                                 filepath += "/";
 976                                 filepath += dirp->d_name;
 977
 978                                 if (filepath.length() >= _POSIX_PATH_MAX) {
 979                                         MWR_LOGD("File path is too long : %s", filepath.c_str());
 980                                         closedir(dp);
 981                                         return;
 982                                 }
 983                                 add_engine(name, filepath);
 984                         }
 985                 } while (NULL != dirp);
 986
 987                 closedir(dp);
 988         }
 989 }
 990
 991 } // wakeup
 992 } // multiassistant