plugins/wakeup-manager/src/wakeup_manager.cpp

   1 /*
   2  * Copyright 2018-2019 Samsung Electronics Co., Ltd
   3  *
   4  * Licensed under the Flora License, Version 1.1 (the License);
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://floralicense.org/license/
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an AS IS BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include "wakeup_manager.h"
  18 #include "wakeup_manager_main.h"
  19 #include "wakeup_policy_default.h"
  20 #include "dependency_resolver.h"
  21
  22 #include <algorithm>
  23 namespace multiassistant
  24 {
  25 namespace wakeup
  26 {
  27
  28 static bool check_language_valid(string language)
  29 {
  30         return true;
  31 }
  32
  33 static bool initialize_wakeup_event_info(mas_wakeup_event_info* wakeup_info)
  34 {
  35         bool ret = false;
  36         if (wakeup_info) {
  37                 ret = true;
  38
  39                 wakeup_info->wakeup_appid = nullptr;
  40                 wakeup_info->wakeup_word = nullptr;
  41                 wakeup_info->wakeup_language = nullptr;
  42                 wakeup_info->wakeup_voice_id = nullptr;
  43                 wakeup_info->wakeup_engine = nullptr;
  44                 wakeup_info->wakeup_confidence_score = 0.0f;
  45
  46                 wakeup_info->wakeup_start_time = 0;
  47                 wakeup_info->wakeup_end_time = 0L;
  48                 wakeup_info->wakeup_time_valid = false;
  49
  50                 wakeup_info->extra_data = nullptr;
  51                 wakeup_info->extra_data_length = 0;
  52                 wakeup_info->extra_data_description = nullptr;
  53         }
  54         return ret;
  55 }
  56
  57 CWakeupManager::CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingValueObserver* setting_observer)
  58 {
  59         initialize_wakeup_event_info(&mLastWakeupEventInfo);
  60
  61         if (wakeup_observer) {
  62                 subscribe_wakeup_observer(wakeup_observer);
  63         }
  64         if (setting_observer) {
  65                 subscribe_setting_observer(setting_observer);
  66         }
  67 }
  68
  69 CWakeupManager::~CWakeupManager()
  70 {
  71         MWR_LOGI("Wakeup Manager is now being destroyed");
  72 }
  73
  74 void CWakeupManager::initialize_wakeup_policy()
  75 {
  76         mWakeupPolicy.reset(new CWakeupPolicyDefault{&mPolicyEventObserver});
  77
  78         /* Default Policy specific initialization */
  79         CWakeupPolicyDefault *policy =
  80                 dynamic_cast<CWakeupPolicyDefault*>(mWakeupPolicy.get());
  81         if (policy) {
  82                 int priority = 0;
  83
  84                 policy->set_delay(mWakeupSettings.get_wakeup_policy_delay());
  85                 MWR_LOGD("Setting Delay : %f", mWakeupSettings.get_wakeup_policy_delay());
  86                 for (const auto& assistant : mWakeupSettings.get_wakeup_policy_priority()) {
  87                         policy->set_assistant_priority(assistant, ++priority);
  88                         MWR_LOGD("Setting Priority : %d %s", priority, assistant.c_str());
  89                 }
  90         }
  91 }
  92
  93 bool CWakeupManager::initialize()
  94 {
  95         MWR_LOGD("[ENTER]");
  96
  97         mPolicyEventObserver.set_wakeup_manager(this);
  98         mEngineEventObserver.set_wakeup_manager(this);
  99         mAudioEventObserver.set_wakeup_manager(this);
 100         mSettingsEventObserver.set_wakeup_manager(this);
 101
 102         mWakeupSettings.subscribe(&mSettingsEventObserver);
 103         mWakeupSettings.initialize();
 104
 105         mAudioEventObserver.set_wakeup_engine_manager(&mWakeupEngineManager);
 106         mAudioManager.subscribe(&mAudioEventObserver);
 107         mAudioManager.initialize();
 108
 109         initialize_wakeup_policy();
 110
 111         mWakeupEngineManager.subscribe(&mEngineEventObserver);
 112         mWakeupEngineManager.initialize();
 113
 114         mas_dependency_plugin_proxy_interface interface;
 115         interface.process_event = wakeup_manager_process_event;
 116         interface.feed_audio_data = wakeup_manager_feed_audio_data;
 117         //interface.send_dependency_module_command = wakeup_manager_send_dependency_module_command;
 118
 119         dependency_resolver_initialize(interface);
 120
 121         MWR_LOGD("[END]");
 122         return true;
 123 }
 124
 125 bool CWakeupManager::deinitialize()
 126 {
 127         MWR_LOGI("[ENTER]");
 128
 129         dependency_resolver_deinitialize();
 130
 131         mWakeupEngineManager.unsubscribe(&mEngineEventObserver);
 132         mWakeupEngineManager.deinitialize();
 133
 134         mAudioManager.unsubscribe(&mAudioEventObserver);
 135         mAudioManager.deinitialize();
 136
 137         mWakeupSettings.deinitialize();
 138         mAssistantLanguageInfo.clear();
 139
 140         MWR_LOGI("[END]");
 141         return true;
 142 }
 143
 144 void CWakeupManager::subscribe_wakeup_observer(IWakeupEventObserver *observer)
 145 {
 146         mWakeupObservers.push_back(observer);
 147 }
 148
 149 void CWakeupManager::unsubscribe_wakeup_observer(IWakeupEventObserver *observer)
 150 {
 151         auto iter = find(mWakeupObservers.begin(), mWakeupObservers.end(), observer);
 152         if (iter != mWakeupObservers.end()) {
 153                 mWakeupObservers.erase(iter);
 154         }
 155 }
 156
 157 void CWakeupManager::subscribe_setting_observer(ISettingValueObserver* observer)
 158 {
 159         mSettingObservers.push_back(observer);
 160 }
 161
 162 void CWakeupManager::unsubscribe_setting_observer(ISettingValueObserver* observer)
 163 {
 164         auto iter = find(mSettingObservers.begin(), mSettingObservers.end(), observer);
 165         if (iter != mSettingObservers.end()) {
 166                 mSettingObservers.erase(iter);
 167         }
 168 }
 169
 170 bool CWakeupManager::activate(void)
 171 {
 172         MWR_LOGD("[ENTER]");
 173
 174         if (WAKEUP_MANAGER_STATE_INACTIVE != mWakeupManagerState)
 175                 return false;
 176
 177         /* Activate assistants that supports current voice input language */
 178         set_language(mWakeupSettings.get_current_language());
 179
 180         change_manager_state(WAKEUP_MANAGER_STATE_LISTENING);
 181         if (mWakeupEngineManager.get_audio_data_required()) {
 182                 mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 183                 mAudioManager.start_recording(true);
 184         }
 185
 186         MWR_LOGD("[END]");
 187         return true;
 188 }
 189
 190 bool CWakeupManager::deactivate(void)
 191 {
 192         MWR_LOGD("[ENTER]");
 193
 194         if (WAKEUP_MANAGER_STATE_INACTIVE == mWakeupManagerState)
 195                 return false;
 196
 197         mAudioManager.stop_recording(true);
 198         change_manager_state(WAKEUP_MANAGER_STATE_INACTIVE);
 199
 200         stop_streaming_utterance_data();
 201         stop_streaming_previous_utterance_data();
 202         stop_streaming_follow_up_data();
 203
 204         MWR_LOGD("[END]");
 205         return true;
 206 }
 207
 208 bool CWakeupManager::add_assistant_language(string appid, string language)
 209 {
 210         MWR_LOGD("[ENTER]");
 211         bool found = false;
 212         for (auto& info : mAssistantLanguageInfo) {
 213                 if(0 == info.appid.compare(appid)) {
 214                         info.languageList.push_back(language);
 215                         found = true;
 216                 }
 217         }
 218         if(false == found) {
 219                 AssistantLanguageInfo info;
 220                 info.appid = appid;
 221                 info.languageList.push_back(language);
 222                 mAssistantLanguageInfo.push_back(info);
 223         }
 224         if (0 == mCurrentLanguage.compare(language)) {
 225                 mAssistantSupportsCurrentLanguage[appid] = true;
 226                 if (true == mAssistantEnabled[appid]) {
 227                         mWakeupEngineManager.set_assistant_activated(appid, true);
 228                 }
 229         }
 230         MWR_LOGD("[END]");
 231         return true;
 232 }
 233
 234 bool CWakeupManager::add_assistant_wakeup_word(string appid, string wakeup_word, string language)
 235 {
 236         MWR_LOGD("[ENTER]");
 237
 238         mWakeupEngineManager.engine_add_wakeup_word(appid, wakeup_word, language);
 239
 240         MWR_LOGD("[END]");
 241         return true;
 242 }
 243
 244 bool CWakeupManager::set_assistant_wakeup_engine(string appid, string engine)
 245 {
 246         MWR_LOGD("[ENTER]");
 247
 248         mWakeupEngineManager.engine_add_target_assistant(engine, appid);
 249
 250         MWR_LOGD("[END]");
 251         return true;
 252 }
 253
 254 bool CWakeupManager::set_assistant_language(string appid, string language)
 255 {
 256         MWR_LOGD("[ENTER] : %s, %s", appid.c_str(), language.c_str());
 257
 258         mWakeupEngineManager.set_assistant_language(appid, language);
 259
 260         MWR_LOGD("[END]");
 261         return true;
 262 }
 263
 264 bool CWakeupManager::set_assistant_enabled(string appid, bool enabled)
 265 {
 266         MWR_LOGD("[ENTER]");
 267
 268         mAssistantEnabled[appid] = enabled;
 269         bool activated = enabled;
 270         if (false == mAssistantSupportsCurrentLanguage[appid]) {
 271                 activated = false;
 272         }
 273         if (0 == appid.compare(mCurrentDefaultAssistant)) {
 274                 activated = true;
 275         }
 276         mWakeupEngineManager.set_assistant_activated(appid, activated);
 277
 278         MWR_LOGD("[END]");
 279         return true;
 280 }
 281
 282 bool CWakeupManager::set_default_assistant(string appid)
 283 {
 284         MWR_LOGD("[ENTER]");
 285
 286         /* Check if previous default assistant has to be deactivated */
 287         bool activated = true;
 288         if (false == mAssistantSupportsCurrentLanguage[mCurrentDefaultAssistant]) {
 289                 activated = false;
 290         }
 291         if (false == mAssistantEnabled[mCurrentDefaultAssistant]) {
 292                 activated = false;
 293         }
 294         mWakeupEngineManager.set_assistant_activated(mCurrentDefaultAssistant, activated);
 295
 296         /* New default assistant has to be activated no matter what */
 297         mWakeupEngineManager.set_assistant_activated(appid, true);
 298         mCurrentDefaultAssistant = appid;
 299
 300         MWR_LOGD("[END]");
 301         return true;
 302 }
 303
 304 string CWakeupManager::get_default_assistant()
 305 {
 306         return mCurrentDefaultAssistant;
 307 }
 308
 309 bool CWakeupManager::get_assistant_enabled(string appid)
 310 {
 311         return mAssistantEnabled[appid];
 312 }
 313
 314 bool CWakeupManager::set_language(string language)
 315 {
 316         bool ret = false;
 317         MWR_LOGD("[ENTER] : %s", language.c_str());
 318
 319         if (check_language_valid(language)) {
 320                 mCurrentLanguage = language;
 321
 322                 bool found = false;
 323                 for (auto& info : mAssistantLanguageInfo) {
 324                         found = false;
 325                         for(auto it = info.languageList.begin(); it != info.languageList.end(); it++) {
 326                                 if(language == *it) {
 327                                         found = true;
 328                                         break;
 329                                 }
 330                         }
 331                         if(false == found) {
 332                                 mAssistantSupportsCurrentLanguage[info.appid] = false;
 333                         } else {
 334                                 mAssistantSupportsCurrentLanguage[info.appid] = true;
 335                         }
 336
 337                         bool activated = found;
 338                         if (false == mAssistantEnabled[info.appid]) {
 339                                 activated = false;
 340                         }
 341                         if (0 == info.appid.compare(mCurrentDefaultAssistant)) {
 342                                 activated = true;
 343                         }
 344                         mWakeupEngineManager.set_assistant_activated(info.appid, activated);
 345                 }
 346
 347                 mWakeupEngineManager.set_language(language);
 348                 ret = true;
 349         } else {
 350                 MWR_LOGE("[ERROR] Not supported language (%s)", language.c_str());
 351         }
 352
 353         MWR_LOGD("[END]");
 354         return ret;
 355 }
 356
 357 bool CWakeupManager::get_voice_key_pressed()
 358 {
 359         return mVoiceKeyPressed;
 360 }
 361
 362 STREAMING_MODE CWakeupManager::get_streaming_mode()
 363 {
 364         return mStreamingMode;
 365 }
 366
 367 bool CWakeupManager::set_streaming_mode(STREAMING_MODE mode)
 368 {
 369         mStreamingMode = mode;
 370         return true;
 371 }
 372
 373 bool CWakeupManager::change_manager_state(wakeup_manager_state_e state)
 374 {
 375         mWakeupManagerState = state;
 376         mWakeupEngineManager.update_manager_state(state);
 377         return true;
 378 }
 379
 380 wakeup_manager_state_e CWakeupManager::get_manager_state()
 381 {
 382         return mWakeupManagerState;
 383 }
 384
 385 bool CWakeupManager::update_voice_feedback_state(string appid, bool state)
 386 {
 387         MWR_LOGD("[ENTER]");
 388
 389         if (state) {
 390                 if (WAKEUP_MANAGER_STATE_LISTENING == mWakeupManagerState ||
 391                         WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState) {
 392                         change_manager_state(WAKEUP_MANAGER_STATE_VOICE_FEEDBACK);
 393                 }
 394         } else {
 395                 if (WAKEUP_MANAGER_STATE_VOICE_FEEDBACK == mWakeupManagerState) {
 396                         change_manager_state(WAKEUP_MANAGER_STATE_LISTENING);
 397                 }
 398         }
 399
 400         MWR_LOGD("[END]");
 401         return true;
 402 }
 403
 404 bool CWakeupManager::send_assistant_specific_command(string appid, string command)
 405 {
 406         MWR_LOGD("[ENTER]");
 407
 408         /*
 409         static const string voice_key_pressed{"voice_key_pressed"};
 410         static const string voice_key_released{"voice_key_released"};
 411
 412         if (0 == command.compare(voice_key_pressed)) {
 413                 process_event(MA_PLUGIN_EVENT_VOICE_KEY_PRESSED, NULL, 0);
 414         } else if (0 == command.compare(voice_key_released)) {
 415                 process_event(MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, NULL, 0);
 416         }
 417         */
 418
 419         mWakeupEngineManager.engine_set_assistant_specific_command(appid, command);
 420
 421         MWR_LOGD("[END]");
 422         return true;
 423 }
 424
 425 bool CWakeupManager::set_background_volume(string appid, double ratio)
 426 {
 427         MWR_LOGD("[DEBUG] set background volume (%f)", ratio);
 428
 429         int ret = 0;
 430         mAudioManager.set_background_volume(ratio);
 431         return ret;
 432 }
 433
 434 bool CWakeupManager::update_recognition_result(string appid, int result)
 435 {
 436         MWR_LOGD("[ENTER]");
 437         if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 438                 mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 439         }
 440         mWakeupEngineManager.update_recognition_result(appid, result);
 441         if (WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState ||
 442                 WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 443                 change_manager_state(WAKEUP_MANAGER_STATE_LISTENING);
 444                 stop_streaming_utterance_data();
 445                 stop_streaming_previous_utterance_data();
 446                 stop_streaming_follow_up_data();
 447         }
 448         MWR_LOGD("[END]");
 449         return true;
 450 }
 451
 452 static long get_current_milliseconds_after_epoch()
 453 {
 454         auto now = chrono::system_clock::now();
 455         auto now_ms = chrono::time_point_cast<chrono::milliseconds>(now);
 456         /* number of milliseconds since the epoch of system_clock */
 457         auto value = now_ms.time_since_epoch();
 458
 459         return value.count();
 460 }
 461
 462 bool CWakeupManager::process_event(mas_plugin_event_e event, void* data, int len)
 463 {
 464         MWR_LOGD("[ENTER] : %d", event);
 465         if (WAKEUP_MANAGER_STATE_INACTIVE == mWakeupManagerState)
 466                 return false;
 467
 468         // LOCK REQUIRED
 469         if (MAS_PLUGIN_EVENT_VOICE_KEY_PRESSED == event) {
 470                 if (mVoiceKeyPressed != true) {
 471                         mAudioManager.stop_recording(true);
 472
 473                         stop_streaming_utterance_data();
 474                         stop_streaming_previous_utterance_data();
 475                         stop_streaming_follow_up_data();
 476
 477                         mAudioManager.clear_audio_data();
 478                         change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE);
 479
 480                         mVoiceKeyPressed = true;
 481
 482                         /* Start recorder thread using appropriate recording device */
 483                         mAudioManager.set_recording_session(RECORDING_SESSION_UTTERANCE);
 484                         mAudioManager.start_recording(true);
 485
 486                         /* Wakeup default assistant */
 487                         /* TODO: apply conversation timeout for selecting assistant here */
 488                         mas_wakeup_event_info wakeup_info;
 489                         initialize_wakeup_event_info(&wakeup_info);
 490                         /* Make sure to use background data */
 491                         wakeup_info.wakeup_time_valid = true;
 492                         wakeup_info.wakeup_end_time = get_current_milliseconds_after_epoch();
 493                         wakeup_info.wakeup_engine = WAKEUP_ENGINE_VOICE_KEY;
 494
 495                         string appid = mWakeupSettings.get_default_assistant_appid();
 496                         wakeup_info.wakeup_appid = appid.c_str();
 497                         MWR_LOGD("wakeup_appid : %s", wakeup_info.wakeup_appid);
 498
 499                         set_last_wakeup_event_info(wakeup_info);
 500                         mWakeupEngineManager.set_selected_wakeup_info(wakeup_info);
 501                         for (const auto& observer : mWakeupObservers) {
 502                                 observer->on_wakeup(wakeup_info);
 503                         }
 504                 }
 505         } else if (MAS_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_PUSH == event) {
 506                 if (mVoiceKeyPressed != false) {
 507                         mAudioManager.finalize_audio_data();
 508                         mVoiceKeyPressed = false;
 509
 510                         if (STREAMING_MODE::UTTERANCE == mStreamingMode) {
 511                                 change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 512                         } else {
 513                                 change_manager_state(WAKEUP_MANAGER_STATE_LISTENING);
 514                         }
 515
 516                         if (mWakeupEngineManager.get_audio_data_required()) {
 517                                 /* Restart recorder thread using appropriate recording device */
 518                                 mAudioManager.stop_recording(true);
 519                                 mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 520                                 mAudioManager.start_recording(true);
 521                         } else {
 522                                 mAudioManager.stop_recording(true);
 523                         }
 524                 }
 525         } else if (MAS_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_TAP == event) {
 526                 if (mVoiceKeyPressed != false) {
 527                         mVoiceKeyPressed = false;
 528                 }
 529         }
 530         // UNLOCK REQUIRED
 531
 532         MWR_LOGD("[END]");
 533         return true;
 534 }
 535
 536 vector<IWakeupEventObserver*> CWakeupManager::get_wakeup_observers()
 537 {
 538         return mWakeupObservers;
 539 }
 540
 541 void CWakeupManager::set_last_wakeup_event_info(mas_wakeup_event_info wakeup_info)
 542 {
 543         mLastWakeupEventInfo = wakeup_info;
 544 }
 545
 546 vector<ISettingValueObserver*> CWakeupManager::get_setting_observers()
 547 {
 548         return mSettingObservers;
 549 }
 550
 551 static Eina_Bool streaming_duration_expired(void *data)
 552 {
 553         MWR_LOGD("[ENTER]");
 554         CWakeupManager *wakeup_manager = static_cast<CWakeupManager*>(data);
 555         if (nullptr == wakeup_manager) return ECORE_CALLBACK_CANCEL;
 556
 557         CAudioManager *audio_manager = wakeup_manager->get_audio_manager();
 558         CWakeupEngineManager *engine_manager = wakeup_manager->get_engine_manager();
 559
 560         if (nullptr == audio_manager) return ECORE_CALLBACK_CANCEL;
 561         if (nullptr == engine_manager) return ECORE_CALLBACK_CANCEL;
 562
 563         switch(wakeup_manager->get_streaming_mode()) {
 564                 case STREAMING_MODE::UTTERANCE:
 565                         audio_manager->stop_streaming_current_utterance_data();
 566                         engine_manager->stop_streaming_current_utterance_data();
 567                 break;
 568                 case STREAMING_MODE::PREVIOUS_UTTERANCE:
 569                         audio_manager->stop_streaming_previous_utterance_data();
 570                 break;
 571                 case STREAMING_MODE::FOLLOW_UP:
 572                         audio_manager->stop_streaming_follow_up_data();
 573                         audio_manager->stop_recording(true);
 574                         audio_manager->clear_audio_data();
 575                         audio_manager->set_recording_session(RECORDING_SESSION_WAKE_WORD);
 576                         audio_manager->start_recording(true);
 577                 break;
 578         }
 579
 580         unsigned char final_buffer[2] = {'\0', };
 581         vector<IWakeupEventObserver*> observers = wakeup_manager->get_wakeup_observers();
 582         for (const auto& observer : observers) {
 583                 observer->on_streaming_audio_data(
 584                         MAS_SPEECH_STREAMING_EVENT_FINISH, final_buffer, sizeof(final_buffer));
 585         }
 586         wakeup_manager->set_streaming_mode(STREAMING_MODE::NONE);
 587
 588         if (WAKEUP_MANAGER_STATE_UTTERANCE == wakeup_manager->get_manager_state()) {
 589                 wakeup_manager->change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 590         }
 591
 592         return ECORE_CALLBACK_CANCEL;
 593 }
 594
 595 bool CWakeupManager::start_streaming_utterance_data()
 596 {
 597         MWR_LOGD("[ENTER]");
 598
 599         mAudioManager.stop_streaming_current_utterance_data();
 600         mWakeupEngineManager.stop_streaming_current_utterance_data();
 601
 602         mStreamingMode = STREAMING_MODE::UTTERANCE;
 603
 604         bool streaming_by_manager = true;
 605         if (false == mLastWakeupEventInfo.wakeup_time_valid) {
 606                 mWakeupEngineManager.start_streaming_current_utterance_data();
 607                 streaming_by_manager = false;
 608         } else {
 609                 mAudioManager.start_streaming_current_utterance_data(mLastWakeupEventInfo.wakeup_end_time);
 610         }
 611
 612         ecore_thread_main_loop_begin();
 613         if (mStreamingDurationTimer) {
 614                 ecore_timer_del(mStreamingDurationTimer);
 615                 mStreamingDurationTimer = nullptr;
 616         }
 617         if (streaming_by_manager) {
 618                 mStreamingDurationTimer = ecore_timer_add(
 619                         mWakeupSettings.get_streaming_duration_max(),
 620                         streaming_duration_expired, this);
 621         }
 622         ecore_thread_main_loop_end();
 623
 624         MWR_LOGD("[END]");
 625         return true;
 626 }
 627
 628 bool CWakeupManager::stop_streaming_utterance_data()
 629 {
 630         MWR_LOGD("[ENTER]");
 631         if (STREAMING_MODE::UTTERANCE != mStreamingMode) return false;
 632
 633         if (mStreamingDurationTimer) {
 634                 ecore_thread_main_loop_begin();
 635                 ecore_timer_del(mStreamingDurationTimer);
 636                 mStreamingDurationTimer = nullptr;
 637                 ecore_thread_main_loop_end();
 638         }
 639         if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 640                 change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 641                 mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 642         }
 643         mAudioManager.stop_streaming_current_utterance_data();
 644         mWakeupEngineManager.stop_streaming_current_utterance_data();
 645
 646         mStreamingMode = STREAMING_MODE::NONE;
 647
 648         MWR_LOGD("[END]");
 649         return true;
 650 }
 651
 652 bool CWakeupManager::start_streaming_follow_up_data()
 653 {
 654         MWR_LOGD("[ENTER]");
 655
 656         mAudioManager.stop_streaming_follow_up_data();
 657         mWakeupEngineManager.stop_streaming_current_utterance_data();
 658
 659         mStreamingMode = STREAMING_MODE::FOLLOW_UP;
 660
 661         /* For the follow up streaming, audio data should be recorded from now on */
 662         mAudioManager.stop_recording(true);
 663         mAudioManager.clear_audio_data();
 664         change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE);
 665         mAudioManager.set_recording_session(RECORDING_SESSION_FOLLOW_UP);
 666         mAudioManager.start_recording(true);
 667
 668         mAudioManager.start_streaming_follow_up_data();
 669
 670         ecore_thread_main_loop_begin();
 671         if (mStreamingDurationTimer) {
 672                 ecore_timer_del(mStreamingDurationTimer);
 673                 mStreamingDurationTimer = nullptr;
 674         }
 675         mStreamingDurationTimer = ecore_timer_add(
 676                 mWakeupSettings.get_streaming_duration_max(),
 677                 streaming_duration_expired, this);
 678         ecore_thread_main_loop_end();
 679
 680         MWR_LOGD("[END]");
 681         return true;
 682 }
 683
 684 bool CWakeupManager::stop_streaming_follow_up_data()
 685 {
 686         MWR_LOGD("[ENTER]");
 687         if (STREAMING_MODE::FOLLOW_UP != mStreamingMode) return false;
 688
 689         if (mStreamingDurationTimer) {
 690                 ecore_thread_main_loop_begin();
 691                 ecore_timer_del(mStreamingDurationTimer);
 692                 mStreamingDurationTimer = nullptr;
 693                 ecore_thread_main_loop_end();
 694         }
 695         if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 696                 change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 697         }
 698         mAudioManager.stop_streaming_follow_up_data();
 699         mWakeupEngineManager.stop_streaming_current_utterance_data();
 700
 701         mStreamingMode = STREAMING_MODE::NONE;
 702
 703         mAudioManager.stop_recording(true);
 704         mAudioManager.clear_audio_data();
 705         mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 706         mAudioManager.start_recording(true);
 707
 708         MWR_LOGD("[END]");
 709         return true;
 710 }
 711
 712 bool CWakeupManager::start_streaming_previous_utterance_data()
 713 {
 714         MWR_LOGD("[ENTER]");
 715
 716         mAudioManager.stop_streaming_previous_utterance_data();
 717         mWakeupEngineManager.stop_streaming_current_utterance_data();
 718
 719         mStreamingMode = STREAMING_MODE::PREVIOUS_UTTERANCE;
 720         mAudioManager.start_streaming_previous_utterance_data();
 721
 722         ecore_thread_main_loop_begin();
 723         if (mStreamingDurationTimer) {
 724                 ecore_timer_del(mStreamingDurationTimer);
 725                 mStreamingDurationTimer = nullptr;
 726         }
 727         mStreamingDurationTimer = ecore_timer_add(
 728                 mWakeupSettings.get_streaming_duration_max(),
 729                 streaming_duration_expired, this);
 730         ecore_thread_main_loop_end();
 731
 732         MWR_LOGD("[END]");
 733         return true;
 734 }
 735
 736 bool CWakeupManager::stop_streaming_previous_utterance_data()
 737 {
 738         MWR_LOGD("[ENTER]");
 739         if (STREAMING_MODE::PREVIOUS_UTTERANCE != mStreamingMode) return false;
 740
 741         if (mStreamingDurationTimer) {
 742                 ecore_thread_main_loop_begin();
 743                 ecore_timer_del(mStreamingDurationTimer);
 744                 mStreamingDurationTimer = nullptr;
 745                 ecore_thread_main_loop_end();
 746         }
 747         if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 748                 change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 749         }
 750         mAudioManager.stop_streaming_previous_utterance_data();
 751         mWakeupEngineManager.stop_streaming_current_utterance_data();
 752
 753         mStreamingMode = STREAMING_MODE::NONE;
 754
 755         MWR_LOGD("[END]");
 756         return true;
 757 }
 758
 759 bool CWakeupManager::get_audio_format(int* rate, int* channel, int* audio_type)
 760 {
 761         MWR_LOGD("[ENTER]");
 762
 763         if (!audio_type || !rate || !channel) {
 764                 MWR_LOGE("[ERROR] Invalid parameter");
 765                 return false;
 766         }
 767
 768         dependency_resolver_get_audio_format(rate, channel, audio_type);
 769
 770         MWR_LOGD("[END] rate(%d), channel(%d), audio_type(%d)", *rate, *channel, *audio_type);
 771         return true;
 772 }
 773
 774 bool CWakeupManager::get_audio_source_type(char** type)
 775 {
 776         MWR_LOGD("[ENTER]");
 777
 778         if (!type) {
 779                 MWR_LOGE("[ERROR] Invalid parameter");
 780                 return false;
 781         }
 782
 783         dependency_resolver_get_audio_source_type(type);
 784
 785         MWR_LOGD("[END] type(%s)", *type);
 786         return true;
 787 }
 788
 789 bool CWakeupManager::set_wake_word_audio_require_flag(bool require)
 790 {
 791         MWR_LOGD("[ENTER]");
 792
 793         mWakeupEngineManager.set_wake_word_audio_require_flag(require);
 794
 795         MWR_LOGD("[END]");
 796         return true;
 797 }
 798
 799 CWakeupPolicy* CWakeupManager::get_wakeup_policy()
 800 {
 801         return mWakeupPolicy.get();
 802 }
 803
 804 CWakeupEngineManager* CWakeupManager::get_engine_manager()
 805 {
 806         return &mWakeupEngineManager;
 807 }
 808
 809 CAudioManager* CWakeupManager::get_audio_manager()
 810 {
 811         return &mAudioManager;
 812 }
 813
 814 CWakeupSettings* CWakeupManager::get_wakeup_settings()
 815 {
 816         return &mWakeupSettings;
 817 }
 818
 819 void CWakeupManager::feed_audio_data(mas_speech_streaming_event_e event, void* buffer, int len)
 820 {
 821         mAudioManager.feed_audio_data(event, buffer, len);
 822 }
 823
 824 void CWakeupManager::send_dependency_module_command(string engine_name, string command)
 825 {
 826         mWakeupEngineManager.engine_set_dependency_module_command(engine_name, command);
 827 }
 828
 829 bool CWakeupManager::CEngineEventObserver::on_wakeup_event(string engine_name, mas_wakeup_event_info wakeup_info)
 830 {
 831         MWR_LOGD("[ENTER]");
 832         if (nullptr == mWakeupManager) return false;
 833         if (nullptr == wakeup_info.wakeup_appid) return false;
 834
 835         if (0 != mWakeupManager->get_default_assistant().compare(wakeup_info.wakeup_appid)) {
 836                 if (false == mWakeupManager->get_assistant_enabled(string{wakeup_info.wakeup_appid})) {
 837                         MWR_LOGE("Wakeup event with deactivated appid : %s", wakeup_info.wakeup_appid);
 838                         return false;
 839                 }
 840         }
 841
 842         CWakeupPolicy* policy = mWakeupManager->get_wakeup_policy();
 843         if (policy) {
 844                 policy->wakeup_candidate(wakeup_info);
 845         }
 846         return true;
 847 }
 848
 849 bool CWakeupManager::CEngineEventObserver::on_speech_status(string engine_name, mas_speech_status_e status)
 850 {
 851         MWR_LOGD("[ENTER]");
 852         if (nullptr == mWakeupManager) return false;
 853
 854         return true;
 855 }
 856
 857 bool CWakeupManager::CEngineEventObserver::on_error(string engine_name, int error_code, string error_message)
 858 {
 859         MWR_LOGD("[ENTER]");
 860         if (nullptr == mWakeupManager) return false;
 861
 862         return true;
 863 }
 864
 865 bool CWakeupManager::CEngineEventObserver::on_audio_data_require_status(string engine_name, bool require)
 866 {
 867         MWR_LOGD("[ENTER]");
 868         if (nullptr == mWakeupManager) return false;
 869         if (WAKEUP_MANAGER_STATE_INACTIVE == mWakeupManager->get_manager_state()) return false;
 870
 871         CAudioManager *audio_manager = mWakeupManager->get_audio_manager();
 872         CWakeupEngineManager *engine_manager = mWakeupManager->get_engine_manager();
 873
 874         if (audio_manager && engine_manager) {
 875                 if (engine_manager->get_audio_data_required()) {
 876                         if (mWakeupManager->get_voice_key_pressed() != true) {
 877                                 audio_manager->set_recording_session(RECORDING_SESSION_WAKE_WORD);
 878                                 audio_manager->start_recording(true);
 879                         }
 880                 } else {
 881                         if (mWakeupManager->get_voice_key_pressed() != true) {
 882                                 audio_manager->stop_recording(true);
 883                         }
 884                 }
 885         }
 886         return true;
 887 }
 888
 889 bool CWakeupManager::CEngineEventObserver::on_streaming_audio_data(
 890         mas_speech_streaming_event_e event, void* buffer, unsigned int len)
 891 {
 892         if (nullptr == mWakeupManager) return false;
 893
 894         vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
 895         for (const auto& observer : observers) {
 896                 observer->on_streaming_audio_data(event, buffer, len);
 897         }
 898         if (MAS_SPEECH_STREAMING_EVENT_FINISH == event) {
 899                 mWakeupManager->set_streaming_mode(STREAMING_MODE::NONE);
 900         }
 901
 902         return true;
 903 }
 904
 905 bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section(
 906         ma_audio_streaming_data_section_e section)
 907 {
 908         if (nullptr == mWakeupManager) return false;
 909
 910         vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
 911         for (const auto& observer : observers) {
 912                 observer->on_audio_streaming_data_section(section);
 913         }
 914
 915         return true;
 916 }
 917
 918 bool CWakeupManager::CEngineEventObserver::on_wakeup_engine_command(
 919         mas_wakeup_engine_command_target_e target, string engine_name, string assistant_name, string command)
 920 {
 921         if (nullptr == mWakeupManager) return false;
 922
 923         if (MAS_WAKEUP_ENGINE_COMMAND_TARGET_DEPENDENCY_MODULE == target) {
 924                 dependency_resolver_process_wakeup_engine_command(engine_name.c_str(), command.c_str());
 925         }
 926
 927         if (MAS_WAKEUP_ENGINE_COMMAND_TARGET_ALL_ASSISTANTS == target ||
 928                 MAS_WAKEUP_ENGINE_COMMAND_TARGET_SPECIFIC_ASSISTANT == target) {
 929                 vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
 930                 for (const auto& observer : observers) {
 931                         observer->on_wakeup_engine_command(target, assistant_name.c_str(), command.c_str());
 932                 }
 933         }
 934
 935         return true;
 936 }
 937
 938 void CWakeupManager::CPolicyEventObserver::on_wakeup(mas_wakeup_event_info wakeup_info)
 939 {
 940         if (nullptr == mWakeupManager) return;
 941
 942         CAudioManager *audio_manager = mWakeupManager->get_audio_manager();
 943         CWakeupEngineManager *engine_manager = mWakeupManager->get_engine_manager();
 944         if (nullptr == audio_manager || nullptr == engine_manager) return;
 945
 946         mWakeupManager->stop_streaming_utterance_data();
 947         mWakeupManager->stop_streaming_previous_utterance_data();
 948         mWakeupManager->stop_streaming_follow_up_data();
 949         mWakeupManager->change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE);
 950
 951         mWakeupManager->set_last_wakeup_event_info(wakeup_info);
 952         engine_manager->set_selected_wakeup_info(wakeup_info);
 953         vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
 954         for (const auto& observer : observers) {
 955                 observer->on_wakeup(wakeup_info);
 956         }
 957
 958         audio_manager->set_recording_session(RECORDING_SESSION_UTTERANCE);
 959 }
 960
 961 bool CWakeupManager::CAudioEventObserver::on_recording_audio_data(long time, void* data, int len)
 962 {
 963         if (nullptr == mWakeupManager) return false;
 964         if (nullptr == mEngineManager) return false;
 965
 966         if (false == mEngineManager->get_audio_data_required()) return false;
 967
 968         if (mWakeupManager->get_voice_key_pressed() != true) {
 969                 /* When follow-up streaming in progress, no need to feed audio data to wakeup engines */
 970                 if (STREAMING_MODE::FOLLOW_UP != mWakeupManager->get_streaming_mode()) {
 971                         mEngineManager->engine_feed_audio_data(time, data, len);
 972                 }
 973         }
 974
 975         return true;
 976 }
 977
 978 bool CWakeupManager::CAudioEventObserver::on_streaming_audio_data(
 979         mas_speech_streaming_event_e event, void* buffer, unsigned int len)
 980 {
 981         if (nullptr == mWakeupManager) return false;
 982
 983         vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
 984         for (const auto& observer : observers) {
 985                 observer->on_streaming_audio_data(event, buffer, len);
 986         }
 987         if (MAS_SPEECH_STREAMING_EVENT_FINISH == event) {
 988                 mWakeupManager->set_streaming_mode(STREAMING_MODE::NONE);
 989         }
 990
 991         return true;
 992 }
 993
 994 bool CWakeupManager::CSettingsEventObserver::on_voice_input_language_changed(
 995         const char* language)
 996 {
 997         if (nullptr == mWakeupManager || nullptr == language) return false;
 998         mWakeupManager->set_language(std::string(language));
 999         vector<ISettingValueObserver*> observers = mWakeupManager->get_setting_observers();
1000         for (const auto& observer : observers) {
1001                 observer->on_value_changed();
1002         }
1003         return true;
1004 }
1005
1006 bool CWakeupManager::CSettingsEventObserver::on_assistant_enabled_info_changed(
1007         const char* appid, bool enabled)
1008 {
1009         if (nullptr == mWakeupManager || nullptr == appid) return false;
1010         mWakeupManager->set_assistant_enabled(std::string(appid), enabled);
1011         vector<ISettingValueObserver*> observers = mWakeupManager->get_setting_observers();
1012         for (const auto& observer : observers) {
1013                 observer->on_value_changed();
1014         }
1015         return true;
1016 }
1017
1018 bool CWakeupManager::CSettingsEventObserver::on_default_assistant_appid_changed(
1019         const char* appid)
1020 {
1021         if (nullptr == mWakeupManager || nullptr == appid) return false;
1022         mWakeupManager->set_default_assistant(std::string(appid));
1023         vector<ISettingValueObserver*> observers = mWakeupManager->get_setting_observers();
1024         for (const auto& observer : observers) {
1025                 observer->on_value_changed();
1026         }
1027         return true;
1028 }
1029
1030 } // wakeup
1031 } // multiassistant