plugins/wakeup-manager/src/wakeup_manager.cpp

   1 /*
   2  * Copyright 2018-2019 Samsung Electronics Co., Ltd
   3  *
   4  * Licensed under the Flora License, Version 1.1 (the License);
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://floralicense.org/license/
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an AS IS BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include "wakeup_manager.h"
  18 #include "wakeup_manager_main.h"
  19 #include "wakeup_policy_default.h"
  20 #include "wakeup_policy_external.h"
  21 #include "dependency_resolver.h"
  22
  23 #include <algorithm>
  24 #include <iostream>
  25
  26 #include <boost/optional.hpp>
  27
  28 namespace multiassistant
  29 {
  30 namespace wakeup
  31 {
  32
  33 static bool check_language_valid(string language)
  34 {
  35         bool valid = true;
  36         if (0 == language.length()) {
  37                 valid = false;
  38         }
  39         return valid;
  40 }
  41
  42 static bool initialize_wakeup_event_info(mas_wakeup_event_info* wakeup_info)
  43 {
  44         bool ret = false;
  45         if (wakeup_info) {
  46                 ret = true;
  47
  48                 wakeup_info->wakeup_appid = nullptr;
  49                 wakeup_info->wakeup_word = nullptr;
  50                 wakeup_info->wakeup_language = nullptr;
  51                 wakeup_info->wakeup_voice_id = nullptr;
  52                 wakeup_info->wakeup_engine = nullptr;
  53                 wakeup_info->wakeup_confidence_score = 0.0f;
  54
  55                 wakeup_info->wakeup_start_time = 0;
  56                 wakeup_info->wakeup_end_time = 0L;
  57                 wakeup_info->wakeup_time_valid = false;
  58
  59                 wakeup_info->extra_data = nullptr;
  60                 wakeup_info->extra_data_length = 0;
  61                 wakeup_info->extra_data_description = nullptr;
  62         }
  63         return ret;
  64 }
  65
  66 CWakeupManager::CWakeupManager(IWakeupEventObserver* wakeup_observer, ISettingValueObserver* setting_observer)
  67 {
  68         initialize_wakeup_event_info(&mLastWakeupEventInfo);
  69
  70         if (wakeup_observer) {
  71                 subscribe_wakeup_observer(wakeup_observer);
  72         }
  73         if (setting_observer) {
  74                 subscribe_setting_observer(setting_observer);
  75         }
  76 }
  77
  78 CWakeupManager::~CWakeupManager()
  79 {
  80         MWR_LOGI("Wakeup Manager is now being destroyed");
  81 }
  82
  83 void CWakeupManager::initialize_wakeup_policy()
  84 {
  85         mWakeupPolicy.reset(new CWakeupPolicyExternal{&mPolicyEventObserver});
  86         if (nullptr == mWakeupPolicy || !(mWakeupPolicy->valid())) {
  87                 mWakeupPolicy.reset(new CWakeupPolicyDefault{&mPolicyEventObserver});
  88
  89                 /* Default Policy specific initialization */
  90                 CWakeupPolicyDefault *policy =
  91                         dynamic_cast<CWakeupPolicyDefault*>(mWakeupPolicy.get());
  92                 if (policy) {
  93                         int priority = 0;
  94
  95                         policy->set_delay(mWakeupSettings.get_wakeup_policy_delay());
  96                         MWR_LOGD("Setting Delay : %f", mWakeupSettings.get_wakeup_policy_delay());
  97                         for (const auto& assistant : mWakeupSettings.get_wakeup_policy_priority()) {
  98                                 policy->set_assistant_priority(assistant, ++priority);
  99                                 MWR_LOGD("Setting Priority : %d %s", priority, assistant.c_str());
 100                         }
 101                 }
 102         }
 103 }
 104
 105 bool CWakeupManager::initialize()
 106 {
 107         MWR_LOGE("[ENTER]");
 108         std::cerr << "WakeupManager Initialize" << std::endl;
 109
 110         mPolicyEventObserver.set_wakeup_manager(this);
 111         mEngineEventObserver.set_wakeup_manager(this);
 112         mAudioEventObserver.set_wakeup_manager(this);
 113         mSettingsEventObserver.set_wakeup_manager(this);
 114
 115         mWakeupSettings.subscribe(&mSettingsEventObserver);
 116         mWakeupSettings.initialize();
 117
 118         mAudioEventObserver.set_wakeup_engine_manager(&mWakeupEngineManager);
 119         mAudioManager.subscribe(&mAudioEventObserver);
 120         mAudioManager.initialize();
 121
 122         mWakeupEngineManager.subscribe(&mEngineEventObserver);
 123         mWakeupEngineManager.initialize();
 124
 125         mAssistantConfigManager.initialize();
 126
 127         mas_dependency_plugin_proxy_interface interface;
 128         interface.process_event = wakeup_manager_process_plugin_event;
 129         interface.feed_audio_data = wakeup_manager_feed_audio_data;
 130         interface.send_command = wakeup_manager_set_dependency_module_command;
 131         interface.wakeup_assistant = wakeup_manager_wakeup_assistant;
 132
 133         dependency_resolver_initialize(interface);
 134
 135         initialize_wakeup_policy();
 136         start_periodic_monitor_timer();
 137
 138         MWR_LOGD("[END]");
 139         return true;
 140 }
 141
 142 bool CWakeupManager::deinitialize()
 143 {
 144         MWR_LOGE("[ENTER]");
 145         std::cerr << "WakeupManager Deinitialize" << std::endl;
 146
 147         stop_periodic_monitor_timer();
 148         stop_streaming_duration_timer();
 149
 150         dependency_resolver_deinitialize();
 151
 152         mAssistantConfigManager.deinitialize();
 153
 154         mWakeupEngineManager.unsubscribe(&mEngineEventObserver);
 155         mWakeupEngineManager.deinitialize();
 156
 157         mAudioManager.unsubscribe(&mAudioEventObserver);
 158         mAudioManager.deinitialize();
 159
 160         mWakeupSettings.deinitialize();
 161         mAssistantLanguageInfo.clear();
 162
 163         MWR_LOGE("[END]");
 164         return true;
 165 }
 166
 167 void CWakeupManager::subscribe_wakeup_observer(IWakeupEventObserver *observer)
 168 {
 169         mWakeupObservers.push_back(observer);
 170 }
 171
 172 void CWakeupManager::unsubscribe_wakeup_observer(IWakeupEventObserver *observer)
 173 {
 174         auto iter = find(mWakeupObservers.begin(), mWakeupObservers.end(), observer);
 175         if (iter != mWakeupObservers.end()) {
 176                 mWakeupObservers.erase(iter);
 177         }
 178 }
 179
 180 void CWakeupManager::subscribe_setting_observer(ISettingValueObserver* observer)
 181 {
 182         mSettingObservers.push_back(observer);
 183 }
 184
 185 void CWakeupManager::unsubscribe_setting_observer(ISettingValueObserver* observer)
 186 {
 187         auto iter = find(mSettingObservers.begin(), mSettingObservers.end(), observer);
 188         if (iter != mSettingObservers.end()) {
 189                 mSettingObservers.erase(iter);
 190         }
 191 }
 192
 193 bool CWakeupManager::activate(void)
 194 {
 195         MWR_LOGI("[ENTER]");
 196
 197         if (WAKEUP_MANAGER_STATE_INACTIVE != mWakeupManagerState)
 198                 return false;
 199
 200         /* Activate assistants that supports current voice input language */
 201         set_language(mWakeupSettings.get_current_language());
 202
 203         change_manager_state(WAKEUP_MANAGER_STATE_LISTENING);
 204         if (mWakeupEngineManager.get_audio_data_required()) {
 205                 mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 206                 mAudioManager.start_recording(true);
 207         }
 208
 209         MWR_LOGD("[END]");
 210         return true;
 211 }
 212
 213 bool CWakeupManager::deactivate(void)
 214 {
 215         MWR_LOGI("[ENTER]");
 216
 217         if (WAKEUP_MANAGER_STATE_INACTIVE == mWakeupManagerState)
 218                 return false;
 219
 220         mAudioManager.stop_recording(true);
 221         mRecordingByVoiceKey = false;
 222         change_manager_state(WAKEUP_MANAGER_STATE_INACTIVE);
 223
 224         stop_streaming_utterance_data();
 225         stop_streaming_previous_utterance_data();
 226         stop_streaming_follow_up_data();
 227
 228         MWR_LOGD("[END]");
 229         return true;
 230 }
 231
 232 bool CWakeupManager::add_assistant_language(string appid, string language)
 233 {
 234         MWR_LOGI("[ENTER]");
 235         bool found = false;
 236         for (auto& info : mAssistantLanguageInfo) {
 237                 if(0 == info.appid.compare(appid)) {
 238                         info.languageList.push_back(language);
 239                         found = true;
 240                 }
 241         }
 242         if(false == found) {
 243                 AssistantLanguageInfo info;
 244                 info.appid = appid;
 245                 info.languageList.push_back(language);
 246                 mAssistantLanguageInfo.push_back(info);
 247         }
 248         if (0 == mCurrentLanguage.compare(language)) {
 249                 mAssistantSupportsCurrentLanguage[appid] = true;
 250                 if (true == mAssistantEnabled[appid]) {
 251                         mWakeupEngineManager.set_assistant_activated(appid, true);
 252                 }
 253         }
 254         MWR_LOGD("[END]");
 255         return true;
 256 }
 257
 258 bool CWakeupManager::add_assistant_wakeup_word(string appid, string wakeup_word, string language)
 259 {
 260         MWR_LOGI("[ENTER]");
 261
 262         mWakeupEngineManager.engine_add_wakeup_word(appid, wakeup_word, language);
 263
 264         MWR_LOGD("[END]");
 265         return true;
 266 }
 267
 268 bool CWakeupManager::remove_assistant_wakeup_word(string appid, string wakeup_word, string language)
 269 {
 270         MWR_LOGI("[ENTER]");
 271
 272         mWakeupEngineManager.engine_remove_wakeup_word(appid, wakeup_word, language);
 273
 274         MWR_LOGD("[END]");
 275         return true;
 276 }
 277
 278 bool CWakeupManager::add_assistant_wakeup_engine(string appid, string engine)
 279 {
 280         MWR_LOGI("[ENTER] %s %s", appid.c_str(), engine.c_str());
 281
 282         mWakeupEngineManager.engine_add_target_assistant(engine, appid);
 283
 284         MWR_LOGD("[END]");
 285         return true;
 286 }
 287
 288 bool CWakeupManager::set_assistant_language(string appid, string language)
 289 {
 290         MWR_LOGI("[ENTER] : %s, %s", appid.c_str(), language.c_str());
 291
 292         mWakeupEngineManager.set_assistant_language(appid, language);
 293         mAssistantConfigManager.set_assistant_language(appid, language);
 294
 295         MWR_LOGD("[END]");
 296         return true;
 297 }
 298
 299 bool CWakeupManager::set_assistant_enabled(string appid, bool enabled)
 300 {
 301         MWR_LOGI("[ENTER]");
 302
 303         mAssistantEnabled[appid] = enabled;
 304         bool activated = enabled;
 305         if (false == mAssistantSupportsCurrentLanguage[appid]) {
 306                 activated = false;
 307         }
 308         if (0 == appid.compare(mWakeupSettings.get_default_assistant_appid())) {
 309                 activated = true;
 310         }
 311         mWakeupEngineManager.set_assistant_activated(appid, activated);
 312
 313         MWR_LOGD("[END]");
 314         return true;
 315 }
 316
 317 bool CWakeupManager::set_default_assistant(string appid)
 318 {
 319         MWR_LOGE("[ENTER] %s %s", appid.c_str(), mCurrentDefaultAssistant.c_str());
 320
 321         /* Passing 'expected' as true since MAS is changing the default assistant config */
 322         process_default_assistant_changed(appid, true);
 323
 324         mWakeupSettings.set_default_assistant_appid(appid);
 325
 326         return true;
 327 }
 328
 329 bool CWakeupManager::process_default_assistant_changed(string appid, bool expected)
 330 {
 331         MWR_LOGE("[ENTER] %s %s", appid.c_str(), mCurrentDefaultAssistant.c_str());
 332
 333         if (mCurrentDefaultAssistant.compare(appid) == 0) {
 334                 MWR_LOGE("Default assistant appid not changed, ignoring...");
 335                 return false;
 336         }
 337
 338         if (mWakeupManagerState == WAKEUP_MANAGER_STATE_UTTERANCE ||
 339                 mWakeupManagerState == WAKEUP_MANAGER_STATE_PROCESSING) {
 340                 if (expected) {
 341                         stop_streaming_utterance_data();
 342                         stop_streaming_previous_utterance_data();
 343                         stop_streaming_follow_up_data();
 344
 345                         mWakeupEngineManager.update_recognition_result(appid, MA_RECOGNITION_RESULT_EVENT_ERROR);
 346                 } else {
 347                         /* If this is an unexpected change, invalidate previous recognition process */
 348                         update_recognition_result(mCurrentDefaultAssistant, MA_RECOGNITION_RESULT_EVENT_ERROR);
 349                 }
 350         }
 351
 352         /* Check if previous default assistant has to be deactivated */
 353         bool activated = true;
 354         if (false == mAssistantSupportsCurrentLanguage[mCurrentDefaultAssistant]) {
 355                 activated = false;
 356         }
 357         if (false == mAssistantEnabled[mCurrentDefaultAssistant]) {
 358                 activated = false;
 359         }
 360         mWakeupEngineManager.set_assistant_activated(mCurrentDefaultAssistant, activated);
 361
 362         /* New default assistant has to be activated no matter what */
 363         mWakeupEngineManager.set_assistant_activated(appid, true);
 364
 365         mCurrentDefaultAssistant = appid;
 366
 367         MWR_LOGD("[END]");
 368         return true;
 369 }
 370
 371 string CWakeupManager::get_default_assistant()
 372 {
 373         return mWakeupSettings.get_default_assistant_appid();
 374 }
 375
 376 bool CWakeupManager::get_assistant_enabled(string appid)
 377 {
 378         return mAssistantEnabled[appid];
 379 }
 380
 381 bool CWakeupManager::set_language(string language)
 382 {
 383         bool valid = false;
 384         MWR_LOGI("[ENTER] : %s", language.c_str());
 385
 386         if (check_language_valid(language)) {
 387                 mCurrentLanguage = language;
 388                 valid = true;
 389         } else {
 390                 MWR_LOGE("[ERROR] Invalid language (%s)", language.c_str());
 391         }
 392
 393         for (auto& info : mAssistantLanguageInfo) {
 394                 bool disable = false;
 395                 if (valid) {
 396                         bool supported = false;
 397                         if (info.languageList.end() !=
 398                                 find(info.languageList.begin(), info.languageList.end(), language)) {
 399                                 supported = true;
 400                         }
 401                         mAssistantSupportsCurrentLanguage[info.appid] = supported;
 402                         /* Disable this assistant if language not supported */
 403                         if (!supported) disable = true;
 404                 } else {
 405                         /* If current language is not valid, assume all languages support it */
 406                         mAssistantSupportsCurrentLanguage[info.appid] = true;
 407                 }
 408
 409                 if (false == mAssistantEnabled[info.appid]) {
 410                         disable = true;
 411                 }
 412                 if (0 == info.appid.compare(mWakeupSettings.get_default_assistant_appid())) {
 413                         /* Default Assistant should be activated no matter what */
 414                         disable = false;
 415                 }
 416                 mWakeupEngineManager.set_assistant_activated(info.appid, !disable);
 417                 std::string assistant_language;
 418                 int ret = mAssistantConfigManager.get_assistant_language(info.appid, assistant_language);
 419                 if (0 != ret || !check_language_valid(assistant_language)) {
 420                         mWakeupEngineManager.set_assistant_language(info.appid, language);
 421                 } else {
 422                         mWakeupEngineManager.set_assistant_language(info.appid, assistant_language);
 423                 }
 424         }
 425
 426         MWR_LOGD("[END]");
 427         return valid;
 428 }
 429
 430 bool CWakeupManager::get_recording_by_voice_key()
 431 {
 432         return mRecordingByVoiceKey;
 433 }
 434
 435 void CWakeupManager::set_recording_by_voice_key(bool recording)
 436 {
 437         mRecordingByVoiceKey = recording;
 438 }
 439
 440 STREAMING_MODE CWakeupManager::get_streaming_mode()
 441 {
 442         return mStreamingMode;
 443 }
 444
 445 bool CWakeupManager::set_streaming_mode(STREAMING_MODE mode)
 446 {
 447         lock_guard<mutex> lock(mMutex);
 448         mStreamingMode = mode;
 449         return true;
 450 }
 451
 452 bool CWakeupManager::change_manager_state(wakeup_manager_state_e state)
 453 {
 454         MWR_LOGI("[ENTER] : %d", state);
 455         mWakeupManagerState = state;
 456         mWakeupEngineManager.update_manager_state(state);
 457         for (const auto& observer : mWakeupObservers) {
 458                 observer->on_wakeup_service_state_changed((ma_service_state_e)state);
 459         }
 460         return true;
 461 }
 462
 463 wakeup_manager_state_e CWakeupManager::get_manager_state()
 464 {
 465         return mWakeupManagerState;
 466 }
 467
 468 bool CWakeupManager::update_voice_feedback_state(string appid, bool state)
 469 {
 470         MWR_LOGI("[ENTER]");
 471
 472         if (state) {
 473                 if (WAKEUP_MANAGER_STATE_LISTENING == mWakeupManagerState ||
 474                         WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState) {
 475                         change_manager_state(WAKEUP_MANAGER_STATE_VOICE_FEEDBACK);
 476                 }
 477         } else {
 478                 if (WAKEUP_MANAGER_STATE_VOICE_FEEDBACK == mWakeupManagerState) {
 479                         change_manager_state(WAKEUP_MANAGER_STATE_LISTENING);
 480                 }
 481         }
 482
 483         MWR_LOGD("[END]");
 484         return true;
 485 }
 486
 487 bool CWakeupManager::set_assistant_specific_command(string appid, string command)
 488 {
 489         MWR_LOGI("[ENTER]");
 490
 491         /*
 492         static const string voice_key_pressed{"voice_key_pressed"};
 493         static const string voice_key_released{"voice_key_released"};
 494
 495         if (0 == command.compare(voice_key_pressed)) {
 496                 process_event(MA_PLUGIN_EVENT_VOICE_KEY_PRESSED, NULL, 0);
 497         } else if (0 == command.compare(voice_key_released)) {
 498                 process_event(MA_PLUGIN_EVENT_VOICE_KEY_RELEASED, NULL, 0);
 499         }
 500         */
 501
 502         mWakeupEngineManager.engine_set_assistant_specific_command(appid, command);
 503
 504         MWR_LOGD("[END]");
 505         return true;
 506 }
 507
 508 bool CWakeupManager::set_background_volume(string appid, double ratio)
 509 {
 510         MWR_LOGD("[DEBUG] set background volume (%f)", ratio);
 511
 512         int ret = 0;
 513         mAudioManager.set_background_volume(ratio);
 514         return ret;
 515 }
 516
 517 bool CWakeupManager::update_recognition_result(string appid, int result)
 518 {
 519         MWR_LOGI("[ENTER]");
 520
 521         stop_streaming_utterance_data();
 522         stop_streaming_previous_utterance_data();
 523         stop_streaming_follow_up_data();
 524
 525         mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 526         mWakeupEngineManager.update_recognition_result(appid, result);
 527         if (WAKEUP_MANAGER_STATE_PROCESSING == mWakeupManagerState ||
 528                 WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 529                 change_manager_state(WAKEUP_MANAGER_STATE_LISTENING);
 530         }
 531         MWR_LOGD("[END]");
 532         return true;
 533 }
 534
 535 static long long get_current_milliseconds_after_epoch()
 536 {
 537         auto now = chrono::steady_clock::now();
 538         auto now_ms = chrono::time_point_cast<chrono::milliseconds>(now);
 539         /* number of milliseconds since the epoch of system_clock */
 540         auto value = now_ms.time_since_epoch();
 541
 542         return value.count();
 543 }
 544
 545 bool CWakeupManager::process_plugin_event(mas_plugin_event_e event, void* data, int len)
 546 {
 547         MWR_LOGE("[ENTER] : %d", event);
 548         if (WAKEUP_MANAGER_STATE_INACTIVE == mWakeupManagerState)
 549                 return false;
 550
 551         bool start_recording = false;
 552         bool stop_recording = false;
 553
 554         boost::optional<ma_voice_key_status_e> next_voice_key_status;
 555         if (MAS_PLUGIN_EVENT_VOICE_KEY_PRESSED == event) {
 556                 if (STREAMING_MODE::FOLLOW_UP == mStreamingMode) {
 557                         MWR_LOGE("Voice key pressed, but currently streaming follow_up audio");
 558                 } else {
 559                         if (VOICE_KEY_SUPPORT_MODE_NONE != mCurrentVoiceKeySupportMode) {
 560                                 start_recording = true;
 561                         }
 562                 }
 563                 next_voice_key_status = MA_VOICE_KEY_STATUS_PRESSED;
 564         } else if (MAS_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_PUSH == event) {
 565                 if (VOICE_KEY_SUPPORT_MODE_PUSH_TO_TALK == mCurrentVoiceKeySupportMode ||
 566                         VOICE_KEY_SUPPORT_MODE_ALL == mCurrentVoiceKeySupportMode) {
 567                         stop_recording = true;
 568                 }
 569                 next_voice_key_status = MA_VOICE_KEY_STATUS_RELEASED_AFTER_PUSH;
 570         } else if (MAS_PLUGIN_EVENT_VOICE_KEY_RELEASED_AFTER_TAP == event) {
 571                 if (VOICE_KEY_SUPPORT_MODE_PUSH_TO_TALK == mCurrentVoiceKeySupportMode) {
 572                         stop_recording = true;
 573                 }
 574                 next_voice_key_status = MA_VOICE_KEY_STATUS_RELEASED_AFTER_TAP;
 575         }
 576
 577         if (start_recording) {
 578                 mRecordingByVoiceKey = true;
 579
 580                 mAudioManager.stop_recording(true);
 581
 582                 stop_streaming_utterance_data();
 583                 stop_streaming_previous_utterance_data();
 584                 stop_streaming_follow_up_data();
 585
 586                 mAudioManager.clear_audio_data();
 587                 change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE);
 588
 589                 /* Start recorder thread using appropriate recording device */
 590                 mAudioManager.set_recording_session(RECORDING_SESSION_UTTERANCE);
 591                 mAudioManager.start_recording(true);
 592
 593                 /* Wakeup default assistant */
 594                 /* TODO: apply conversation timeout for selecting assistant here */
 595                 mas_wakeup_event_info wakeup_info;
 596                 initialize_wakeup_event_info(&wakeup_info);
 597                 /* Make sure to use background data */
 598                 wakeup_info.wakeup_time_valid = true;
 599                 wakeup_info.wakeup_end_time = get_current_milliseconds_after_epoch();
 600                 wakeup_info.wakeup_engine = WAKEUP_ENGINE_VOICE_KEY;
 601
 602                 std::string default_assistant_appid = mWakeupSettings.get_default_assistant_appid();
 603                 wakeup_info.wakeup_appid = default_assistant_appid.c_str();
 604                 MWR_LOGD("wakeup_appid : %s", wakeup_info.wakeup_appid);
 605
 606                 set_last_wakeup_event_info(wakeup_info);
 607                 mWakeupEngineManager.set_selected_wakeup_info(wakeup_info);
 608                 for (const auto& observer : mWakeupObservers) {
 609                         observer->on_wakeup(wakeup_info);
 610                 }
 611         }
 612         if (stop_recording) {
 613                 mRecordingByVoiceKey = false;
 614
 615                 stop_streaming_duration_timer();
 616                 mAudioManager.finalize_audio_data();
 617
 618                 if (STREAMING_MODE::UTTERANCE == mStreamingMode) {
 619                         change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 620                 } else {
 621                         change_manager_state(WAKEUP_MANAGER_STATE_LISTENING);
 622                 }
 623
 624                 mAudioManager.stop_recording(true);
 625                 mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 626                 if (mWakeupEngineManager.get_audio_data_required()) {
 627                         /* Restart recorder thread using appropriate recording device */
 628                         mAudioManager.start_recording(true);
 629                 }
 630         }
 631
 632         if (next_voice_key_status) {
 633                 for (const auto& observer : mWakeupObservers) {
 634                         observer->on_voice_key_status_changed(*next_voice_key_status);
 635                 }
 636         }
 637
 638         MWR_LOGD("[END]");
 639         return true;
 640 }
 641
 642 vector<IWakeupEventObserver*> CWakeupManager::get_wakeup_observers()
 643 {
 644         return mWakeupObservers;
 645 }
 646
 647 void CWakeupManager::set_last_wakeup_event_info(mas_wakeup_event_info wakeup_info)
 648 {
 649         mLastWakeupEventInfo = wakeup_info;
 650 }
 651
 652 vector<ISettingValueObserver*> CWakeupManager::get_setting_observers()
 653 {
 654         return mSettingObservers;
 655 }
 656
 657 static Eina_Bool streaming_duration_expired(void *data)
 658 {
 659         MWR_LOGW("[ENTER]");
 660         CWakeupManager *wakeup_manager = static_cast<CWakeupManager*>(data);
 661         if (nullptr == wakeup_manager) return ECORE_CALLBACK_CANCEL;
 662
 663         wakeup_manager->set_streaming_duration_timer(nullptr);
 664
 665         CAudioManager *audio_manager = wakeup_manager->get_audio_manager();
 666         CWakeupEngineManager *engine_manager = wakeup_manager->get_engine_manager();
 667
 668         if (nullptr == audio_manager) return ECORE_CALLBACK_CANCEL;
 669         if (nullptr == engine_manager) return ECORE_CALLBACK_CANCEL;
 670
 671         switch(wakeup_manager->get_streaming_mode()) {
 672                 case STREAMING_MODE::UTTERANCE:
 673                         audio_manager->stop_streaming_current_utterance_data();
 674                         engine_manager->stop_streaming_current_utterance_data();
 675                 break;
 676                 case STREAMING_MODE::PREVIOUS_UTTERANCE:
 677                         audio_manager->stop_streaming_previous_utterance_data();
 678                 break;
 679                 case STREAMING_MODE::FOLLOW_UP:
 680                         audio_manager->stop_streaming_follow_up_data();
 681                         audio_manager->stop_recording(true);
 682                         wakeup_manager->set_recording_by_voice_key(false);
 683                         audio_manager->clear_audio_data();
 684                 break;
 685         }
 686
 687         wakeup_manager->set_streaming_mode(STREAMING_MODE::NONE);
 688
 689         if (WAKEUP_MANAGER_STATE_UTTERANCE == wakeup_manager->get_manager_state()) {
 690                 audio_manager->stop_recording(true);
 691                 wakeup_manager->set_recording_by_voice_key(false);
 692                 audio_manager->set_recording_session(RECORDING_SESSION_WAKE_WORD);
 693                 if (engine_manager->get_audio_data_required()) {
 694                         /* Restart recorder thread using appropriate recording device */
 695                         audio_manager->start_recording(true);
 696                 }
 697                 wakeup_manager->change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 698         }
 699
 700         return ECORE_CALLBACK_CANCEL;
 701 }
 702
 703 bool CWakeupManager::start_streaming_utterance_data()
 704 {
 705         MWR_LOGI("[ENTER]");
 706
 707         mAudioManager.stop_streaming_current_utterance_data();
 708         mWakeupEngineManager.stop_streaming_current_utterance_data();
 709
 710         mStreamingMode = STREAMING_MODE::UTTERANCE;
 711
 712         bool streaming_by_manager = true;
 713         if (false == mLastWakeupEventInfo.wakeup_time_valid) {
 714                 mWakeupEngineManager.start_streaming_current_utterance_data();
 715                 streaming_by_manager = false;
 716         } else {
 717                 mAudioManager.start_streaming_current_utterance_data(mLastWakeupEventInfo.wakeup_end_time);
 718         }
 719
 720         stop_streaming_duration_timer();
 721         if (streaming_by_manager) {
 722                 start_streaming_duration_timer();
 723         }
 724
 725         MWR_LOGD("[END]");
 726         return true;
 727 }
 728
 729 bool CWakeupManager::stop_streaming_utterance_data()
 730 {
 731         MWR_LOGI("[ENTER]");
 732
 733         if (STREAMING_MODE::UTTERANCE != mStreamingMode) return false;
 734
 735         mAudioManager.stop_streaming_current_utterance_data();
 736         mWakeupEngineManager.stop_streaming_current_utterance_data();
 737
 738         if (mStreamingDurationTimer) {
 739                 stop_streaming_duration_timer();
 740         }
 741         if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 742                 change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 743                 mAudioManager.stop_recording(true);
 744                 mRecordingByVoiceKey = false;
 745                 mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 746                 if (mWakeupEngineManager.get_audio_data_required()) {
 747                         /* Restart recorder thread using appropriate recording device */
 748                         mAudioManager.start_recording(true);
 749                 }
 750         }
 751
 752         mStreamingMode = STREAMING_MODE::NONE;
 753
 754         MWR_LOGD("[END]");
 755         return true;
 756 }
 757
 758 bool CWakeupManager::start_streaming_follow_up_data()
 759 {
 760         MWR_LOGI("[ENTER]");
 761
 762         mAudioManager.stop_streaming_follow_up_data();
 763         mWakeupEngineManager.stop_streaming_current_utterance_data();
 764
 765         mStreamingMode = STREAMING_MODE::FOLLOW_UP;
 766
 767         /* For the follow up streaming, audio data should be recorded from now on */
 768         mAudioManager.stop_recording(true);
 769         mAudioManager.clear_audio_data();
 770         change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE);
 771         mAudioManager.set_recording_session(RECORDING_SESSION_FOLLOW_UP);
 772         mAudioManager.start_recording(true);
 773
 774         mAudioManager.start_streaming_follow_up_data();
 775
 776         stop_streaming_duration_timer();
 777         start_streaming_duration_timer();
 778
 779         MWR_LOGD("[END]");
 780         return true;
 781 }
 782
 783 bool CWakeupManager::stop_streaming_follow_up_data()
 784 {
 785         MWR_LOGI("[ENTER]");
 786         if (STREAMING_MODE::FOLLOW_UP != mStreamingMode) return false;
 787
 788         mAudioManager.stop_streaming_follow_up_data();
 789         mWakeupEngineManager.stop_streaming_current_utterance_data();
 790
 791         if (mStreamingDurationTimer) {
 792                 stop_streaming_duration_timer();
 793         }
 794
 795         if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 796                 change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 797                 mAudioManager.stop_recording(true);
 798                 mRecordingByVoiceKey = false;
 799                 mAudioManager.set_recording_session(RECORDING_SESSION_WAKE_WORD);
 800                 if (mWakeupEngineManager.get_audio_data_required()) {
 801                         /* Restart recorder thread using appropriate recording device */
 802                         mAudioManager.start_recording(true);
 803                 }
 804         }
 805
 806         mStreamingMode = STREAMING_MODE::NONE;
 807
 808         mAudioManager.clear_audio_data();
 809
 810         MWR_LOGD("[END]");
 811         return true;
 812 }
 813
 814 bool CWakeupManager::start_streaming_previous_utterance_data()
 815 {
 816         MWR_LOGI("[ENTER]");
 817
 818         mAudioManager.stop_streaming_previous_utterance_data();
 819         mWakeupEngineManager.stop_streaming_current_utterance_data();
 820
 821         mStreamingMode = STREAMING_MODE::PREVIOUS_UTTERANCE;
 822         mAudioManager.start_streaming_previous_utterance_data();
 823
 824         stop_streaming_duration_timer();
 825         start_streaming_duration_timer();
 826
 827         MWR_LOGD("[END]");
 828         return true;
 829 }
 830
 831 bool CWakeupManager::stop_streaming_previous_utterance_data()
 832 {
 833         MWR_LOGI("[ENTER]");
 834         if (STREAMING_MODE::PREVIOUS_UTTERANCE != mStreamingMode) return false;
 835
 836         mAudioManager.stop_streaming_previous_utterance_data();
 837         mWakeupEngineManager.stop_streaming_current_utterance_data();
 838
 839         if (mStreamingDurationTimer) {
 840                 stop_streaming_duration_timer();
 841         }
 842
 843         if (WAKEUP_MANAGER_STATE_UTTERANCE == mWakeupManagerState) {
 844                 change_manager_state(WAKEUP_MANAGER_STATE_PROCESSING);
 845         }
 846
 847         mStreamingMode = STREAMING_MODE::NONE;
 848
 849         MWR_LOGD("[END]");
 850         return true;
 851 }
 852
 853 bool CWakeupManager::get_audio_format(int* rate, int* channel, int* audio_type)
 854 {
 855         MWR_LOGI("[ENTER]");
 856
 857         if (!audio_type || !rate || !channel) {
 858                 MWR_LOGE("[ERROR] Invalid parameter");
 859                 return false;
 860         }
 861
 862         dependency_resolver_get_audio_format(rate, channel, audio_type);
 863
 864         MWR_LOGD("[END] rate(%d), channel(%d), audio_type(%d)", *rate, *channel, *audio_type);
 865         return true;
 866 }
 867
 868 bool CWakeupManager::get_audio_source_type(char** type)
 869 {
 870         MWR_LOGI("[ENTER]");
 871
 872         if (!type) {
 873                 MWR_LOGE("[ERROR] Invalid parameter");
 874                 return false;
 875         }
 876
 877         dependency_resolver_get_audio_source_type(type);
 878
 879         MWR_LOGD("[END] type(%s)", *type);
 880         return true;
 881 }
 882
 883 bool CWakeupManager::set_wake_word_audio_require_flag(bool require)
 884 {
 885         MWR_LOGI("[ENTER]");
 886
 887         mWakeupEngineManager.set_wake_word_audio_require_flag(require);
 888
 889         MWR_LOGD("[END]");
 890         return true;
 891 }
 892
 893 bool CWakeupManager::set_voice_key_tap_duration(float duration)
 894 {
 895         MWR_LOGI("[ENTER]");
 896
 897         dependency_resolver_set_voice_key_tap_duration(duration);
 898
 899         MWR_LOGD("[END]");
 900         return true;
 901 }
 902
 903 bool CWakeupManager::unset_voice_key_tap_duration()
 904 {
 905         MWR_LOGI("[ENTER]");
 906
 907         dependency_resolver_unset_voice_key_tap_duration();
 908
 909         MWR_LOGD("[END]");
 910         return true;
 911 }
 912
 913 bool CWakeupManager::set_voice_key_support_mode(VOICE_KEY_SUPPORT_MODE mode)
 914 {
 915         MWR_LOGI("Voice key support mode : %d", mode);
 916         mCurrentVoiceKeySupportMode = mode;
 917         return true;
 918 }
 919
 920 CWakeupPolicy* CWakeupManager::get_wakeup_policy()
 921 {
 922         return mWakeupPolicy.get();
 923 }
 924
 925 CWakeupEngineManager* CWakeupManager::get_engine_manager()
 926 {
 927         return &mWakeupEngineManager;
 928 }
 929
 930 CAudioManager* CWakeupManager::get_audio_manager()
 931 {
 932         return &mAudioManager;
 933 }
 934
 935 CWakeupSettings* CWakeupManager::get_wakeup_settings()
 936 {
 937         return &mWakeupSettings;
 938 }
 939
 940 void CWakeupManager::feed_audio_data(mas_speech_streaming_event_e event, void* buffer, int len)
 941 {
 942         const std::chrono::seconds interval(5);
 943         static auto last = std::chrono::steady_clock::now();
 944         auto now = std::chrono::steady_clock::now();
 945         if (now - last > interval) {
 946                 std::cerr << "Feeding Audio : " << len << std::endl;
 947                 last = now;
 948         }
 949
 950         mAudioManager.feed_audio_data(event, buffer, len);
 951 }
 952
 953 void CWakeupManager::set_dependency_module_command(string engine_name, string command)
 954 {
 955         mWakeupEngineManager.engine_set_dependency_module_command(engine_name, command);
 956 }
 957
 958 static Eina_Bool periodic_monitor_func(void *data)
 959 {
 960         std::cerr << "MAS PERIODIC HEALTH CHECK" << std::endl;
 961         return ECORE_CALLBACK_RENEW;
 962 }
 963
 964 void CWakeupManager::start_periodic_monitor_timer()
 965 {
 966         MWR_LOGI("MONITOR_TIMER START");
 967         mPeriodicMonitorTimer = ecore_timer_add(
 968                 30.0f,
 969                 periodic_monitor_func, nullptr);
 970 }
 971
 972 void CWakeupManager::stop_periodic_monitor_timer()
 973 {
 974         if (mPeriodicMonitorTimer) {
 975                 ecore_timer_del(mPeriodicMonitorTimer);
 976                 mPeriodicMonitorTimer = nullptr;
 977         }
 978 }
 979
 980 void CWakeupManager::start_streaming_duration_timer()
 981 {
 982         MWR_LOGI("DURATION_TIMER START");
 983         ecore_main_loop_thread_safe_call_async([](void* data) {
 984                 MWR_LOGI("DURATION_TIMER START - async");
 985                 CWakeupManager* manager = static_cast<CWakeupManager*>(data);
 986                 if (!manager) return;
 987
 988                 CWakeupSettings *settings = manager->get_wakeup_settings();
 989                 if (settings) {
 990                         Ecore_Timer* timer = ecore_timer_add(
 991                                 settings->get_streaming_duration_max(),
 992                                 streaming_duration_expired, manager);
 993                         manager->set_streaming_duration_timer(timer);
 994                         MWR_LOGI("DURATION_TIMER STARTED : %p", timer);
 995                 }
 996         }, this);
 997 }
 998
 999 void CWakeupManager::stop_streaming_duration_timer()
1000 {
1001         MWR_LOGI("DURATION_TIMER STOP");
1002         if (mStreamingDurationTimer) {
1003                 MWR_LOGI("DURATION_TIMER STOP - has timer");
1004                 ecore_main_loop_thread_safe_call_async([](void* data) {
1005                         MWR_LOGI("DURATION_TIMER STOP - async");
1006                         CWakeupManager* manager = static_cast<CWakeupManager*>(data);
1007                         if (!manager) return;
1008
1009                         Ecore_Timer* timer = manager->get_streaming_duration_timer();
1010                         void* ret = ecore_timer_del(timer);
1011                         MWR_LOGI("DURATION_TIMER EXISTS : %p %p", timer, ret);
1012                         manager->set_streaming_duration_timer(nullptr);
1013                 }, this);
1014         }
1015 }
1016
1017 void CWakeupManager::set_streaming_duration_timer(Ecore_Timer* timer)
1018 {
1019         MWR_LOGI("DURATION_TIMER SET : %p", timer);
1020         mStreamingDurationTimer = timer;
1021 }
1022
1023 Ecore_Timer* CWakeupManager::get_streaming_duration_timer()
1024 {
1025         return mStreamingDurationTimer;
1026 }
1027
1028 bool CWakeupManager::CEngineEventObserver::on_wakeup_event(string engine_name, mas_wakeup_event_info wakeup_info)
1029 {
1030         MWR_LOGI("[ENTER]");
1031         if (nullptr == mWakeupManager) return false;
1032         if (nullptr == wakeup_info.wakeup_appid) return false;
1033
1034         if (0 != mWakeupManager->get_default_assistant().compare(wakeup_info.wakeup_appid)) {
1035                 if (false == mWakeupManager->get_assistant_enabled(string{wakeup_info.wakeup_appid})) {
1036                         MWR_LOGE("Wakeup event with deactivated appid : %s", wakeup_info.wakeup_appid);
1037                         return false;
1038                 }
1039         }
1040
1041         CWakeupPolicy* policy = mWakeupManager->get_wakeup_policy();
1042         if (policy) {
1043                 policy->wakeup_candidate(wakeup_info);
1044         }
1045         return true;
1046 }
1047
1048 bool CWakeupManager::CEngineEventObserver::on_speech_status(string engine_name, mas_speech_status_e status)
1049 {
1050         MWR_LOGI("[ENTER]");
1051         if (nullptr == mWakeupManager) return false;
1052
1053         return true;
1054 }
1055
1056 bool CWakeupManager::CEngineEventObserver::on_error(string engine_name, int error_code, string error_message)
1057 {
1058         MWR_LOGI("[ENTER]");
1059         if (nullptr == mWakeupManager) return false;
1060
1061         return true;
1062 }
1063
1064 bool CWakeupManager::CEngineEventObserver::on_audio_data_require_status(string engine_name, bool require)
1065 {
1066         MWR_LOGI("[ENTER]");
1067         if (nullptr == mWakeupManager) return false;
1068         if (WAKEUP_MANAGER_STATE_INACTIVE == mWakeupManager->get_manager_state()) return false;
1069
1070         CAudioManager *audio_manager = mWakeupManager->get_audio_manager();
1071         CWakeupEngineManager *engine_manager = mWakeupManager->get_engine_manager();
1072
1073         if (audio_manager && engine_manager) {
1074                 if (engine_manager->get_audio_data_required()) {
1075                         if (mWakeupManager->get_recording_by_voice_key() != true) {
1076                                 audio_manager->set_recording_session(RECORDING_SESSION_WAKE_WORD);
1077                                 audio_manager->start_recording(true);
1078                         }
1079                 } else {
1080                         if (mWakeupManager->get_recording_by_voice_key() != true) {
1081                                 audio_manager->stop_recording(true);
1082                         }
1083                 }
1084         }
1085         return true;
1086 }
1087
1088 bool CWakeupManager::CEngineEventObserver::on_streaming_audio_data(
1089         mas_speech_streaming_event_e event, void* buffer, unsigned int len)
1090 {
1091         if (nullptr == mWakeupManager) return false;
1092
1093         vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
1094         for (const auto& observer : observers) {
1095                 observer->on_streaming_audio_data(event, buffer, len);
1096         }
1097
1098         return true;
1099 }
1100
1101 bool CWakeupManager::CEngineEventObserver::on_audio_streaming_data_section(
1102         ma_audio_streaming_data_section_e section)
1103 {
1104         if (nullptr == mWakeupManager) return false;
1105
1106         vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
1107         for (const auto& observer : observers) {
1108                 observer->on_audio_streaming_data_section(section);
1109         }
1110
1111         return true;
1112 }
1113
1114 bool CWakeupManager::CEngineEventObserver::on_wakeup_engine_command(
1115         mas_wakeup_engine_command_target_e target, string engine_name, string assistant_name, string command)
1116 {
1117         if (nullptr == mWakeupManager) return false;
1118
1119         if (MAS_WAKEUP_ENGINE_COMMAND_TARGET_DEPENDENCY_MODULE == target) {
1120                 dependency_resolver_process_wakeup_engine_command(engine_name.c_str(), command.c_str());
1121         }
1122
1123         if (MAS_WAKEUP_ENGINE_COMMAND_TARGET_ALL_ASSISTANTS == target ||
1124                 MAS_WAKEUP_ENGINE_COMMAND_TARGET_SPECIFIC_ASSISTANT == target) {
1125                 vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
1126                 for (const auto& observer : observers) {
1127                         observer->on_wakeup_engine_command(target, assistant_name.c_str(), command.c_str());
1128                 }
1129         }
1130
1131         return true;
1132 }
1133
1134 void CWakeupManager::CPolicyEventObserver::on_wakeup(mas_wakeup_event_info wakeup_info)
1135 {
1136         if (nullptr == mWakeupManager) return;
1137
1138         CAudioManager *audio_manager = mWakeupManager->get_audio_manager();
1139         CWakeupEngineManager *engine_manager = mWakeupManager->get_engine_manager();
1140         CWakeupSettings* settings = mWakeupManager->get_wakeup_settings();
1141         if (nullptr == audio_manager || nullptr == engine_manager || nullptr == settings) return;
1142
1143         if (wakeup_info.wakeup_appid && strlen(wakeup_info.wakeup_appid) > 0) {
1144                 mWakeupManager->set_default_assistant(wakeup_info.wakeup_appid);
1145         }
1146
1147         mWakeupManager->stop_streaming_utterance_data();
1148         mWakeupManager->stop_streaming_previous_utterance_data();
1149         mWakeupManager->stop_streaming_follow_up_data();
1150         mWakeupManager->change_manager_state(WAKEUP_MANAGER_STATE_UTTERANCE);
1151
1152         mWakeupManager->set_last_wakeup_event_info(wakeup_info);
1153         engine_manager->set_selected_wakeup_info(wakeup_info);
1154         vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
1155         for (const auto& observer : observers) {
1156                 observer->on_wakeup(wakeup_info);
1157         }
1158
1159         audio_manager->set_recording_session(RECORDING_SESSION_UTTERANCE);
1160 }
1161
1162 bool CWakeupManager::CAudioEventObserver::on_recording_audio_data(long time, void* data, int len)
1163 {
1164         if (nullptr == mWakeupManager) return false;
1165         if (nullptr == mEngineManager) return false;
1166
1167         if (false == mEngineManager->get_audio_data_required()) return false;
1168
1169         if (mWakeupManager->get_recording_by_voice_key() != true) {
1170                 /* When follow-up streaming in progress, no need to feed audio data to wakeup engines */
1171                 if (STREAMING_MODE::FOLLOW_UP != mWakeupManager->get_streaming_mode()) {
1172                         mEngineManager->engine_feed_audio_data(time, data, len);
1173                 }
1174         }
1175
1176         return true;
1177 }
1178
1179 bool CWakeupManager::CAudioEventObserver::on_streaming_audio_data(
1180         mas_speech_streaming_event_e event, void* buffer, unsigned int len)
1181 {
1182         if (nullptr == mWakeupManager) return false;
1183
1184         vector<IWakeupEventObserver*> observers = mWakeupManager->get_wakeup_observers();
1185         for (const auto& observer : observers) {
1186                 observer->on_streaming_audio_data(event, buffer, len);
1187         }
1188         if (MAS_SPEECH_STREAMING_EVENT_FINISH == event) {
1189                 mWakeupManager->set_streaming_mode(STREAMING_MODE::NONE);
1190                 mWakeupManager->stop_streaming_duration_timer();
1191         }
1192
1193         return true;
1194 }
1195
1196 bool CWakeupManager::CSettingsEventObserver::on_voice_input_language_changed(
1197         const char* language)
1198 {
1199         if (nullptr == mWakeupManager || nullptr == language) return false;
1200         mWakeupManager->set_language(std::string(language));
1201         vector<ISettingValueObserver*> observers = mWakeupManager->get_setting_observers();
1202         for (const auto& observer : observers) {
1203                 observer->on_value_changed();
1204         }
1205         return true;
1206 }
1207
1208 bool CWakeupManager::CSettingsEventObserver::on_assistant_enabled_info_changed(
1209         const char* appid, bool enabled)
1210 {
1211         if (nullptr == mWakeupManager || nullptr == appid) return false;
1212         mWakeupManager->set_assistant_enabled(std::string(appid), enabled);
1213         vector<ISettingValueObserver*> observers = mWakeupManager->get_setting_observers();
1214         for (const auto& observer : observers) {
1215                 observer->on_value_changed();
1216         }
1217         return true;
1218 }
1219
1220 bool CWakeupManager::CSettingsEventObserver::on_default_assistant_appid_changed(
1221         const char* appid)
1222 {
1223         if (nullptr == mWakeupManager || nullptr == appid) return false;
1224         /* Passing 'expected' as false since the change was occurred outside of MAS */
1225         mWakeupManager->process_default_assistant_changed(std::string(appid), false);
1226         vector<ISettingValueObserver*> observers = mWakeupManager->get_setting_observers();
1227         for (const auto& observer : observers) {
1228                 observer->on_value_changed();
1229         }
1230         return true;
1231 }
1232
1233 } // wakeup
1234 } // multiassistant